me-web? 0.5: the shared exit measured, the route from the server, the network's upstreams

- Behind your provider's shared address, me-web? now shows how its exit hands out ports
  (kept, one after another, from one block, or at random), the range it gave this device,
  and how many customers may share the address at most; the estimate sharpens with each
  check on the same address.
- It tests whether two devices behind the same address can reach each other (hairpin), and
  how long an unused connection stays open: the card says "measuring" for about two minutes
  after the check, then the answer.
- With the test from outside, the publisher's server also traces the route back to your
  address: your provider's routers and the networks in between appear on the path, with
  their names.
- The provider card says which networks your provider reaches the internet through.
- On mobile data, the path shows the operator and the type of network (5G, 4G) read from the
  phone; on Android 13 and later the phone asks once for permission to read the network type.
- The speed card adds the round trip to the publisher's server.
- The network data now comes from the publisher's server, refreshed there from open sources:
  iptoasn.com, DB-IP Lite (CC BY 4.0), the University of Oregon Route Views Project (CC BY 4.0)
  and the Android Open Source Project's list of mobile operators (Apache 2.0); the release
  notes list each source's date and checksum.
- Source links in the cards wrap instead of being cut off.
- New strings, drafted in every language.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB

tools/knowledge.cs

@@ -1,25 +1,53 @@
 #:project ../src/MeWeb.Core/MeWeb.Core.csproj
-using System.IO.Compression;
-using System.Net;
+#:project ../src/MeWeb.Shared/MeWeb.Shared.csproj
+using System.Net.Http.Json;
 using System.Security.Cryptography;
 
 using MeWeb.Core.Knowledge;
+using MeWeb.Shared;
 
-const string Source = "https://iptoasn.com/data/ip2asn-combined.tsv.gz";
+const string Fallback = "https://iptoasn.com/data/ip2asn-combined.tsv.gz";
 var root = FindRoot();
+var output = Path.Combine(root, "artifacts", "knowledge");
 var cache = Path.Combine(root, "tools", "cache");
-var output = Path.Combine(root, "artifacts", "knowledge", "ip2asn.bin");
+Directory.CreateDirectory(output);
 Directory.CreateDirectory(cache);
-Directory.CreateDirectory(Path.GetDirectoryName(output));
-var download = Path.Combine(cache, "ip2asn-combined.tsv.gz");
+var server = Environment.GetEnvironmentVariable("MEWEB_KNOWLEDGE_URL") ?? Endpoints.Base;
 
-using var http = new HttpClient();
-http.DefaultRequestHeaders.UserAgent.ParseAdd("me-web-knowledge/1 (+https://git.thepra.dev/Arasaka/MeWeb)");
-var fresh = !File.Exists(download) || File.GetLastWriteTimeUtc(download) < DateTime.UtcNow.AddHours(-12) || args.Contains("--refresh");
-var sourceDate = File.Exists(download) ? new DateTimeOffset(File.GetLastWriteTimeUtc(download), TimeSpan.Zero) : DateTimeOffset.UtcNow;
-if (fresh)
+using var http = new HttpClient { Timeout = TimeSpan.FromMinutes(2) };
+http.DefaultRequestHeaders.UserAgent.ParseAdd("me-web-knowledge/2 (+https://git.thepra.dev/Arasaka/MeWeb)");
+
+if (!args.Contains("--local"))
 {
-  using var response = await http.GetAsync(Source, HttpCompletionOption.ResponseHeadersRead);
+  try
+  {
+    var manifest = await http.GetFromJsonAsync(server + Endpoints.Knowledge.GET_Manifest, MeWebJsonContext.Default.ViewKnowledge);
+    if (manifest is { Files.Count: > 0 })
+    {
+      foreach (var file in manifest.Files)
+      {
+        var bytes = await http.GetByteArrayAsync(server + Endpoints.Knowledge.GET_M_File(file.Name));
+        var hash = Convert.ToHexStringLower(SHA256.HashData(bytes));
+        if (hash != file.Sha256) throw new InvalidDataException($"{file.Name}: sha256 {hash}, the manifest says {file.Sha256}");
+        File.WriteAllBytes(Path.Combine(output, file.Name), bytes);
+        Console.WriteLine($"knowledge: {file.Name} {bytes.Length / 1024} KiB from {server}");
+      }
+      File.Move(Path.Combine(output, "SOURCES.txt"), Path.Combine(output, "SOURCE.txt"), overwrite: true);
+      Report();
+      return 0;
+    }
+  }
+  catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or InvalidDataException or System.Text.Json.JsonException)
+  {
+    Console.Error.WriteLine($"knowledge: the server's files could not be used ({ex.Message}); building the provider map alone");
+  }
+}
+
+var download = Path.Combine(cache, "ip2asn-combined.tsv.gz");
+var sourceDate = File.Exists(download) ? new DateTimeOffset(File.GetLastWriteTimeUtc(download), TimeSpan.Zero) : DateTimeOffset.UtcNow;
+if (!File.Exists(download) || File.GetLastWriteTimeUtc(download) < DateTime.UtcNow.AddHours(-12) || args.Contains("--refresh"))
+{
+  using var response = await http.GetAsync(Fallback, HttpCompletionOption.ResponseHeadersRead);
   response.EnsureSuccessStatusCode();
   sourceDate = response.Content.Headers.LastModified ?? DateTimeOffset.UtcNow;
   await using (var file = File.Create(download + ".tmp"))
@@ -27,33 +55,27 @@ if (fresh)
   File.Move(download + ".tmp", download, overwrite: true);
   File.SetLastWriteTimeUtc(download, sourceDate.UtcDateTime);
 }
-
-var hash = Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(download)));
-var ranges = new List<AsnRange>();
-using (var reader = new StreamReader(new GZipStream(File.OpenRead(download), CompressionMode.Decompress)))
-{
-  string line;
-  while ((line = reader.ReadLine()) is not null)
-  {
-    var parts = line.Split('\t');
-    if (parts.Length < 5 || !uint.TryParse(parts[2], out var number) || number == 0) continue;
-    if (!IPAddress.TryParse(parts[0], out var start) || !IPAddress.TryParse(parts[1], out var end)) continue;
-    ranges.Add(new AsnRange { Start = start, End = end, Number = number, Country = parts[3] == "None" ? "" : parts[3], Name = parts[4] });
-  }
-}
-
-await using (var file = File.Create(output + ".tmp"))
-  AsnTable.Write(file, sourceDate, ranges);
-File.Move(output + ".tmp", output, overwrite: true);
-
-using var check = File.OpenRead(output);
-var table = AsnTable.Read(check);
-var probe = table.Find(IPAddress.Parse("1.1.1.1"));
-Console.WriteLine($"knowledge: {ranges.Count} routed ranges, {table.Count} networks, source {sourceDate:yyyy-MM-dd HH:mm} UTC, sha256 {hash}");
-Console.WriteLine($"knowledge: {output} {new FileInfo(output).Length / 1024} KiB; 1.1.1.1 -> AS{probe?.Number} {probe?.Name}");
-File.WriteAllText(Path.Combine(root, "artifacts", "knowledge", "SOURCE.txt"), $"{Source}\nlast-modified {sourceDate:O}\nsha256 {hash}\n");
+using (var input = File.OpenRead(download))
+using (var file = File.Create(Path.Combine(output, "ip2asn.bin")))
+  AsnTable.Write(file, sourceDate, Sources.IpToAsn(input));
+foreach (var stale in new[] { "networks.bin", "carriers.json" }) File.Delete(Path.Combine(output, stale));
+string sha;
+using (var input = File.OpenRead(download)) sha = Convert.ToHexStringLower(SHA256.HashData(input));
+File.WriteAllText(Path.Combine(output, "SOURCE.txt"), $"iptoasn.com, Public Domain Dedication and License (PDDL) 1.0\n{Fallback}\nlast-modified {sourceDate:O}\nsha256 {sha}\n");
+Report();
 return 0;
 
+void Report()
+{
+  using var stream = File.OpenRead(Path.Combine(output, "ip2asn.bin"));
+  var table = AsnTable.Read(stream);
+  var path = Path.Combine(output, "networks.bin");
+  NetworkTable networks = null;
+  if (File.Exists(path))
+    using (var input = File.OpenRead(path)) networks = NetworkTable.Read(input);
+  Console.WriteLine($"knowledge: {table.Count} networks in the provider map (source {table.SourceDate:yyyy-MM-dd}), {networks?.Count ?? 0} with names or upstreams");
+}
+
 static string FindRoot()
 {
   var dir = new DirectoryInfo(Directory.GetCurrentDirectory());

tools/release.cs

@@ -83,10 +83,10 @@ void WriteNotes()
   [
     $"me-web? {version}, commit {commit}, built {builtAt}.",
     "",
-    "Built with the provider map from iptoasn.com (PDDL 1.0):",
+    "Built with these open network datasets:",
     "",
   ]);
-  lines.AddRange(File.Exists(source) ? File.ReadAllLines(source).Where(l => l.Length > 0).Select(l => $"    {l}") : ["    (no provider map in this build)"]);
+  lines.AddRange(File.Exists(source) ? File.ReadAllLines(source).Select(l => l.Length > 0 ? $"    {l}" : "") : ["    (no network data in this build)"]);
   lines.AddRange(["", "SHA256SUMS.txt lists every file. The APK's signing certificate SHA-256 is 5f52cff4f52f4e0b6da6cd044bfceaf78d1d42db24f2470602db7f19cbfdf846."]);
   Directory.CreateDirectory(Path.Combine(root, "artifacts"));
   File.WriteAllLines(Path.Combine(root, "artifacts", "release-notes.md"), lines);