me-web? 0.6.0: provider notes from shared checks, and the internet seen from eight places

- Notes on providers in any country: findings people choose to share are counted on the publisher's server, one
  vote per network. A total appears on the provider card only after a person has reviewed it, and only when at
  least five different networks of that provider agree four times in five; the card says how many agree.
  Notes researched by hand still come first, and on mobile data the totals for the mobile network replace the
  notes about home lines.
- "Connected to the internet through" now comes from eight RouteViews collectors (Oregon, Ashburn, London,
  Sao Paulo, Johannesburg, Dubai, Singapore, Sydney) instead of one in Oregon, each counting the same, so
  networks far from the United States are described from nearby. A network that only exchanges traffic with a
  collector's neighbour is no longer listed as one of its upstreams.
- Ready for measuring servers outside Europe: once the publisher adds one, the app times each server and runs
  the speed test and the test from outside against the closest. None exists yet, so nothing changes today.
- When me-web? has no notes on a provider, it says that findings shared at the end of a check help write them.
- The privacy page states the rule for publishing totals.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB

src/MeWeb.Core/Knowledge/Observed.cs

@@ -0,0 +1,42 @@
+using System.Text.Json.Serialization;
+
+namespace MeWeb.Core.Knowledge;
+
+public class ObservedFact
+{
+  public const int MinNetworks = 5;
+
+  public string Value { get; set; }
+  public int Agree { get; set; }
+  public int Of { get; set; }
+
+  [JsonIgnore]
+  public bool Ready => Value is not null && Of >= MinNetworks && Agree * 5 >= Of * 4;
+
+  public static ObservedFact Tally(IEnumerable<string> values)
+  {
+    var counts = values.Where(v => v is not null).GroupBy(v => v).Select(g => (Value: g.Key, Count: g.Count())).ToList();
+    if (counts.Count == 0) return null;
+    var top = counts.OrderByDescending(c => c.Count).ThenBy(c => c.Value, StringComparer.Ordinal).First();
+    return new ObservedFact { Value = top.Value, Agree = top.Count, Of = counts.Sum(c => c.Count) };
+  }
+}
+
+public class ObservedNetwork
+{
+  public const string Home = "home";
+  public const string Mobile = "mobile";
+
+  public uint Asn { get; set; }
+  public string Line { get; set; }
+  public string Name { get; set; }
+  public int Networks { get; set; }
+  public string From { get; set; }
+  public string To { get; set; }
+  public ObservedFact Ipv4 { get; set; }
+  public ObservedFact Ipv6 { get; set; }
+  public ObservedFact Port25 { get; set; }
+
+  [JsonIgnore]
+  public string Key => $"AS{Asn}/{Line}";
+}

src/MeWeb.Core/Knowledge/Providers.cs

@@ -28,6 +28,7 @@ public class ProviderProfile
 [JsonSourceGenerationOptions(JsonSerializerDefaults.Web)]
 [JsonSerializable(typeof(ProviderProfile))]
 [JsonSerializable(typeof(List<Carrier>))]
+[JsonSerializable(typeof(List<ObservedNetwork>))]
 public partial class KnowledgeJsonContext : JsonSerializerContext
 {
 }
@@ -38,6 +39,7 @@ public sealed class KnowledgeBase
   readonly Lazy<List<ProviderProfile>> providers = new(LoadProviders);
   readonly Lazy<NetworkTable> networks = new(LoadNetworks);
   readonly Lazy<Dictionary<string, string>> carriers = new(LoadCarriers);
+  readonly Lazy<List<ObservedNetwork>> observed = new(LoadObserved);
 
   public bool HasAsnTable => typeof(KnowledgeBase).Assembly.GetManifestResourceInfo("ip2asn.bin") is not null;
 
@@ -47,6 +49,9 @@ public sealed class KnowledgeBase
 
   public ProviderProfile ProviderFor(uint number) => number == 0 ? null : Providers.FirstOrDefault(p => p.Asns.Contains(number));
 
+  public ObservedNetwork ObservedFor(uint number, bool mobile) =>
+      number == 0 ? null : observed.Value.FirstOrDefault(o => o.Asn == number && o.Line == (mobile ? ObservedNetwork.Mobile : ObservedNetwork.Home));
+
   public NetworkInfo NetworkFor(uint number) => number == 0 ? null : networks.Value?.Find(number);
 
   public string CarrierFor(string mccMnc) => string.IsNullOrEmpty(mccMnc) ? null : carriers.Value.GetValueOrDefault(mccMnc);
@@ -67,6 +72,12 @@ public sealed class KnowledgeBase
     return list.Where(c => c.MccMnc is not null && c.Name is not null).GroupBy(c => c.MccMnc).ToDictionary(g => g.Key, g => g.First().Name);
   }
 
+  static List<ObservedNetwork> LoadObserved()
+  {
+    using var stream = typeof(KnowledgeBase).Assembly.GetManifestResourceStream("observed.json");
+    return stream is null ? [] : JsonSerializer.Deserialize(stream, KnowledgeJsonContext.Default.ListObservedNetwork) ?? [];
+  }
+
   static AsnTable LoadAsn()
   {
     using var stream = typeof(KnowledgeBase).Assembly.GetManifestResourceStream("ip2asn.bin");

src/MeWeb.Core/Knowledge/Sources.cs

@@ -46,31 +46,6 @@ public static class Sources
     return seen.ToDictionary(p => p.Key, p => p.Value.MaxBy(n => n.Value).Key);
   }
 
-  public static Dictionary<uint, List<uint>> Upstreams(Stream mrt, int keep = 4, double share = 0.05)
-  {
-    var counts = new Dictionary<uint, Dictionary<uint, int>>();
-    var header = new byte[12];
-    var body = new byte[1 << 16];
-    while (Fill(mrt, header))
-    {
-      var type = BinaryPrimitives.ReadUInt16BigEndian(header.AsSpan(4));
-      var subtype = BinaryPrimitives.ReadUInt16BigEndian(header.AsSpan(6));
-      var length = (int)BinaryPrimitives.ReadUInt32BigEndian(header.AsSpan(8));
-      if (body.Length < length) body = new byte[length];
-      if (!Fill(mrt, body.AsSpan(0, length))) break;
-      if (type != 13 || subtype is not (2 or 4)) continue;
-      Rib(body.AsSpan(0, length), counts);
-    }
-    var result = new Dictionary<uint, List<uint>>();
-    foreach (var (origin, neighbours) in counts)
-    {
-      var total = (double)neighbours.Values.Sum();
-      var chosen = neighbours.Where(n => n.Value / total >= share).OrderByDescending(n => n.Value).ThenBy(n => n.Key).Take(keep).Select(n => n.Key).ToList();
-      if (chosen.Count > 0) result[origin] = chosen;
-    }
-    return result;
-  }
-
   public static List<Carrier> Carriers(TextReader textpb)
   {
     var list = new List<Carrier>();
@@ -113,82 +88,7 @@ public static class Sources
     return start >= 0 && end > start ? text[(start + 1)..end] : null;
   }
 
-  static void Rib(ReadOnlySpan<byte> rib, Dictionary<uint, Dictionary<uint, int>> counts)
-  {
-    if (rib.Length < 5) return;
-    var at = 4;
-    var bits = rib[at++];
-    at += (bits + 7) / 8;
-    if (at + 2 > rib.Length) return;
-    var entries = BinaryPrimitives.ReadUInt16BigEndian(rib[at..]);
-    at += 2;
-    for (var e = 0; e < entries && at + 8 <= rib.Length; e++)
-    {
-      at += 6;
-      var size = BinaryPrimitives.ReadUInt16BigEndian(rib[at..]);
-      at += 2;
-      if (at + size > rib.Length) return;
-      Attributes(rib.Slice(at, size), counts);
-      at += size;
-    }
-  }
-
-  static void Attributes(ReadOnlySpan<byte> attributes, Dictionary<uint, Dictionary<uint, int>> counts)
-  {
-    var at = 0;
-    while (at + 3 <= attributes.Length)
-    {
-      var flags = attributes[at];
-      var type = attributes[at + 1];
-      var extended = (flags & 0x10) != 0;
-      if (extended && at + 4 > attributes.Length) return;
-      var length = extended ? BinaryPrimitives.ReadUInt16BigEndian(attributes[(at + 2)..]) : attributes[at + 2];
-      var start = at + (extended ? 4 : 3);
-      if (start + length > attributes.Length) return;
-      if (type == 2)
-      {
-        Path(attributes.Slice(start, length), counts);
-        return;
-      }
-      at = start + length;
-    }
-  }
-
-  static void Path(ReadOnlySpan<byte> path, Dictionary<uint, Dictionary<uint, int>> counts)
-  {
-    uint last = 0;
-    uint before = 0;
-    var endsInSet = false;
-    var at = 0;
-    while (at + 2 <= path.Length)
-    {
-      var segment = path[at];
-      var count = path[at + 1];
-      at += 2;
-      if (at + count * 4 > path.Length) return;
-      for (var i = 0; i < count; i++)
-      {
-        var number = BinaryPrimitives.ReadUInt32BigEndian(path[(at + i * 4)..]);
-        if (segment != 2)
-        {
-          endsInSet = true;
-          continue;
-        }
-        endsInSet = false;
-        if (number == last) continue;
-        before = last;
-        last = number;
-      }
-      at += count * 4;
-    }
-    if (endsInSet || last == 0 || before == 0 || Reserved(before) || Reserved(last)) return;
-    if (!counts.TryGetValue(last, out var neighbours)) counts[last] = neighbours = [];
-    neighbours[before] = neighbours.GetValueOrDefault(before) + 1;
-  }
-
-  static bool Reserved(uint number) => number is 23456 or (>= 64496 and <= 131071) or >= 4200000000;
-
-  static bool Fill(Stream stream, Span<byte> buffer)
+  internal static bool Fill(Stream stream, Span<byte> buffer)
   {
     var read = 0;
     while (read < buffer.Length)

src/MeWeb.Core/Knowledge/UpstreamTally.cs

@@ -0,0 +1,127 @@
+using System.Buffers.Binary;
+
+namespace MeWeb.Core.Knowledge;
+
+public sealed class UpstreamTally
+{
+  readonly Dictionary<uint, Dictionary<uint, double>> weights = [];
+  readonly Dictionary<uint, int> seen = [];
+
+  public int Collectors { get; private set; }
+
+  public int Add(Stream mrt)
+  {
+    var counts = new Dictionary<uint, Dictionary<uint, int>>();
+    var header = new byte[12];
+    var body = new byte[1 << 16];
+    while (Sources.Fill(mrt, header))
+    {
+      var type = BinaryPrimitives.ReadUInt16BigEndian(header.AsSpan(4));
+      var subtype = BinaryPrimitives.ReadUInt16BigEndian(header.AsSpan(6));
+      var length = (int)BinaryPrimitives.ReadUInt32BigEndian(header.AsSpan(8));
+      if (body.Length < length) body = new byte[length];
+      if (!Sources.Fill(mrt, body.AsSpan(0, length))) break;
+      if (type != 13 || subtype is not (2 or 4 or 8 or 10)) continue;
+      Rib(body.AsSpan(0, length), subtype is 8 or 10, counts);
+    }
+    foreach (var (origin, neighbours) in counts)
+    {
+      var total = (double)neighbours.Values.Sum();
+      if (!weights.TryGetValue(origin, out var sum)) weights[origin] = sum = [];
+      foreach (var (neighbour, count) in neighbours) sum[neighbour] = sum.GetValueOrDefault(neighbour) + count / total;
+      seen[origin] = seen.GetValueOrDefault(origin) + 1;
+    }
+    Collectors++;
+    return counts.Count;
+  }
+
+  public Dictionary<uint, List<uint>> Upstreams(int keep = 4, double share = 0.05)
+  {
+    var result = new Dictionary<uint, List<uint>>();
+    foreach (var (origin, neighbours) in weights)
+    {
+      var collectors = (double)seen[origin];
+      var chosen = neighbours.Where(n => n.Value / collectors >= share).OrderByDescending(n => n.Value).ThenBy(n => n.Key).Take(keep).Select(n => n.Key).ToList();
+      if (chosen.Count > 0) result[origin] = chosen;
+    }
+    return result;
+  }
+
+  static void Rib(ReadOnlySpan<byte> rib, bool addPath, Dictionary<uint, Dictionary<uint, int>> counts)
+  {
+    if (rib.Length < 5) return;
+    var at = 4;
+    var bits = rib[at++];
+    at += (bits + 7) / 8;
+    if (at + 2 > rib.Length) return;
+    var entries = BinaryPrimitives.ReadUInt16BigEndian(rib[at..]);
+    at += 2;
+    var head = addPath ? 10 : 6;
+    for (var e = 0; e < entries && at + head + 2 <= rib.Length; e++)
+    {
+      at += head;
+      var size = BinaryPrimitives.ReadUInt16BigEndian(rib[at..]);
+      at += 2;
+      if (at + size > rib.Length) return;
+      Attributes(rib.Slice(at, size), counts);
+      at += size;
+    }
+  }
+
+  static void Attributes(ReadOnlySpan<byte> attributes, Dictionary<uint, Dictionary<uint, int>> counts)
+  {
+    var at = 0;
+    while (at + 3 <= attributes.Length)
+    {
+      var flags = attributes[at];
+      var type = attributes[at + 1];
+      var extended = (flags & 0x10) != 0;
+      if (extended && at + 4 > attributes.Length) return;
+      var length = extended ? BinaryPrimitives.ReadUInt16BigEndian(attributes[(at + 2)..]) : attributes[at + 2];
+      var start = at + (extended ? 4 : 3);
+      if (start + length > attributes.Length) return;
+      if (type == 2)
+      {
+        Path(attributes.Slice(start, length), counts);
+        return;
+      }
+      at = start + length;
+    }
+  }
+
+  static void Path(ReadOnlySpan<byte> path, Dictionary<uint, Dictionary<uint, int>> counts)
+  {
+    uint last = 0;
+    uint before = 0;
+    var distinct = 0;
+    var endsInSet = false;
+    var at = 0;
+    while (at + 2 <= path.Length)
+    {
+      var segment = path[at];
+      var count = path[at + 1];
+      at += 2;
+      if (at + count * 4 > path.Length) return;
+      for (var i = 0; i < count; i++)
+      {
+        var number = BinaryPrimitives.ReadUInt32BigEndian(path[(at + i * 4)..]);
+        if (segment != 2)
+        {
+          endsInSet = true;
+          continue;
+        }
+        endsInSet = false;
+        if (number == last) continue;
+        before = last;
+        last = number;
+        distinct++;
+      }
+      at += count * 4;
+    }
+    if (endsInSet || distinct < 3 || Reserved(before) || Reserved(last)) return;
+    if (!counts.TryGetValue(last, out var neighbours)) counts[last] = neighbours = [];
+    neighbours[before] = neighbours.GetValueOrDefault(before) + 1;
+  }
+
+  static bool Reserved(uint number) => number is 0 or 23456 or (>= 64496 and <= 131071) or >= 4200000000;
+}