MeWeb/src/MeWeb.Core/Knowledge/Sources.cs

using System.Buffers.Binary;
using System.IO.Compression;
using System.Net;

namespace MeWeb.Core.Knowledge;

public class Carrier
{
  public string MccMnc { get; set; }
  public string Name { get; set; }
}

public static class Sources
{
  public static IEnumerable<AsnRange> IpToAsn(Stream gzip)
  {
    var names = new Dictionary<string, string>(StringComparer.Ordinal);
    using var reader = new StreamReader(new GZipStream(gzip, CompressionMode.Decompress));
    string line;
    while ((line = reader.ReadLine()) is not null)
    {
      var parts = line.Split('\t');
      if (parts.Length < 5 || !uint.TryParse(parts[2], out var number) || number == 0) continue;
      if (!IPAddress.TryParse(parts[0], out var start) || !IPAddress.TryParse(parts[1], out var end)) continue;
      yield return new AsnRange { Start = start, End = end, Number = number, Country = Intern(names, parts[3] == "None" ? "" : parts[3]), Name = Intern(names, parts[4]) };
    }
  }

  public static Dictionary<uint, string> DbIpOrganisations(Stream gzip)
  {
    var seen = new Dictionary<uint, Dictionary<string, int>>();
    using var reader = new StreamReader(new GZipStream(gzip, CompressionMode.Decompress));
    string line;
    while ((line = reader.ReadLine()) is not null)
    {
      var first = line.IndexOf(',');
      var second = first < 0 ? -1 : line.IndexOf(',', first + 1);
      var third = second < 0 ? -1 : line.IndexOf(',', second + 1);
      if (third < 0 || !uint.TryParse(line.AsSpan(second + 1, third - second - 1), out var number) || number == 0) continue;
      var name = line[(third + 1)..].Trim();
      if (name.Length >= 2 && name[0] == '"' && name[^1] == '"') name = name[1..^1].Replace("\"\"", "\"");
      if (name.Length == 0) continue;
      if (!seen.TryGetValue(number, out var names)) seen[number] = names = [];
      names[name] = names.GetValueOrDefault(name) + 1;
    }
    return seen.ToDictionary(p => p.Key, p => p.Value.MaxBy(n => n.Value).Key);
  }

  public static Dictionary<uint, List<uint>> Upstreams(Stream mrt, int keep = 4, double share = 0.05)
  {
    var counts = new Dictionary<uint, Dictionary<uint, int>>();
    var header = new byte[12];
    var body = new byte[1 << 16];
    while (Fill(mrt, header))
    {
      var type = BinaryPrimitives.ReadUInt16BigEndian(header.AsSpan(4));
      var subtype = BinaryPrimitives.ReadUInt16BigEndian(header.AsSpan(6));
      var length = (int)BinaryPrimitives.ReadUInt32BigEndian(header.AsSpan(8));
      if (body.Length < length) body = new byte[length];
      if (!Fill(mrt, body.AsSpan(0, length))) break;
      if (type != 13 || subtype is not (2 or 4)) continue;
      Rib(body.AsSpan(0, length), counts);
    }
    var result = new Dictionary<uint, List<uint>>();
    foreach (var (origin, neighbours) in counts)
    {
      var total = (double)neighbours.Values.Sum();
      var chosen = neighbours.Where(n => n.Value / total >= share).OrderByDescending(n => n.Value).ThenBy(n => n.Key).Take(keep).Select(n => n.Key).ToList();
      if (chosen.Count > 0) result[origin] = chosen;
    }
    return result;
  }

  public static List<Carrier> Carriers(TextReader textpb)
  {
    var list = new List<Carrier>();
    string name = null;
    string tuple = null;
    var plain = false;
    string line;
    while ((line = textpb.ReadLine()) is not null)
    {
      var text = line.Trim();
      if (text.StartsWith("carrier_id {", StringComparison.Ordinal)) name = null;
      else if (text.StartsWith("carrier_name:", StringComparison.Ordinal)) name = Quoted(text);
      else if (text.StartsWith("carrier_attribute {", StringComparison.Ordinal))
      {
        tuple = null;
        plain = true;
      }
      else if (text.StartsWith("mccmnc_tuple:", StringComparison.Ordinal)) tuple = tuple is null ? Quoted(text) : null;
      else if (text == "}" && tuple is not null)
      {
        if (plain && name is not null && list.All(c => c.MccMnc != tuple)) list.Add(new Carrier { MccMnc = tuple, Name = name });
        tuple = null;
      }
      else if (text.Length > 0 && text != "}") plain = false;
    }
    return list.OrderBy(c => c.MccMnc, StringComparer.Ordinal).ToList();
  }

  static string Intern(Dictionary<string, string> names, string text)
  {
    if (names.TryGetValue(text, out var known)) return known;
    names[text] = text;
    return text;
  }

  static string Quoted(string text)
  {
    var start = text.IndexOf('"');
    var end = text.LastIndexOf('"');
    return start >= 0 && end > start ? text[(start + 1)..end] : null;
  }

  static void Rib(ReadOnlySpan<byte> rib, Dictionary<uint, Dictionary<uint, int>> counts)
  {
    if (rib.Length < 5) return;
    var at = 4;
    var bits = rib[at++];
    at += (bits + 7) / 8;
    if (at + 2 > rib.Length) return;
    var entries = BinaryPrimitives.ReadUInt16BigEndian(rib[at..]);
    at += 2;
    for (var e = 0; e < entries && at + 8 <= rib.Length; e++)
    {
      at += 6;
      var size = BinaryPrimitives.ReadUInt16BigEndian(rib[at..]);
      at += 2;
      if (at + size > rib.Length) return;
      Attributes(rib.Slice(at, size), counts);
      at += size;
    }
  }

  static void Attributes(ReadOnlySpan<byte> attributes, Dictionary<uint, Dictionary<uint, int>> counts)
  {
    var at = 0;
    while (at + 3 <= attributes.Length)
    {
      var flags = attributes[at];
      var type = attributes[at + 1];
      var extended = (flags & 0x10) != 0;
      if (extended && at + 4 > attributes.Length) return;
      var length = extended ? BinaryPrimitives.ReadUInt16BigEndian(attributes[(at + 2)..]) : attributes[at + 2];
      var start = at + (extended ? 4 : 3);
      if (start + length > attributes.Length) return;
      if (type == 2)
      {
        Path(attributes.Slice(start, length), counts);
        return;
      }
      at = start + length;
    }
  }

  static void Path(ReadOnlySpan<byte> path, Dictionary<uint, Dictionary<uint, int>> counts)
  {
    uint last = 0;
    uint before = 0;
    var endsInSet = false;
    var at = 0;
    while (at + 2 <= path.Length)
    {
      var segment = path[at];
      var count = path[at + 1];
      at += 2;
      if (at + count * 4 > path.Length) return;
      for (var i = 0; i < count; i++)
      {
        var number = BinaryPrimitives.ReadUInt32BigEndian(path[(at + i * 4)..]);
        if (segment != 2)
        {
          endsInSet = true;
          continue;
        }
        endsInSet = false;
        if (number == last) continue;
        before = last;
        last = number;
      }
      at += count * 4;
    }
    if (endsInSet || last == 0 || before == 0 || Reserved(before) || Reserved(last)) return;
    if (!counts.TryGetValue(last, out var neighbours)) counts[last] = neighbours = [];
    neighbours[before] = neighbours.GetValueOrDefault(before) + 1;
  }

  static bool Reserved(uint number) => number is 23456 or (>= 64496 and <= 131071) or >= 4200000000;

  static bool Fill(Stream stream, Span<byte> buffer)
  {
    var read = 0;
    while (read < buffer.Length)
    {
      var n = stream.Read(buffer[read..]);
      if (n == 0) return false;
      read += n;
    }
    return true;
  }
}