MeWeb/src/MeWeb.Core/Knowledge/Sources.cs

using System.Buffers.Binary;
using System.IO.Compression;
using System.Net;
using System.Net.Sockets;
using System.Text;

namespace MeWeb.Core.Knowledge;

public class Carrier
{
  public string MccMnc { get; set; }
  public string Name { get; set; }
}

public static class Sources
{
  static readonly HashSet<string> Allocations = ["ALLOCATED PA", "ALLOCATED UNSPECIFIED", "ALLOCATED-BY-RIR"];

  public static IEnumerable<AsnRange> IpToAsn(Stream gzip)
  {
    var names = new Dictionary<string, string>(StringComparer.Ordinal);
    using var reader = new StreamReader(new GZipStream(gzip, CompressionMode.Decompress));
    string line;
    while ((line = reader.ReadLine()) is not null)
    {
      var parts = line.Split('\t');
      if (parts.Length < 5 || !uint.TryParse(parts[2], out var number) || number == 0) continue;
      if (!IPAddress.TryParse(parts[0], out var start) || !IPAddress.TryParse(parts[1], out var end)) continue;
      yield return new AsnRange { Start = start, End = end, Number = number, Country = Intern(names, parts[3] == "None" ? "" : parts[3]), Name = Intern(names, parts[4]) };
    }
  }

  public static Dictionary<uint, string> DbIpOrganisations(Stream gzip)
  {
    var seen = new Dictionary<uint, Dictionary<string, int>>();
    using var reader = new StreamReader(new GZipStream(gzip, CompressionMode.Decompress));
    string line;
    while ((line = reader.ReadLine()) is not null)
    {
      var first = line.IndexOf(',');
      var second = first < 0 ? -1 : line.IndexOf(',', first + 1);
      var third = second < 0 ? -1 : line.IndexOf(',', second + 1);
      if (third < 0 || !uint.TryParse(line.AsSpan(second + 1, third - second - 1), out var number) || number == 0) continue;
      var name = line[(third + 1)..].Trim();
      if (name.Length >= 2 && name[0] == '"' && name[^1] == '"') name = name[1..^1].Replace("\"\"", "\"");
      if (name.Length == 0) continue;
      if (!seen.TryGetValue(number, out var names)) seen[number] = names = [];
      names[name] = names.GetValueOrDefault(name) + 1;
    }
    return seen.ToDictionary(p => p.Key, p => p.Value.MaxBy(n => n.Value).Key);
  }

  public static IEnumerable<RegistryRange> RipeBlocks(Stream gzip)
  {
    using var reader = new StreamReader(new GZipStream(gzip, CompressionMode.Decompress), Encoding.Latin1);
    RegistryRange range = null;
    string status = null;
    string line;
    while ((line = reader.ReadLine()) is not null)
    {
      if (line.Length == 0)
      {
        if (range?.Name is not null && !Allocations.Contains(status ?? "")) yield return range;
        range = null;
        status = null;
        continue;
      }
      if (range is null)
      {
        range = RipeRange(line);
        continue;
      }
      if (line.StartsWith("netname:", StringComparison.Ordinal)) range.Name ??= Value(line);
      else if (line.StartsWith("country:", StringComparison.Ordinal)) range.Country ??= Value(line).ToUpperInvariant();
      else if (line.StartsWith("status:", StringComparison.Ordinal)) status ??= Value(line);
    }
    if (range?.Name is not null && !Allocations.Contains(status ?? "")) yield return range;
  }

  static RegistryRange RipeRange(string line)
  {
    if (line.StartsWith("inetnum:", StringComparison.Ordinal))
    {
      var parts = Value(line).Split('-', StringSplitOptions.TrimEntries);
      if (parts.Length != 2 || !IPAddress.TryParse(parts[0], out var first) || !IPAddress.TryParse(parts[1], out var last)) return null;
      if (first.AddressFamily != AddressFamily.InterNetwork || last.AddressFamily != AddressFamily.InterNetwork) return null;
      return new RegistryRange { Start = BinaryPrimitives.ReadUInt32BigEndian(first.GetAddressBytes()), End = BinaryPrimitives.ReadUInt32BigEndian(last.GetAddressBytes()) };
    }
    if (!line.StartsWith("inet6num:", StringComparison.Ordinal) || !IPNetwork.TryParse(Value(line), out var network)) return null;
    var start = BinaryPrimitives.ReadUInt128BigEndian(network.BaseAddress.GetAddressBytes());
    var span = network.PrefixLength == 0 ? UInt128.MaxValue : (UInt128.One << (128 - network.PrefixLength)) - 1;
    return new RegistryRange { V6 = true, Start = start, End = start | span };
  }

  static string Value(string line)
  {
    var text = line.AsSpan(line.IndexOf(':') + 1);
    var comment = text.IndexOf('#');
    return (comment < 0 ? text : text[..comment]).Trim().ToString();
  }

  public static List<Carrier> Carriers(TextReader textpb)
  {
    var list = new List<Carrier>();
    string name = null;
    string tuple = null;
    var plain = false;
    string line;
    while ((line = textpb.ReadLine()) is not null)
    {
      var text = line.Trim();
      if (text.StartsWith("carrier_id {", StringComparison.Ordinal)) name = null;
      else if (text.StartsWith("carrier_name:", StringComparison.Ordinal)) name = Quoted(text);
      else if (text.StartsWith("carrier_attribute {", StringComparison.Ordinal))
      {
        tuple = null;
        plain = true;
      }
      else if (text.StartsWith("mccmnc_tuple:", StringComparison.Ordinal)) tuple = tuple is null ? Quoted(text) : null;
      else if (text == "}" && tuple is not null)
      {
        if (plain && name is not null && list.All(c => c.MccMnc != tuple)) list.Add(new Carrier { MccMnc = tuple, Name = name });
        tuple = null;
      }
      else if (text.Length > 0 && text != "}") plain = false;
    }
    return list.OrderBy(c => c.MccMnc, StringComparer.Ordinal).ToList();
  }

  static string Intern(Dictionary<string, string> names, string text)
  {
    if (names.TryGetValue(text, out var known)) return known;
    names[text] = text;
    return text;
  }

  static string Quoted(string text)
  {
    var start = text.IndexOf('"');
    var end = text.LastIndexOf('"');
    return start >= 0 && end > start ? text[(start + 1)..end] : null;
  }

  internal static bool Fill(Stream stream, Span<byte> buffer)
  {
    var read = 0;
    while (read < buffer.Length)
    {
      var n = stream.Read(buffer[read..]);
      if (n == 0) return false;
      read += n;
    }
    return true;
  }
}