MeWeb/tools/languages.cs

using System.Globalization;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Nodes;

const string Cldr = "47.0.0";
string[] unLanguages = ["ar", "zh", "en", "fr", "ru", "es"];
string[] reviewed = ["en", "it"];
string[] officialStatuses = ["official", "de_facto_official"];
string[] caselessScripts = ["ka"];
var supplied = new Dictionary<string, (string Name, string English, bool Rtl)>
{
  ["ay"] = ("Aymar aru", "Aymara", false),
  ["bi"] = ("Bislama", "Bislama", false),
  ["dv"] = ("ދިވެހި", "Dhivehi", true),
  ["fj"] = ("Na Vosa Vakaviti", "Fijian", false),
  ["gil"] = ("Taetae ni Kiribati", "Gilbertese", false),
  ["gn"] = ("Avañe'ẽ", "Guarani", false),
  ["hif"] = ("Fiji Baat", "Fiji Hindi", false),
  ["ho"] = ("Hiri Motu", "Hiri Motu", false),
  ["ht"] = ("Kreyòl ayisyen", "Haitian Creole", false),
  ["mh"] = ("Kajin M̧ajeļ", "Marshallese", false),
  ["ms-Arab"] = ("بهاس ملايو", "Malay (Arabic script)", true),
  ["na"] = ("Dorerin Naoero", "Nauruan", false),
  ["ny"] = ("Chichewa", "Chichewa", false),
  ["pau"] = ("Tekoi ra Belau", "Palauan", false),
  ["sm"] = ("Gagana Samoa", "Samoan", false),
  ["ss"] = ("SiSwati", "Swati", false),
  ["tet"] = ("Tetun", "Tetum", false),
  ["tpi"] = ("Tok Pisin", "Tok Pisin", false),
  ["tvl"] = ("Te Ggana Tuuvalu", "Tuvaluan", false),
  ["wni"] = ("Shindzuani", "Ndzwani Comorian", false),
  ["zdj"] = ("Shingazidja", "Ngazidja Comorian", false),
};

var root = FindRoot();
var target = Path.Combine(root, "data", "languages.json");
using var http = new HttpClient();
http.DefaultRequestHeaders.UserAgent.ParseAdd("meweb-languages/1.0");
var baseUrl = $"https://raw.githubusercontent.com/unicode-org/cldr-json/{Cldr}/cldr-json/cldr-core/supplemental";
var territories = JsonNode.Parse(await http.GetStringAsync($"{baseUrl}/territoryInfo.json"))["supplemental"]["territoryInfo"].AsObject();
var containment = JsonNode.Parse(await http.GetStringAsync($"{baseUrl}/territoryContainment.json"))["supplemental"]["territoryContainment"];
var members = containment["UN"]["_contains"].AsArray().Select(n => (string)n).ToHashSet();

var known = CultureInfo.GetCultures(CultureTypes.NeutralCultures | CultureTypes.SpecificCultures).Select(c => c.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
var codes = new SortedSet<string>(unLanguages, StringComparer.Ordinal);
var missing = new SortedSet<string>(StringComparer.Ordinal);
foreach (var (territory, info) in territories)
{
  if (!members.Contains(territory)) continue;
  var population = info["languagePopulation"]?.AsObject();
  if (population is null) continue;
  foreach (var (language, details) in population)
  {
    var status = (string)details["_officialStatus"];
    if (status is null || !officialStatuses.Contains(status)) continue;
    var code = language.Replace('_', '-');
    if (known.Contains(code) || supplied.ContainsKey(code)) codes.Add(code);
    else missing.Add(code);
  }
}

var previous = File.Exists(target)
  ? JsonNode.Parse(File.ReadAllText(target)).AsArray().ToDictionary(n => (string)n["code"], n => (string)n["status"])
  : [];
var list = new JsonArray();
foreach (var code in codes)
{
  var (name, english, rtl) = supplied.TryGetValue(code, out var given) ? given : FromCulture(code);
  var status = reviewed.Contains(code) ? "reviewed" : previous.GetValueOrDefault(code) ?? "draft";
  list.Add((JsonNode)new JsonObject
  {
    ["code"] = code,
    ["name"] = name,
    ["english"] = english,
    ["rtl"] = rtl,
    ["status"] = status,
  });
}
File.WriteAllText(target, list.ToJsonString(new JsonSerializerOptions { WriteIndented = true, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }) + "\n");
Console.WriteLine($"{list.Count} languages written to data/languages.json (CLDR {Cldr}, {members.Count} UN member states)");
if (missing.Count > 0) Console.WriteLine($"official but neither in ICU nor supplied, left out: {string.Join(", ", missing)}");
return 0;

(string Name, string English, bool Rtl) FromCulture(string code)
{
  var culture = CultureInfo.GetCultureInfo(code);
  var native = culture.NativeName;
  var name = caselessScripts.Contains(code.Split('-')[0]) || native.Length == 0
    ? native
    : culture.TextInfo.ToUpper(native[0]) + native[1..];
  return (name, culture.EnglishName, culture.TextInfo.IsRightToLeft);
}

static string FindRoot()
{
  var dir = new DirectoryInfo(Directory.GetCurrentDirectory());
  while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "MeWeb.slnx"))) dir = dir.Parent;
  return dir?.FullName ?? throw new InvalidOperationException("run from inside the repository (MeWeb.slnx not found)");
}