MeWeb/tools/languages.cs

using System.Globalization;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Nodes;

const string Cldr = "47.0.0";
string[] unLanguages = ["ar", "zh", "en", "fr", "ru", "es"];
string[] reviewed = ["en", "it"];
string[] officialStatuses = ["official", "de_facto_official"];
string[] caselessScripts = ["ka"];
var supplied = new Dictionary<string, (string Name, string English, bool Rtl)>
{
  ["ay"] = ("Aymar aru", "Aymara", false),
  ["bi"] = ("Bislama", "Bislama", false),
  ["dv"] = ("ދިވެހި", "Dhivehi", true),
  ["fj"] = ("Na Vosa Vakaviti", "Fijian", false),
  ["gil"] = ("Taetae ni Kiribati", "Gilbertese", false),
  ["gn"] = ("Avañe'ẽ", "Guarani", false),
  ["hif"] = ("Fiji Baat", "Fiji Hindi", false),
  ["ho"] = ("Hiri Motu", "Hiri Motu", false),
  ["ht"] = ("Kreyòl ayisyen", "Haitian Creole", false),
  ["mh"] = ("Kajin M̧ajeļ", "Marshallese", false),
  ["ms-Arab"] = ("بهاس ملايو", "Malay (Arabic script)", true),
  ["na"] = ("Dorerin Naoero", "Nauruan", false),
  ["ny"] = ("Chichewa", "Chichewa", false),
  ["pau"] = ("Tekoi ra Belau", "Palauan", false),
  ["sm"] = ("Gagana Samoa", "Samoan", false),
  ["ss"] = ("SiSwati", "Swati", false),
  ["tet"] = ("Tetun", "Tetum", false),
  ["tpi"] = ("Tok Pisin", "Tok Pisin", false),
  ["tvl"] = ("Te Ggana Tuuvalu", "Tuvaluan", false),
  ["wni"] = ("Shindzuani", "Ndzwani Comorian", false),
  ["zdj"] = ("Shingazidja", "Ngazidja Comorian", false),
};
var suppliedQuotes = new Dictionary<string, string[]>
{
  ["ay"] = ["«", "»"],
  ["bi"] = ["“", "”"],
  ["dv"] = ["”", "“"],
  ["fj"] = ["“", "”"],
  ["gil"] = ["“", "”"],
  ["gn"] = ["«", "»"],
  ["hif"] = ["“", "”"],
  ["ho"] = ["“", "”"],
  ["ht"] = ["«\u00A0", "\u00A0»"],
  ["mh"] = ["“", "”"],
  ["ms-Arab"] = ["“", "”"],
  ["na"] = ["“", "”"],
  ["ny"] = ["“", "”"],
  ["pau"] = ["“", "”"],
  ["sm"] = ["“", "”"],
  ["ss"] = ["“", "”"],
  ["tet"] = ["“", "”"],
  ["tpi"] = ["“", "”"],
  ["tvl"] = ["“", "”"],
  ["wni"] = ["«\u00A0", "\u00A0»"],
  ["zdj"] = ["«\u00A0", "\u00A0»"],
};
var spacedQuotes = new HashSet<string> { "fr" };
var fixedQuotes = new Dictionary<string, string[]> { ["st"] = ["“", "”"] };

var root = FindRoot();
var target = Path.Combine(root, "data", "languages.json");
using var http = new HttpClient();
http.DefaultRequestHeaders.UserAgent.ParseAdd("meweb-languages/1.0");
var baseUrl = $"https://raw.githubusercontent.com/unicode-org/cldr-json/{Cldr}/cldr-json/cldr-core/supplemental";
var territories = JsonNode.Parse(await http.GetStringAsync($"{baseUrl}/territoryInfo.json"))["supplemental"]["territoryInfo"].AsObject();
var containment = JsonNode.Parse(await http.GetStringAsync($"{baseUrl}/territoryContainment.json"))["supplemental"]["territoryContainment"];
var members = containment["UN"]["_contains"].AsArray().Select(n => (string)n).ToHashSet();
var miscUrl = $"https://raw.githubusercontent.com/unicode-org/cldr-json/{Cldr}/cldr-json/cldr-misc-full/main";

var known = CultureInfo.GetCultures(CultureTypes.NeutralCultures | CultureTypes.SpecificCultures).Select(c => c.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
var codes = new SortedSet<string>(unLanguages, StringComparer.Ordinal);
var missing = new SortedSet<string>(StringComparer.Ordinal);
foreach (var (territory, info) in territories)
{
  if (!members.Contains(territory)) continue;
  var population = info["languagePopulation"]?.AsObject();
  if (population is null) continue;
  foreach (var (language, details) in population)
  {
    var status = (string)details["_officialStatus"];
    if (status is null || !officialStatuses.Contains(status)) continue;
    var code = language.Replace('_', '-');
    if (known.Contains(code) || supplied.ContainsKey(code)) codes.Add(code);
    else missing.Add(code);
  }
}

var previous = File.Exists(target)
  ? JsonNode.Parse(File.ReadAllText(target)).AsArray().ToDictionary(n => (string)n["code"], n => (string)n["status"])
  : [];
var list = new JsonArray();
foreach (var code in codes)
{
  var (name, english, rtl) = supplied.TryGetValue(code, out var given) ? given : FromCulture(code);
  var status = reviewed.Contains(code) ? "reviewed" : previous.GetValueOrDefault(code) ?? "draft";
  var quotes = await Quotes(code);
  list.Add((JsonNode)new JsonObject
  {
    ["code"] = code,
    ["name"] = name,
    ["english"] = english,
    ["rtl"] = rtl,
    ["status"] = status,
    ["quotes"] = new JsonArray(quotes[0], quotes[1]),
  });
}
File.WriteAllText(target, list.ToJsonString(new JsonSerializerOptions { WriteIndented = true, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }) + "\n");
Console.WriteLine($"{list.Count} languages written to data/languages.json (CLDR {Cldr}, {members.Count} UN member states)");
if (missing.Count > 0) Console.WriteLine($"official but neither in ICU nor supplied, left out: {string.Join(", ", missing)}");
return 0;

async Task<string[]> Quotes(string code)
{
  if (suppliedQuotes.TryGetValue(code, out var given) || fixedQuotes.TryGetValue(code, out given)) return given;
  foreach (var locale in new[] { code, code.Split('-')[0] }.Distinct())
  {
    using var response = await http.GetAsync($"{miscUrl}/{locale}/delimiters.json");
    if (!response.IsSuccessStatusCode) continue;
    var delimiters = JsonNode.Parse(await response.Content.ReadAsStringAsync())["main"].AsObject().First().Value["delimiters"];
    var open = (string)delimiters["quotationStart"];
    var close = (string)delimiters["quotationEnd"];
    return spacedQuotes.Contains(code) ? [open + "\u00A0", "\u00A0" + close] : [open, close];
  }
  throw new InvalidOperationException($"{code}: no quotation marks in CLDR and none supplied");
}

(string Name, string English, bool Rtl) FromCulture(string code)
{
  var culture = CultureInfo.GetCultureInfo(code);
  var native = culture.NativeName;
  var name = caselessScripts.Contains(code.Split('-')[0]) || native.Length == 0
    ? native
    : culture.TextInfo.ToUpper(native[0]) + native[1..];
  return (name, culture.EnglishName, culture.TextInfo.IsRightToLeft);
}

static string FindRoot()
{
  var dir = new DirectoryInfo(Directory.GetCurrentDirectory());
  while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "MeWeb.slnx"))) dir = dir.Parent;
  return dir?.FullName ?? throw new InvalidOperationException("run from inside the repository (MeWeb.slnx not found)");
}