using System.Globalization; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Nodes; const string Cldr = "47.0.0"; string[] unLanguages = ["ar", "zh", "en", "fr", "ru", "es"]; string[] reviewed = ["en", "it"]; string[] officialStatuses = ["official", "de_facto_official"]; string[] caselessScripts = ["ka"]; var supplied = new Dictionary { ["ay"] = ("Aymar aru", "Aymara", false), ["bi"] = ("Bislama", "Bislama", false), ["dv"] = ("ދިވެހި", "Dhivehi", true), ["fj"] = ("Na Vosa Vakaviti", "Fijian", false), ["gil"] = ("Taetae ni Kiribati", "Gilbertese", false), ["gn"] = ("Avañe'ẽ", "Guarani", false), ["hif"] = ("Fiji Baat", "Fiji Hindi", false), ["ho"] = ("Hiri Motu", "Hiri Motu", false), ["ht"] = ("Kreyòl ayisyen", "Haitian Creole", false), ["mh"] = ("Kajin M̧ajeļ", "Marshallese", false), ["ms-Arab"] = ("بهاس ملايو", "Malay (Arabic script)", true), ["na"] = ("Dorerin Naoero", "Nauruan", false), ["ny"] = ("Chichewa", "Chichewa", false), ["pau"] = ("Tekoi ra Belau", "Palauan", false), ["sm"] = ("Gagana Samoa", "Samoan", false), ["ss"] = ("SiSwati", "Swati", false), ["tet"] = ("Tetun", "Tetum", false), ["tpi"] = ("Tok Pisin", "Tok Pisin", false), ["tvl"] = ("Te Ggana Tuuvalu", "Tuvaluan", false), ["wni"] = ("Shindzuani", "Ndzwani Comorian", false), ["zdj"] = ("Shingazidja", "Ngazidja Comorian", false), }; var suppliedQuotes = new Dictionary { ["ay"] = ["«", "»"], ["bi"] = ["“", "”"], ["dv"] = ["”", "“"], ["fj"] = ["“", "”"], ["gil"] = ["“", "”"], ["gn"] = ["«", "»"], ["hif"] = ["“", "”"], ["ho"] = ["“", "”"], ["ht"] = ["«\u00A0", "\u00A0»"], ["mh"] = ["“", "”"], ["ms-Arab"] = ["“", "”"], ["na"] = ["“", "”"], ["ny"] = ["“", "”"], ["pau"] = ["“", "”"], ["sm"] = ["“", "”"], ["ss"] = ["“", "”"], ["tet"] = ["“", "”"], ["tpi"] = ["“", "”"], ["tvl"] = ["“", "”"], ["wni"] = ["«\u00A0", "\u00A0»"], ["zdj"] = ["«\u00A0", "\u00A0»"], }; var spacedQuotes = new HashSet { "fr" }; var fixedQuotes = new Dictionary { ["st"] = ["“", "”"] }; var root = FindRoot(); var target = Path.Combine(root, "data", "languages.json"); using var http = new HttpClient(); http.DefaultRequestHeaders.UserAgent.ParseAdd("meweb-languages/1.0"); var baseUrl = $"https://raw.githubusercontent.com/unicode-org/cldr-json/{Cldr}/cldr-json/cldr-core/supplemental"; var territories = JsonNode.Parse(await http.GetStringAsync($"{baseUrl}/territoryInfo.json"))["supplemental"]["territoryInfo"].AsObject(); var containment = JsonNode.Parse(await http.GetStringAsync($"{baseUrl}/territoryContainment.json"))["supplemental"]["territoryContainment"]; var members = containment["UN"]["_contains"].AsArray().Select(n => (string)n).ToHashSet(); var miscUrl = $"https://raw.githubusercontent.com/unicode-org/cldr-json/{Cldr}/cldr-json/cldr-misc-full/main"; var known = CultureInfo.GetCultures(CultureTypes.NeutralCultures | CultureTypes.SpecificCultures).Select(c => c.Name).ToHashSet(StringComparer.OrdinalIgnoreCase); var codes = new SortedSet(unLanguages, StringComparer.Ordinal); var missing = new SortedSet(StringComparer.Ordinal); foreach (var (territory, info) in territories) { if (!members.Contains(territory)) continue; var population = info["languagePopulation"]?.AsObject(); if (population is null) continue; foreach (var (language, details) in population) { var status = (string)details["_officialStatus"]; if (status is null || !officialStatuses.Contains(status)) continue; var code = language.Replace('_', '-'); if (known.Contains(code) || supplied.ContainsKey(code)) codes.Add(code); else missing.Add(code); } } var previous = File.Exists(target) ? JsonNode.Parse(File.ReadAllText(target)).AsArray().ToDictionary(n => (string)n["code"], n => (string)n["status"]) : []; var list = new JsonArray(); foreach (var code in codes) { var (name, english, rtl) = supplied.TryGetValue(code, out var given) ? given : FromCulture(code); var status = reviewed.Contains(code) ? "reviewed" : previous.GetValueOrDefault(code) ?? "draft"; var quotes = await Quotes(code); list.Add((JsonNode)new JsonObject { ["code"] = code, ["name"] = name, ["english"] = english, ["rtl"] = rtl, ["status"] = status, ["quotes"] = new JsonArray(quotes[0], quotes[1]), }); } File.WriteAllText(target, list.ToJsonString(new JsonSerializerOptions { WriteIndented = true, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }) + "\n"); Console.WriteLine($"{list.Count} languages written to data/languages.json (CLDR {Cldr}, {members.Count} UN member states)"); if (missing.Count > 0) Console.WriteLine($"official but neither in ICU nor supplied, left out: {string.Join(", ", missing)}"); return 0; async Task Quotes(string code) { if (suppliedQuotes.TryGetValue(code, out var given) || fixedQuotes.TryGetValue(code, out given)) return given; foreach (var locale in new[] { code, code.Split('-')[0] }.Distinct()) { using var response = await http.GetAsync($"{miscUrl}/{locale}/delimiters.json"); if (!response.IsSuccessStatusCode) continue; var delimiters = JsonNode.Parse(await response.Content.ReadAsStringAsync())["main"].AsObject().First().Value["delimiters"]; var open = (string)delimiters["quotationStart"]; var close = (string)delimiters["quotationEnd"]; return spacedQuotes.Contains(code) ? [open + "\u00A0", "\u00A0" + close] : [open, close]; } throw new InvalidOperationException($"{code}: no quotation marks in CLDR and none supplied"); } (string Name, string English, bool Rtl) FromCulture(string code) { var culture = CultureInfo.GetCultureInfo(code); var native = culture.NativeName; var name = caselessScripts.Contains(code.Split('-')[0]) || native.Length == 0 ? native : culture.TextInfo.ToUpper(native[0]) + native[1..]; return (name, culture.EnglishName, culture.TextInfo.IsRightToLeft); } static string FindRoot() { var dir = new DirectoryInfo(Directory.GetCurrentDirectory()); while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "MeWeb.slnx"))) dir = dir.Parent; return dir?.FullName ?? throw new InvalidOperationException("run from inside the repository (MeWeb.slnx not found)"); }