Initial commit.

.gitignore

@@ -0,0 +1,2 @@
+/target
+Cargo.lock

Cargo.toml

@@ -0,0 +1,5 @@
+[workspace]
+members = [
+    "txtlang",
+    "txtlang-gen",
+]

txtlang-gen/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "txtlang-gen"
+version = "0.1.0"
+authors = ["Anders Olsson <anders.e.olsson@gmail.com>"]
+edition = "2018"
+
+[dependencies]
+anyhow = "1.0"
+isolang = "1.0"
+quick-xml = "0.18"
+thiserror = "1.0"

txtlang-gen/src/main.rs

@@ -0,0 +1,157 @@
+use std::env;
+use std::fs::File;
+use std::io::BufReader;
+use std::io::{self, Write};
+
+use anyhow::Result;
+use isolang::Language;
+use quick_xml::{events::Event, Reader};
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+enum Error {
+    #[error("xml error")]
+    Xml(#[from] quick_xml::Error),
+}
+
+#[derive(PartialEq, Debug)]
+struct LocalDisplayName {
+    name: String,
+    lang: Language,
+}
+
+fn main() -> Result<()> {
+    let inputs = env::args().skip(1);
+
+    let mut results = Vec::new();
+
+    for input in inputs {
+        let mut reader = File::open(&input)
+            .map(BufReader::new)
+            .map(Reader::from_reader)?;
+
+        reader.trim_text(true);
+
+        let mut buf = Vec::new();
+
+        loop {
+            match reader.read_event(&mut buf) {
+                Ok(Event::Start(ref e)) if e.name() == b"localeDisplayNames" => {
+                    results.extend(parse::locale_display_names(&mut reader, &mut buf));
+                }
+                Err(e) => panic!("Error at position {}: {:?}", reader.buffer_position(), e),
+                Ok(Event::Eof) => break,
+                _ => (),
+            }
+
+            buf.clear();
+        }
+    }
+
+    let before = results.len();
+
+    results.sort_unstable_by(|a, b| a.name.cmp(&b.name));
+    results.dedup();
+
+    let dedup = results.len();
+
+    let mut remove = Vec::new();
+
+    results.dedup_by(|a, b| {
+        if a.name == b.name {
+            remove.push(a.name.clone());
+        }
+
+        a.name == b.name
+    });
+
+    results.retain(|x| !remove.contains(&x.name));
+
+    let clean = results.len();
+
+    eprintln!("before={}", before);
+    eprintln!("dedup={}", dedup);
+    eprintln!("clean={}", clean);
+
+    let stdout = io::stdout();
+    let mut lock = stdout.lock();
+
+    for result in results {
+        assert!(!result.name.contains('\n'));
+        assert!(!result.name.contains('\t'));
+
+        writeln!(lock, "{}\t{}", result.name, result.lang.to_639_3())
+            .expect("failed to write to stdout");
+    }
+
+    lock.flush().expect("failed to flush to stdout");
+
+    Ok(())
+}
+
+mod parse {
+    use std::fs::File;
+    use std::io::BufReader;
+
+    use anyhow::Result;
+    use isolang::Language;
+    use quick_xml::{events::Event, Reader};
+
+    use crate::LocalDisplayName;
+
+    pub(crate) fn locale_display_names(
+        reader: &mut Reader<BufReader<File>>,
+        buf: &mut Vec<u8>,
+    ) -> Vec<LocalDisplayName> {
+        let mut result = Vec::new();
+
+        let mut parse = false;
+        let mut language = None;
+
+        loop {
+            match reader.read_event(buf) {
+                Ok(Event::End(ref e)) if e.name() == b"localeDisplayNames" => {
+                    break;
+                }
+                Ok(Event::Start(ref e)) if e.name() == b"languages" => {
+                    parse = true;
+                }
+                Ok(Event::End(ref e)) if e.name() == b"languages" => {
+                    parse = false;
+                }
+                Ok(Event::Start(ref e)) if e.name() == b"language" && parse => {
+                    let a = e
+                        .attributes()
+                        .filter_map(Result::ok)
+                        .find(|a| a.key == b"type");
+
+                    if let Some(a) = a {
+                        if let Ok(lang) = reader.decode_without_bom(&a.value) {
+                            language = Language::from_639_1(lang)
+                                .or_else(|| Language::from_639_3(lang))
+                                .or_else(|| Language::from_locale(lang));
+                        }
+                    }
+                }
+                Ok(Event::End(ref e)) if e.name() == b"language" && parse => {
+                    language = None;
+                }
+                Ok(Event::Text(e)) if language.is_some() => {
+                    if let Ok(text) = reader.decode_without_bom(&e) {
+                        result.push(LocalDisplayName {
+                            name: text.to_lowercase(),
+                            lang: language.take().unwrap(),
+                        });
+                    }
+                }
+                Err(e) => panic!("Error at position {}: {:?}", reader.buffer_position(), e),
+                Ok(Event::Eof) => break,
+                _ => (),
+            }
+
+            buf.clear();
+        }
+
+        result
+    }
+}

txtlang/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "txtlang"
+version = "0.1.0"
+authors = ["Anders Olsson <anders.e.olsson@gmail.com>"]
+edition = "2018"
+build = "build.rs"
+
+[dependencies]
+isolang = "1.0"
+
+[build-dependencies]
+isolang = "1.0"
+phf_codegen = "0.8"

txtlang/build.rs

@@ -0,0 +1,44 @@
+use std::collections::HashMap;
+use std::env;
+use std::error::Error;
+use std::fs::{self, File};
+use std::io::{self, BufRead, BufReader, BufWriter, Write};
+use std::path::Path;
+
+fn title(s: &str) -> String {
+    let mut c = s.chars();
+    match c.next() {
+        None => String::new(),
+        Some(f) => f.to_uppercase().chain(c).collect(),
+    }
+}
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let mut map = phf_codegen::Map::new();
+
+    let lines = File::open("languages.txt")
+        .map(BufReader::new)?
+        .lines()
+        .filter_map(Result::ok)
+        .collect::<Vec<_>>();
+
+    for line in &lines {
+        let parts = line.splitn(2, '\t').collect::<Vec<_>>();
+
+        let name = parts[0];
+        let language = parts[1];
+
+        map.entry(name, &format!("Language::{}", title(language)));
+    }
+
+    let path = Path::new(&env::var("OUT_DIR")?).join("languages.rs");
+    let mut file = BufWriter::new(File::create(&path)?);
+
+    write!(
+        &mut file,
+        "static LANGUAGES: phf::Map<&'static str, Language> = {};\n",
+        map.build()
+    )?;
+
+    Ok(())
+}

txtlang/src/lib.rs

@@ -0,0 +1,13 @@
+use isolang::Language;
+
+include!(concat!(env!("OUT_DIR"), "/languages.rs"));
+
+pub trait LanguageExt {
+    fn from_name(name: &str) -> Option<Language>;
+}
+
+impl LanguageExt for Language {
+    fn from_name(name: &str) -> Option<Language> {
+        LANGUAGES.get(name).cloned()
+    }
+}