Initial commit.

.gitignore

@@ -0,0 +1,4 @@
+/target
+Cargo.lock
+
+*.fst

Cargo.toml

@@ -0,0 +1,5 @@
+[workspace]
+members = [
+    "wordlang",
+    "wordlang-gen",
+]

wordlang-gen/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "wordlang-gen"
+version = "0.1.0"
+authors = ["Anders Olsson <anders.e.olsson@gmail.com>"]
+edition = "2018"
+
+[dependencies]
+anyhow = "1.0"
+isolang = "1.0"
+quick-xml = "0.18"
+regex = "1.3"
+thiserror = "1.0"

wordlang-gen/src/main.rs

@@ -0,0 +1,527 @@
+use std::cmp::Ordering;
+use std::collections::{hash_map::Entry, HashMap};
+use std::env;
+use std::fs::File;
+use std::io::{self, BufReader, Write};
+
+use anyhow::Result;
+use isolang::Language;
+use quick_xml::{events::Event, Reader};
+use regex::Regex;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+enum Error {
+    #[error("xml error")]
+    Xml(#[from] quick_xml::Error),
+}
+
+fn title(s: &str) -> String {
+    let mut c = s.chars();
+
+    match c.next() {
+        None => String::new(),
+        Some(f) => f.to_uppercase().chain(c).collect(),
+    }
+}
+
+fn main() -> Result<()> {
+    let inputs = env::args().skip(1);
+
+    let mut entries: HashMap<Language, Vec<String>> = HashMap::new();
+
+    let mut buf = Vec::new();
+
+    for input in inputs {
+        let mut reader = File::open(&input)
+            .map(BufReader::new)
+            .map(Reader::from_reader)?;
+
+        reader.trim_text(true);
+
+        let mut language = None;
+        let mut results = Vec::new();
+
+        loop {
+            match reader.read_event(&mut buf) {
+                Ok(Event::Start(ref e)) if e.name() == b"identity" => {
+                    buf.clear();
+
+                    loop {
+                        match reader.read_event(&mut buf) {
+                            Ok(Event::End(ref e)) if e.name() == b"identity" => {
+                                break;
+                            }
+                            Ok(Event::Empty(ref e)) if e.name() == b"language" => {
+                                let a = e
+                                    .attributes()
+                                    .filter_map(Result::ok)
+                                    .find(|a| a.key == b"type");
+
+                                if let Some(a) = a {
+                                    if let Ok(lang) = reader.decode_without_bom(&a.value) {
+                                        language = Language::from_639_1(lang)
+                                            .or_else(|| Language::from_639_3(lang))
+                                            .or_else(|| Language::from_locale(lang));
+                                    }
+                                }
+                            }
+
+                            Err(e) => {
+                                panic!("Error at position {}: {:?}", reader.buffer_position(), e)
+                            }
+                            Ok(Event::Eof) => break,
+                            _ => (),
+                        }
+
+                        buf.clear();
+                    }
+                }
+                Ok(Event::Start(ref e)) if e.name() == b"characters" => {
+                    buf.clear();
+
+                    results.extend(parse::exemplar_characters(&mut reader, &mut buf));
+                }
+                Err(e) => panic!("Error at position {}: {:?}", reader.buffer_position(), e),
+                Ok(Event::Eof) => break,
+                _ => (),
+            }
+
+            buf.clear();
+        }
+
+        buf.clear();
+
+        if results.is_empty() || language.is_none() {
+            continue;
+        }
+
+        // Safe due to if statement above.
+        let language = language.unwrap();
+
+        match entries.entry(language) {
+            Entry::Occupied(entry) => {
+                entry.into_mut().extend(results);
+            }
+            Entry::Vacant(entry) => {
+                entry.insert(results);
+            }
+        }
+    }
+
+    let mut languages = Vec::new();
+    let mut matchers = Vec::new();
+
+    for (language, mut results) in entries.into_iter() {
+        results.sort_unstable();
+        results.dedup();
+
+        let (single, groups): (Vec<String>, _) = results
+            .into_iter()
+            .partition(|string| string.chars().count() == 1);
+
+        let single = single
+            .into_iter()
+            .map(|string| string.chars().next().unwrap())
+            .collect::<Vec<_>>();
+
+        let mut ranges = ranger(single);
+
+        ranges.sort_unstable();
+
+        let mut regex = String::new();
+
+        regex.push_str("^(");
+
+        if !ranges.is_empty() {
+            regex.push('[');
+
+            for kind in ranges {
+                match kind {
+                    Kind::Single(ch) => regex.push(ch),
+                    Kind::Range(from, to) => {
+                        regex.push(from);
+                        regex.push('-');
+                        regex.push(to);
+                    }
+                }
+            }
+
+            regex.push(']');
+
+            if !groups.is_empty() {
+                regex.push('|');
+            }
+        }
+
+        if !groups.is_empty() {
+            let mut first = true;
+
+            for group in groups {
+                if !first {
+                    regex.push('|');
+                } else {
+                    first = false;
+                }
+
+                regex.push_str(&group);
+            }
+        }
+
+        regex.push_str(")+$");
+
+        let re = Regex::new(&regex).expect("failed to build regex");
+
+        languages.push(language);
+        matchers.push(re);
+    }
+
+    assert_eq!(languages.len(), matchers.len());
+
+    if languages.is_empty() {
+        return Ok(());
+    }
+
+    let stdout = io::stdout();
+    let mut lock = stdout.lock();
+
+    writeln!(lock, "use isolang::Language;").expect("failed to write to stdout");
+    writeln!(lock, "use once_cell::sync::Lazy;").expect("failed to write to stdout");
+    writeln!(lock, "use regex::{{Regex, RegexSet}};").expect("failed to write to stdout");
+
+    writeln!(lock).expect("failed to write to stdout");
+
+    writeln!(
+        lock,
+        "pub static LANGUAGES: [Language; {}] = [",
+        languages.len()
+    )
+    .expect("failed to write to stdout");
+
+    for language in &languages {
+        writeln!(lock, "    Language::{},", title(language.to_639_3()))
+            .expect("failed to write to stdout");
+    }
+
+    writeln!(lock, "];").expect("failed to write to stdout");
+
+    writeln!(lock).expect("failed to write to stdout");
+
+    writeln!(
+        lock,
+        "pub static REGEX_SET: Lazy<RegexSet> = Lazy::new(|| {{",
+    )
+    .expect("failed to write to stdout");
+    writeln!(lock, "    RegexSet::new(vec![",).expect("failed to write to stdout");
+
+    for matcher in &matchers {
+        writeln!(lock, "        \"{}\",", matcher).expect("failed to write to stdout");
+    }
+
+    writeln!(lock, "    ]).unwrap()",).expect("failed to write to stdout");
+
+    writeln!(lock, "}});").expect("failed to write to stdout");
+
+    writeln!(lock).expect("failed to write to stdout");
+
+    for (language, matcher) in languages.iter().zip(matchers.iter()) {
+        writeln!(
+            lock,
+            "pub static RE_{}: Lazy<Regex> = Lazy::new(|| Regex::new(\"{}\").unwrap());",
+            language.to_639_3().to_ascii_uppercase(),
+            matcher,
+        )
+        .expect("failed to write to stdout");
+    }
+
+    writeln!(lock).expect("failed to write to stdout");
+
+    writeln!(lock, "impl super::LanguageExt for Language {{",).expect("failed to write to stdout");
+    writeln!(
+        lock,
+        "    fn is_match(&self, word: &str) -> Option<bool> {{",
+    )
+    .expect("failed to write to stdout");
+    writeln!(lock, "        match self {{",).expect("failed to write to stdout");
+
+    for language in &languages {
+        writeln!(
+            lock,
+            "            Language::{} => Some(RE_{}.is_match(word)),",
+            title(language.to_639_3()),
+            language.to_639_3().to_ascii_uppercase(),
+        )
+        .expect("failed to write to stdout");
+    }
+
+    writeln!(lock, "            _ => None,",).expect("failed to write to stdout");
+
+    writeln!(lock, "        }}",).expect("failed to write to stdout");
+    writeln!(lock, "    }}",).expect("failed to write to stdout");
+    writeln!(lock, "}}",).expect("failed to write to stdout");
+
+    lock.flush().expect("failed to flush to stdout");
+
+    Ok(())
+}
+
+#[derive(Clone, Copy, Eq, Debug)]
+enum Kind {
+    Single(char),
+    Range(char, char),
+}
+
+impl PartialOrd for Kind {
+    fn partial_cmp(&self, other: &Kind) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for Kind {
+    fn cmp(&self, other: &Kind) -> Ordering {
+        match (self, other) {
+            (Kind::Single(a), Kind::Single(b)) => a.cmp(&b),
+            (Kind::Single(a), Kind::Range(b, _)) => a.cmp(&b),
+            (Kind::Range(a, _), Kind::Single(b)) => a.cmp(&b),
+            (Kind::Range(a, _), Kind::Range(b, _)) => a.cmp(&b),
+        }
+    }
+}
+
+impl PartialEq for Kind {
+    fn eq(&self, other: &Kind) -> bool {
+        match (self, other) {
+            (Kind::Single(a), Kind::Single(b)) => a == b,
+            (Kind::Single(_), Kind::Range(_, _)) => false,
+            (Kind::Range(_, _), Kind::Single(_)) => false,
+            (Kind::Range(a, b), Kind::Range(c, d)) => a == c && b == d,
+        }
+    }
+}
+
+fn ranger(mut chars: Vec<char>) -> Vec<Kind> {
+    chars.sort_unstable();
+
+    let mut set = HashMap::new();
+
+    for c in chars {
+        if let Some(p) = (' '..c).rev().next() {
+            if !set.contains_key(&p) {
+                set.insert(c, (c, 1));
+            } else {
+                let (q, n) = set.remove(&p).unwrap();
+                set.insert(c, (q, n + 1));
+            }
+        } else {
+            set.insert(c, (c, 1));
+        }
+    }
+
+    set.into_iter()
+        .flat_map(|(to, (from, n))| match n {
+            1 => vec![Kind::Single(from)],
+            2 => vec![Kind::Single(from), Kind::Single(to)],
+            _ => vec![Kind::Range(from, to)],
+        })
+        .collect::<Vec<_>>()
+}
+
+mod parse {
+    use std::char;
+    use std::fs::File;
+    use std::io::BufReader;
+
+    use anyhow::Result;
+    use quick_xml::{events::Event, Reader};
+
+    pub fn exemplar_characters(
+        reader: &mut Reader<BufReader<File>>,
+        buf: &mut Vec<u8>,
+    ) -> Vec<String> {
+        let mut results = Vec::new();
+        let mut parse = false;
+
+        loop {
+            match reader.read_event(buf) {
+                Ok(Event::End(ref e)) if e.name() == b"characters" => {
+                    break;
+                }
+                Ok(Event::Start(ref e)) if e.name() == b"exemplarCharacters" => {
+                    parse = e.attributes().count() == 0
+                        || e.attributes()
+                            .filter_map(Result::ok)
+                            .any(|a| a.key == b"type" && a.value.as_ref() == b"auxiliary");
+                }
+                Ok(Event::End(ref e)) if e.name() == b"exemplarCharacters" && parse => {
+                    parse = false;
+                }
+                Ok(Event::Text(e)) if parse => {
+                    if let Ok(text) = reader.decode_without_bom(&e) {
+                        results.extend(characters(text));
+                    } else {
+                        panic!("failed to decode text");
+                    }
+                }
+                Err(e) => panic!("Error at position {}: {:?}", reader.buffer_position(), e),
+                Ok(Event::Eof) => break,
+                _ => (),
+            }
+
+            buf.clear();
+        }
+
+        results
+    }
+
+    fn characters(input: &str) -> Vec<String> {
+        #[derive(Debug)]
+        enum State {
+            Start,
+            Next,
+            Escaped,
+            Char(char),
+            Range(char),
+            Unicode(String),
+            Sequence(SequenceState, String),
+            End,
+        }
+
+        #[derive(Debug)]
+        enum SequenceState {
+            Default,
+            Escaped,
+            Unicode(String),
+        }
+
+        let mut chars = Vec::new();
+        let mut state = State::Start;
+
+        // http://unicode.org/reports/tr35/tr35-6.html#Character_Elements
+
+        for ch in input.chars() {
+            // eprintln!("{:?}, {:?}", state, ch);
+
+            state = match (state, ch) {
+                (State::Start, '[') => State::Next,
+                (State::Start, ']') => State::End,
+
+                (State::Next, ' ') => State::Next,
+                (State::Next, ']') => State::End,
+                (State::Next, '\\') => State::Escaped,
+                (State::Next, '{') => State::Sequence(SequenceState::Default, String::new()),
+                (State::Next, ch) => State::Char(ch),
+
+                (State::Escaped, 'u') => State::Unicode(String::new()),
+                (State::Escaped, ch) => {
+                    chars.push(ch.to_string());
+
+                    State::Next
+                }
+
+                (State::Char(ch), ' ') => {
+                    chars.push(ch.to_string());
+
+                    State::Next
+                }
+                (State::Char(ch), ']') => {
+                    chars.push(ch.to_string());
+
+                    State::End
+                }
+                (State::Char(ch), '-') => State::Range(ch),
+
+                (State::Range(_), ' ') => panic!("oh crap"),
+                (State::Range(_), ']') => panic!("oh crap"),
+                (State::Range(from), to) => {
+                    chars.extend((from..=to).map(|ch| ch.to_string()));
+
+                    State::Next
+                }
+
+                (State::Unicode(string), ' ') => {
+                    let value = u32::from_str_radix(&string, 16).unwrap();
+                    let value = char::from_u32(value).unwrap();
+
+                    chars.push(value.to_string());
+
+                    State::Next
+                }
+                (State::Unicode(string), '\\') => {
+                    let value = u32::from_str_radix(&string, 16).unwrap();
+                    let value = char::from_u32(value).unwrap();
+
+                    chars.push(value.to_string());
+
+                    State::Escaped
+                }
+                (State::Unicode(string), ']') => {
+                    let value = u32::from_str_radix(&string, 16).unwrap();
+                    let value = char::from_u32(value).unwrap();
+
+                    chars.push(value.to_string());
+
+                    State::End
+                }
+                (State::Unicode(mut string), ch) => {
+                    string.push(ch);
+
+                    State::Unicode(string)
+                }
+
+                (State::Sequence(SequenceState::Default, string), '}') => {
+                    chars.push(string);
+
+                    State::Next
+                }
+                (State::Sequence(SequenceState::Default, string), '\\') => {
+                    State::Sequence(SequenceState::Escaped, string)
+                }
+                (State::Sequence(SequenceState::Default, mut string), ch) => {
+                    string.push(ch);
+
+                    State::Sequence(SequenceState::Default, string)
+                }
+
+                (State::Sequence(SequenceState::Escaped, string), 'u') => {
+                    State::Sequence(SequenceState::Unicode(String::new()), string)
+                }
+
+                (State::Sequence(SequenceState::Unicode(unicode), mut string), '}') => {
+                    let value = u32::from_str_radix(&unicode, 16).unwrap();
+                    let value = char::from_u32(value).unwrap();
+
+                    string.push(value);
+                    chars.push(string);
+
+                    State::Next
+                }
+                (State::Sequence(SequenceState::Unicode(unicode), mut string), '\\') => {
+                    let value = u32::from_str_radix(&unicode, 16).unwrap();
+                    let value = char::from_u32(value).unwrap();
+
+                    string.push(value);
+
+                    State::Sequence(SequenceState::Escaped, string)
+                }
+                (State::Sequence(SequenceState::Unicode(mut unicode), mut string), ch) => {
+                    unicode.push(ch);
+
+                    if unicode.len() < 4 {
+                        State::Sequence(SequenceState::Unicode(unicode), string)
+                    } else {
+                        let value = u32::from_str_radix(&unicode, 16).unwrap();
+                        let value = char::from_u32(value).unwrap();
+
+                        string.push(value);
+
+                        State::Sequence(SequenceState::Default, string)
+                    }
+                }
+
+                (state, _) => state,
+            };
+        }
+
+        chars
+    }
+}

wordlang/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "wordlang"
+version = "0.1.0"
+authors = ["Anders Olsson <anders.e.olsson@gmail.com>"]
+edition = "2018"
+
+[dependencies]
+isolang = "1.0"
+once_cell = "1.4"
+regex = "1.3"

wordlang/src/lib.rs

@@ -0,0 +1,32 @@
+pub use isolang::Language;
+
+mod data;
+
+pub trait LanguageExt {
+    fn is_match(&self, word: &str) -> Option<bool>;
+}
+
+pub fn matches(word: &str) -> Vec<Language> {
+    data::REGEX_SET
+        .matches(word)
+        .into_iter()
+        .map(|idx| data::LANGUAGES[idx])
+        .collect::<Vec<_>>()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_matches() {
+        let matches = matches("hello");
+
+        assert_eq!(matches.len(), 147);
+    }
+
+    #[test]
+    fn test_is_match() {
+        assert_eq!(Language::Swe.is_match("hello"), Some(true));
+    }
+}