diff --git a/crates/ufst/.gitignore b/crates/u-fst/.gitignore similarity index 100% rename from crates/ufst/.gitignore rename to crates/u-fst/.gitignore diff --git a/crates/ufst/COPYING b/crates/u-fst/COPYING similarity index 100% rename from crates/ufst/COPYING rename to crates/u-fst/COPYING diff --git a/crates/ufst/Cargo.toml b/crates/u-fst/Cargo.toml similarity index 96% rename from crates/ufst/Cargo.toml rename to crates/u-fst/Cargo.toml index c52e36c..1a3ae0b 100644 --- a/crates/ufst/Cargo.toml +++ b/crates/u-fst/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "ufst" +name = "u-fst" version = "0.4.7" #:version authors = ["Andrew Gallant ", "Anders Olsson "] description = """ diff --git a/crates/ufst/LICENSE-MIT b/crates/u-fst/LICENSE-MIT similarity index 100% rename from crates/ufst/LICENSE-MIT rename to crates/u-fst/LICENSE-MIT diff --git a/crates/ufst/README.md b/crates/u-fst/README.md similarity index 100% rename from crates/ufst/README.md rename to crates/u-fst/README.md diff --git a/crates/ufst/UNLICENSE b/crates/u-fst/UNLICENSE similarity index 100% rename from crates/ufst/UNLICENSE rename to crates/u-fst/UNLICENSE diff --git a/crates/ufst/build.rs b/crates/u-fst/build.rs similarity index 100% rename from crates/ufst/build.rs rename to crates/u-fst/build.rs diff --git a/crates/ufst/data/wiki-urls-10000 b/crates/u-fst/data/wiki-urls-10000 similarity index 100% rename from crates/ufst/data/wiki-urls-10000 rename to crates/u-fst/data/wiki-urls-10000 diff --git a/crates/ufst/data/wiki-urls-100000 b/crates/u-fst/data/wiki-urls-100000 similarity index 100% rename from crates/ufst/data/wiki-urls-100000 rename to crates/u-fst/data/wiki-urls-100000 diff --git a/crates/ufst/data/words-10000 b/crates/u-fst/data/words-10000 similarity index 100% rename from crates/ufst/data/words-10000 rename to crates/u-fst/data/words-10000 diff --git a/crates/ufst/data/words-100000 b/crates/u-fst/data/words-100000 similarity index 100% rename from crates/ufst/data/words-100000 rename to crates/u-fst/data/words-100000 diff --git a/crates/ufst/rustfmt.toml b/crates/u-fst/rustfmt.toml similarity index 100% rename from crates/ufst/rustfmt.toml rename to crates/u-fst/rustfmt.toml diff --git a/crates/ufst/scripts/gen-common-inputs b/crates/u-fst/scripts/gen-common-inputs similarity index 100% rename from crates/ufst/scripts/gen-common-inputs rename to crates/u-fst/scripts/gen-common-inputs diff --git a/crates/ufst/src/automaton/mod.rs b/crates/u-fst/src/automaton/mod.rs similarity index 100% rename from crates/ufst/src/automaton/mod.rs rename to crates/u-fst/src/automaton/mod.rs diff --git a/crates/ufst/src/bytes.rs b/crates/u-fst/src/bytes.rs similarity index 100% rename from crates/ufst/src/bytes.rs rename to crates/u-fst/src/bytes.rs diff --git a/crates/ufst/src/error.rs b/crates/u-fst/src/error.rs similarity index 100% rename from crates/ufst/src/error.rs rename to crates/u-fst/src/error.rs diff --git a/crates/ufst/src/lib.rs b/crates/u-fst/src/lib.rs similarity index 100% rename from crates/ufst/src/lib.rs rename to crates/u-fst/src/lib.rs diff --git a/crates/ufst/src/raw/build.rs b/crates/u-fst/src/raw/build.rs similarity index 100% rename from crates/ufst/src/raw/build.rs rename to crates/u-fst/src/raw/build.rs diff --git a/crates/ufst/src/raw/common_inputs.rs b/crates/u-fst/src/raw/common_inputs.rs similarity index 100% rename from crates/ufst/src/raw/common_inputs.rs rename to crates/u-fst/src/raw/common_inputs.rs diff --git a/crates/ufst/src/raw/counting_writer.rs b/crates/u-fst/src/raw/counting_writer.rs similarity index 100% rename from crates/ufst/src/raw/counting_writer.rs rename to crates/u-fst/src/raw/counting_writer.rs diff --git a/crates/ufst/src/raw/error.rs b/crates/u-fst/src/raw/error.rs similarity index 100% rename from crates/ufst/src/raw/error.rs rename to crates/u-fst/src/raw/error.rs diff --git a/crates/ufst/src/raw/mod.rs b/crates/u-fst/src/raw/mod.rs similarity index 100% rename from crates/ufst/src/raw/mod.rs rename to crates/u-fst/src/raw/mod.rs diff --git a/crates/ufst/src/raw/node.rs b/crates/u-fst/src/raw/node.rs similarity index 100% rename from crates/ufst/src/raw/node.rs rename to crates/u-fst/src/raw/node.rs diff --git a/crates/ufst/src/raw/registry.rs b/crates/u-fst/src/raw/registry.rs similarity index 100% rename from crates/ufst/src/raw/registry.rs rename to crates/u-fst/src/raw/registry.rs diff --git a/crates/ufst/src/raw/registry_minimal.rs b/crates/u-fst/src/raw/registry_minimal.rs similarity index 100% rename from crates/ufst/src/raw/registry_minimal.rs rename to crates/u-fst/src/raw/registry_minimal.rs diff --git a/crates/ufst/src/raw/tests.rs b/crates/u-fst/src/raw/tests.rs similarity index 100% rename from crates/ufst/src/raw/tests.rs rename to crates/u-fst/src/raw/tests.rs diff --git a/crates/ufst/src/stream.rs b/crates/u-fst/src/stream.rs similarity index 100% rename from crates/ufst/src/stream.rs rename to crates/u-fst/src/stream.rs diff --git a/crates/unf/Cargo.toml b/crates/u-norm/Cargo.toml similarity index 52% rename from crates/unf/Cargo.toml rename to crates/u-norm/Cargo.toml index cdc13d2..7194dd1 100644 --- a/crates/unf/Cargo.toml +++ b/crates/u-norm/Cargo.toml @@ -1,16 +1,25 @@ [package] -name = "unf" +name = "u-norm" version = "0.1.0" edition = "2021" +[lib] +bench = false + +[[bench]] +name = "bench" +harness = false + [dependencies] fst = "0.4.7" tinyvec = { version = "1.6.0", features = ["alloc"] } -ufst = { path = "../ufst" } +u-fst = { path = "../u-fst" } [build-dependencies] -ufst = { path = "../ufst" } +u-fst = { path = "../u-fst" } [dev-dependencies] +criterion = "0.3.5" proptest = "1.0.0" similar-asserts = "1.2.0" +unicode-normalization = "0.1.19" diff --git a/crates/u-norm/README.md b/crates/u-norm/README.md new file mode 100644 index 0000000..fd0b117 --- /dev/null +++ b/crates/u-norm/README.md @@ -0,0 +1,5 @@ +# UNF + +## Todo + +- [ ] Change Decomposition to be `struct Decomposition(u64)` that implements `Iterator` diff --git a/crates/u-norm/benches/bench.rs b/crates/u-norm/benches/bench.rs new file mode 100644 index 0000000..967f24d --- /dev/null +++ b/crates/u-norm/benches/bench.rs @@ -0,0 +1,29 @@ +use std::fs; + +use criterion::{criterion_group, criterion_main, Criterion}; + +use u_norm::nfd; +use unicode_normalization::UnicodeNormalization; + +const ASCII: &str = "all types of normalized"; + +fn criterion_benchmark(c: &mut Criterion) { + let mut group = c.benchmark_group("ASCII"); + + group.bench_function("unf", |b| b.iter(|| nfd(ASCII).count())); + group.bench_function("unicode-normalization", |b| b.iter(|| ASCII.nfd().count())); + + group.finish(); + + let long = fs::read_to_string("benches/long.txt").unwrap(); + + let mut group = c.benchmark_group("Long"); + + group.bench_function("unf", |b| b.iter(|| nfd(&long).count())); + group.bench_function("unicode-normalization", |b| b.iter(|| long.nfd().count())); + + group.finish(); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/crates/u-norm/benches/long.txt b/crates/u-norm/benches/long.txt new file mode 100644 index 0000000..f5259f4 --- /dev/null +++ b/crates/u-norm/benches/long.txt @@ -0,0 +1,2366 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + unicode-normalization/long.txt at master · unicode-rs/unicode-normalization · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Skip to content + + + + + + + + + + +
+ +
+ + + + + + + +
+ + + +
+ + + + + + + + + + +
+
+
+ + + + + + + + + + + + + + + + + +
+ + + + + + +
+ + +
+ + + + + + + + +Permalink + +
+ +
+
+ + + master + + + + +
+
+
+ Switch branches/tags + +
+ + + +
+ +
+ +
+ + +
+ +
+ + + + + + + + + + + + + + + +
+ + + + +
+
+ +
+ +
+ +

+ unicode-normalization/benches/long.txt +

+ Go to file + + +
+ + + + + +
    +
  • + + Go to file + +
    • +
  • +
    • +
  • +
  • + + + Copy path + +
    • +
  • + + + + Copy permalink + + +
    • +
+
+
+ + + + + + + + + +
+ +
+
+ + + +
+
+ David Judd + + + More benchmarks + +
+ + + + + +
+
+ + Latest commit + a31ce6b + Dec 27, 2018 + + + + + + History + + +
+
+ +
+ +
+
+ + + 0 + + contributors + + +
+ +

+ Users who have contributed to this file +

+
+ + + + + + + +
+
+
+ + + + + + + + +
+ +
+ + +
+ + 204 lines (154 sloc) + + 13.1 KB +
+ +
+ + +
+ Raw + + Blame + +
+ +
+ + + + + + + +
+
+ +
+
+ + + + + +
+
+
+ + + +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Original by Markus Kuhn, adapted for HTML by Martin Dürst.
+
UTF-8 encoded sample plain-text file
‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
+
Markus Kuhn [ˈmaʳkʊs kuːn] <mkuhn@acm.org> — 1999-08-20
+
+
The ASCII compatible UTF-8 encoding of ISO 10646 and Unicode
plain-text files is defined in RFC 2279 and in ISO 10646-1 Annex R.
+
+
Using Unicode/UTF-8, you can write in emails and source code things such as
+
Mathematics and Sciences:
+
∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i), ∀x∈ℝ: ⌈x⌉ = −⌊−x⌋, α ∧ ¬β = ¬(¬α ∨ β),
+
ℕ ⊆ ℕ₀ ⊂ ℤ ⊂ ℚ ⊂ ℝ ⊂ ℂ, ⊥ < a ≠ b ≡ c ≤ d ≪ ⊤ ⇒ (A ⇔ B),
+
2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm
+
Linguistics and dictionaries:
+
ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn
Y [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ]
+
APL:
+
((V⍳V)=⍳⍴V)/V←,V ⌷←⍳→⍴∆∇⊃‾⍎⍕⌈
+
Nicer typography in plain text files:
+
╔══════════════════════════════════════════╗
║ ║
║ • ‘single’ and “double” quotes ║
║ ║
║ • Curly apostrophes: “We’ve been here” ║
║ ║
║ • Latin-1 apostrophe and accents: '´` ║
║ ║
║ • ‚deutsche‘ „Anführungszeichen“ ║
║ ║
║ • †, ‡, ‰, •, 3–4, —, −5/+5, ™, … ║
║ ║
║ • ASCII safety test: 1lI|, 0OD, 8B ║
║ ╭─────────╮ ║
║ • the euro symbol: │ 14.95 € │ ║
║ ╰─────────╯ ║
╚══════════════════════════════════════════╝
+
Greek (in Polytonic):
+
The Greek anthem:
+
Σὲ γνωρίζω ἀπὸ τὴν κόψη
τοῦ σπαθιοῦ τὴν τρομερή,
σὲ γνωρίζω ἀπὸ τὴν ὄψη
ποὺ μὲ βία μετράει τὴ γῆ.
+
᾿Απ᾿ τὰ κόκκαλα βγαλμένη
τῶν ῾Ελλήνων τὰ ἱερά
καὶ σὰν πρῶτα ἀνδρειωμένη
χαῖρε, ὦ χαῖρε, ᾿Ελευθεριά!
+
From a speech of Demosthenes in the 4th century BC:
+
Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι,
ὅταν τ᾿ εἰς τὰ πράγματα ἀποβλέψω καὶ ὅταν πρὸς τοὺς
λόγους οὓς ἀκούω· τοὺς μὲν γὰρ λόγους περὶ τοῦ
τιμωρήσασθαι Φίλιππον ὁρῶ γιγνομένους, τὰ δὲ πράγματ᾿
εἰς τοῦτο προήκοντα, ὥσθ᾿ ὅπως μὴ πεισόμεθ᾿ αὐτοὶ
πρότερον κακῶς σκέψασθαι δέον. οὐδέν οὖν ἄλλο μοι δοκοῦσιν
οἱ τὰ τοιαῦτα λέγοντες ἢ τὴν ὑπόθεσιν, περὶ ἧς βουλεύεσθαι,
οὐχὶ τὴν οὖσαν παριστάντες ὑμῖν ἁμαρτάνειν. ἐγὼ δέ, ὅτι μέν
ποτ᾿ ἐξῆν τῇ πόλει καὶ τὰ αὑτῆς ἔχειν ἀσφαλῶς καὶ Φίλιππον
τιμωρήσασθαι, καὶ μάλ᾿ ἀκριβῶς οἶδα· ἐπ᾿ ἐμοῦ γάρ, οὐ πάλαι
γέγονεν ταῦτ᾿ ἀμφότερα· νῦν μέντοι πέπεισμαι τοῦθ᾿ ἱκανὸν
προλαβεῖν ἡμῖν εἶναι τὴν πρώτην, ὅπως τοὺς συμμάχους
σώσομεν. ἐὰν γὰρ τοῦτο βεβαίως ὑπάρξῃ, τότε καὶ περὶ τοῦ
τίνα τιμωρήσεταί τις καὶ ὃν τρόπον ἐξέσται σκοπεῖν· πρὶν δὲ
τὴν ἀρχὴν ὀρθῶς ὑποθέσθαι, μάταιον ἡγοῦμαι περὶ τῆς
τελευτῆς ὁντινοῦν ποιεῖσθαι λόγον.
+
Δημοσθένους, Γ´ ᾿Ολυνθιακὸς
+
Georgian:
+
From a Unicode conference invitation:
+
გთხოვთ ახლავე გაიაროთ რეგისტრაცია Unicode-ის მეათე საერთაშორისო
კონფერენციაზე დასასწრებად, რომელიც გაიმართება 10-12 მარტს,
ქ. მაინცში, გერმანიაში. კონფერენცია შეჰკრებს ერთად მსოფლიოს
ექსპერტებს ისეთ დარგებში როგორიცაა ინტერნეტი და Unicode-ი,
ინტერნაციონალიზაცია და ლოკალიზაცია, Unicode-ის გამოყენება
ოპერაციულ სისტემებსა, და გამოყენებით პროგრამებში, შრიფტებში,
ტექსტების დამუშავებასა და მრავალენოვან კომპიუტერულ სისტემებში.
+
Russian:
+
From a Unicode conference invitation:
+
Зарегистрируйтесь сейчас на Десятую Международную Конференцию по
Unicode, которая состоится 10-12 марта 1997 года в Майнце в Германии.
Конференция соберет широкий круг экспертов по вопросам глобального
Интернета и Unicode, локализации и интернационализации, воплощению и
применению Unicode в различных операционных системах и программных
приложениях, шрифтах, верстке и многоязычных компьютерных системах.
+
Thai (UCS Level 2):
+
Excerpt from a poetry on The Romance of The Three Kingdoms (a Chinese
classic 'San Gua'):
+
[----------------------------|------------------------]
๏ แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช พระปกเกศกองบู๊กู้ขึ้นใหม่
สิบสองกษัตริย์ก่อนหน้าแลถัดไป สององค์ไซร้โง่เขลาเบาปัญญา
ทรงนับถือขันทีเป็นที่พึ่ง บ้านเมืองจึงวิปริตเป็นนักหนา
โฮจิ๋นเรียกทัพทั่วหัวเมืองมา หมายจะฆ่ามดชั่วตัวสำคัญ
เหมือนขับไสไล่เสือจากเคหา รับหมาป่าเข้ามาเลยอาสัญ
ฝ่ายอ้องอุ้นยุแยกให้แตกกัน ใช้สาวนั้นเป็นชนวนชื่นชวนใจ
พลันลิฉุยกุยกีกลับก่อเหตุ ช่างอาเพศจริงหนาฟ้าร้องไห้
ต้องรบราฆ่าฟันจนบรรลัย ฤๅหาใครค้ำชูกู้บรรลังก์ ฯ
+
(The above is a two-column text. If combining characters are handled
correctly, the lines of the second column should be aligned with the
| character above.)
+
Ethiopian:
+
Proverbs in the Amharic language:
+
ሰማይ አይታረስ ንጉሥ አይከሰስ።
ብላ ካለኝ እንደአባቴ በቆመጠኝ።
ጌጥ ያለቤቱ ቁምጥና ነው።
ደሀ በሕልሙ ቅቤ ባይጠጣ ንጣት በገደለው።
የአፍ ወለምታ በቅቤ አይታሽም።
አይጥ በበላ ዳዋ ተመታ።
ሲተረጉሙ ይደረግሙ።
ቀስ በቀስ፥ ዕንቁላል በእግሩ ይሄዳል።
ድር ቢያብር አንበሳ ያስር።
ሰው እንደቤቱ እንጅ እንደ ጉረቤቱ አይተዳደርም።
እግዜር የከፈተውን ጉሮሮ ሳይዘጋው አይድርም።
የጎረቤት ሌባ፥ ቢያዩት ይስቅ ባያዩት ያጠልቅ።
ሥራ ከመፍታት ልጄን ላፋታት።
ዓባይ ማደሪያ የለው፥ ግንድ ይዞ ይዞራል።
የእስላም አገሩ መካ የአሞራ አገሩ ዋርካ።
ተንጋሎ ቢተፉ ተመልሶ ባፉ።
ወዳጅህ ማር ቢሆን ጨርስህ አትላሰው።
እግርህን በፍራሽህ ልክ ዘርጋ።
+
Runes:
+
ᚻᛖ ᚳᚹᚫᚦ ᚦᚫᛏ ᚻᛖ ᛒᚢᛞᛖ ᚩᚾ ᚦᚫᛗ ᛚᚪᚾᛞᛖ ᚾᚩᚱᚦᚹᛖᚪᚱᛞᚢᛗ ᚹᛁᚦ ᚦᚪ ᚹᛖᛥᚫ
+
(Old English, which transcribed into Latin reads 'He cwaeth that he
bude thaem lande northweardum with tha Westsae.' and means 'He said
that he lived in the northern land near the Western Sea.')
+
Braille:
+
⡌⠁⠧⠑ ⠼⠁⠒ ⡍⠜⠇⠑⠹⠰⠎ ⡣⠕⠌
+
⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠙⠑⠁⠙⠒ ⠞⠕ ⠃⠑⠛⠔ ⠺⠊⠹⠲ ⡹⠻⠑ ⠊⠎ ⠝⠕ ⠙⠳⠃⠞
⠱⠁⠞⠑⠧⠻ ⠁⠃⠳⠞ ⠹⠁⠞⠲ ⡹⠑ ⠗⠑⠛⠊⠌⠻ ⠕⠋ ⠙⠊⠎ ⠃⠥⠗⠊⠁⠇ ⠺⠁⠎
⠎⠊⠛⠝⠫ ⠃⠹ ⠹⠑ ⠊⠇⠻⠛⠹⠍⠁⠝⠂ ⠹⠑ ⠊⠇⠻⠅⠂ ⠹⠑ ⠥⠝⠙⠻⠞⠁⠅⠻⠂
⠁⠝⠙ ⠹⠑ ⠡⠊⠑⠋ ⠍⠳⠗⠝⠻⠲ ⡎⠊⠗⠕⠕⠛⠑ ⠎⠊⠛⠝⠫ ⠊⠞⠲ ⡁⠝⠙
⡎⠊⠗⠕⠕⠛⠑⠰⠎ ⠝⠁⠍⠑ ⠺⠁⠎ ⠛⠕⠕⠙ ⠥⠏⠕⠝ ⠰⡡⠁⠝⠛⠑⠂ ⠋⠕⠗ ⠁⠝⠹⠹⠔⠛ ⠙⠑
⠡⠕⠎⠑ ⠞⠕ ⠏⠥⠞ ⠙⠊⠎ ⠙⠁⠝⠙ ⠞⠕⠲
+
⡕⠇⠙ ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲
+
⡍⠔⠙⠖ ⡊ ⠙⠕⠝⠰⠞ ⠍⠑⠁⠝ ⠞⠕ ⠎⠁⠹ ⠹⠁⠞ ⡊ ⠅⠝⠪⠂ ⠕⠋ ⠍⠹
⠪⠝ ⠅⠝⠪⠇⠫⠛⠑⠂ ⠱⠁⠞ ⠹⠻⠑ ⠊⠎ ⠏⠜⠞⠊⠊⠥⠇⠜⠇⠹ ⠙⠑⠁⠙ ⠁⠃⠳⠞
⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ ⡊ ⠍⠊⠣⠞ ⠙⠁⠧⠑ ⠃⠑⠲ ⠔⠊⠇⠔⠫⠂ ⠍⠹⠎⠑⠇⠋⠂ ⠞⠕
⠗⠑⠛⠜⠙ ⠁ ⠊⠕⠋⠋⠔⠤⠝⠁⠊⠇ ⠁⠎ ⠹⠑ ⠙⠑⠁⠙⠑⠌ ⠏⠊⠑⠊⠑ ⠕⠋ ⠊⠗⠕⠝⠍⠕⠝⠛⠻⠹
⠔ ⠹⠑ ⠞⠗⠁⠙⠑⠲ ⡃⠥⠞ ⠹⠑ ⠺⠊⠎⠙⠕⠍ ⠕⠋ ⠳⠗ ⠁⠝⠊⠑⠌⠕⠗⠎
⠊⠎ ⠔ ⠹⠑ ⠎⠊⠍⠊⠇⠑⠆ ⠁⠝⠙ ⠍⠹ ⠥⠝⠙⠁⠇⠇⠪⠫ ⠙⠁⠝⠙⠎
⠩⠁⠇⠇ ⠝⠕⠞ ⠙⠊⠌⠥⠗⠃ ⠊⠞⠂ ⠕⠗ ⠹⠑ ⡊⠳⠝⠞⠗⠹⠰⠎ ⠙⠕⠝⠑ ⠋⠕⠗⠲ ⡹⠳
⠺⠊⠇⠇ ⠹⠻⠑⠋⠕⠗⠑ ⠏⠻⠍⠊⠞ ⠍⠑ ⠞⠕ ⠗⠑⠏⠑⠁⠞⠂ ⠑⠍⠏⠙⠁⠞⠊⠊⠁⠇⠇⠹⠂ ⠹⠁⠞
⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲
+
(The first couple of paragraphs of "A Christmas Carol" by Dickens)
+
Compact font selection example text:
+
ABCDEFGHIJKLMNOPQRSTUVWXYZ /0123456789
abcdefghijklmnopqrstuvwxyz £©µÀÆÖÞßéöÿ
–—‘“”„†•…‰™œŠŸž€ ΑΒΓΔΩαβγδω АБВГДабвгд
∀∂∈ℝ∧∪≡∞ ↑↗↨↻⇣ ┐┼╔╘░►☺♀ fi�⑀₂ἠḂӥẄɐː⍎אԱა
+
Greetings in various languages:
+
Hello world, Καλημέρα κόσμε, コンニチハ
+
Box drawing alignment tests: █
▉
╔══╦══╗ ┌──┬──┐ ╭──┬──╮ ╭──┬──╮ ┏━━┳━━┓ ┎┒┏┑ ╷ ╻ ┏┯┓ ┌┰┐ ▊ ╱╲╱╲╳╳╳
║┌─╨─┐║ │╔═╧═╗│ │╒═╪═╕│ │╓─╁─╖│ ┃┌─╂─┐┃ ┗╃╄┙ ╶┼╴╺╋╸┠┼┨ ┝╋┥ ▋ ╲╱╲╱╳╳╳
║│╲ ╱│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╿ │┃ ┍╅╆┓ ╵ ╹ ┗┷┛ └┸┘ ▌ ╱╲╱╲╳╳╳
╠╡ ╳ ╞╣ ├╢ ╟┤ ├┼─┼─┼┤ ├╫─╂─╫┤ ┣┿╾┼╼┿┫ ┕┛┖┚ ┌┄┄┐ ╎ ┏┅┅┓ ┋ ▍ ╲╱╲╱╳╳╳
║│╱ ╲│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╽ │┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▎
║└─╥─┘║ │╚═╤═╝│ │╘═╪═╛│ │╙─╀─╜│ ┃└─╂─┘┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▏
╚══╩══╝ └──┴──┘ ╰──┴──╯ ╰──┴──╯ ┗━━┻━━┛ └╌╌┘ ╎ ┗╍╍┛ ┋ ▁▂▃▄▅▆▇█
+
+ + + +
+ +
+ + + + +
+ + +
+ + +
 +
+ + +
+ +
+ + +
+ + +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + diff --git a/crates/unf/build.rs b/crates/u-norm/build.rs similarity index 98% rename from crates/unf/build.rs rename to crates/u-norm/build.rs index f28bac7..3832533 100644 --- a/crates/unf/build.rs +++ b/crates/u-norm/build.rs @@ -2,7 +2,7 @@ use std::env; use std::fs; use std::path::Path; -use ufst::raw::Fst; +use u_fst::raw::Fst; fn main() { let data = fs::read_to_string("data/UnicodeData.txt").unwrap(); diff --git a/crates/unf/data/DerivedNormalizationProps.txt b/crates/u-norm/data/DerivedNormalizationProps.txt similarity index 100% rename from crates/unf/data/DerivedNormalizationProps.txt rename to crates/u-norm/data/DerivedNormalizationProps.txt diff --git a/crates/unf/data/NormalizationTest.txt b/crates/u-norm/data/NormalizationTest.txt similarity index 100% rename from crates/unf/data/NormalizationTest.txt rename to crates/u-norm/data/NormalizationTest.txt diff --git a/crates/unf/data/StandardizedVariants.txt b/crates/u-norm/data/StandardizedVariants.txt similarity index 100% rename from crates/unf/data/StandardizedVariants.txt rename to crates/u-norm/data/StandardizedVariants.txt diff --git a/crates/unf/data/UnicodeData.txt b/crates/u-norm/data/UnicodeData.txt similarity index 100% rename from crates/unf/data/UnicodeData.txt rename to crates/u-norm/data/UnicodeData.txt diff --git a/crates/unf/src/lib.rs b/crates/u-norm/src/lib.rs similarity index 96% rename from crates/unf/src/lib.rs rename to crates/u-norm/src/lib.rs index 6353a8a..3d0e713 100644 --- a/crates/unf/src/lib.rs +++ b/crates/u-norm/src/lib.rs @@ -6,8 +6,6 @@ use tinyvec::TinyVec; pub mod table; -use table::Decomposition; - pub fn nfd(s: &str) -> Decompositions> { Decompositions { iter: s.chars().fuse(), @@ -28,6 +26,7 @@ impl Buffer { } } + #[inline(always)] fn push_back(&mut self, ch: char) { let class = table::lookup(ch).combining_class(); @@ -41,10 +40,12 @@ impl Buffer { } } + #[inline(always)] fn sort_pending(&mut self) { self.buffer[self.ready.end..].sort_by_key(|k| k.0); } + #[inline(always)] fn reset(&mut self) { let pending = self.buffer.len() - self.ready.end; @@ -56,6 +57,7 @@ impl Buffer { self.ready = 0..0; } + #[inline(always)] fn increment_next_ready(&mut self) { let next = self.ready.start + 1; @@ -134,14 +136,8 @@ fn decompose(c: char, buffer: &mut Buffer) { } if let Some(decomposed) = table::lookup(c).decomposition() { - match decomposed { - Decomposition::Single(f) => { - decompose(f, buffer); - } - Decomposition::Double(f, s) => { - decompose(f, buffer); - decompose(s, buffer); - } + for d in decomposed { + decompose(d, buffer); } return; } diff --git a/crates/u-norm/src/main.rs b/crates/u-norm/src/main.rs new file mode 100644 index 0000000..9ffa21d --- /dev/null +++ b/crates/u-norm/src/main.rs @@ -0,0 +1,14 @@ +use u_norm::table; + +fn main() { + for c in '\x00'..='\x7f' { + let d = table::lookup(c); + + println!( + "{:?} class: {}, decomp: {:?}", + c, + d.combining_class(), + d.decomposition().map(|d| d.collect::>()) + ); + } +} diff --git a/crates/unf/src/table.rs b/crates/u-norm/src/table.rs similarity index 67% rename from crates/unf/src/table.rs rename to crates/u-norm/src/table.rs index 1d1d06e..ea24d6f 100644 --- a/crates/unf/src/table.rs +++ b/crates/u-norm/src/table.rs @@ -1,8 +1,9 @@ -use ufst::raw::Fst; +use u_fst::raw::Fst; const TABLE: Fst<&'static [u8]> = Fst::new_unchecked(include_bytes!(concat!(env!("OUT_DIR"), "/table.fst"))); +#[inline(always)] pub fn lookup(ch: char) -> Entry { Entry::new( TABLE @@ -13,9 +14,23 @@ pub fn lookup(ch: char) -> Entry { } #[derive(Clone, Copy, PartialEq, Debug)] -pub enum Decomposition { - Single(char), - Double(char, char), +pub struct Decomposition(u64); + +impl Iterator for Decomposition { + type Item = char; + + #[inline(always)] + fn next(&mut self) -> Option { + let d = (self.0 & 0x1FFFFF) as u32; + + if d > 0 { + self.0 >>= 21; + + Some(unsafe { char::from_u32_unchecked(d) }) + } else { + None + } + } } #[derive(Clone, Copy, PartialEq, Debug)] @@ -26,26 +41,17 @@ impl Entry { Self(data) } + #[inline(always)] pub fn combining_class(&self) -> u8 { (self.0 & 0xFF) as u8 } - pub(crate) fn decomposition(&self) -> Option { - let m1 = ((self.0 >> 8) & 0x1FFFFF) as u32; + #[inline(always)] + pub fn decomposition(&self) -> Option { + let data = self.0 >> 8; - if m1 > 0 { - let m2 = ((self.0 >> 29) & 0x1FFFFF) as u32; - - if m2 > 0 { - unsafe { - Some(Decomposition::Double( - char::from_u32_unchecked(m1), - char::from_u32_unchecked(m2), - )) - } - } else { - unsafe { Some(Decomposition::Single(char::from_u32_unchecked(m1))) } - } + if data > 0 { + Some(Decomposition(data)) } else { None } @@ -99,16 +105,14 @@ mod tests { prop_assert_eq!(b.combining_class(), combining_class, "data = {:064b}", data); + let c = b.decomposition().map(|i| i.collect::>()); + match mapping_count { - 0 => prop_assert_eq!(b.decomposition(), None, "data = {:064b}", data), - 1 => prop_assert_eq!(b.decomposition(), Some(Decomposition::Single(decomposition_first)), "data = {:064b}", data), - 2 => prop_assert_eq!(b.decomposition(), Some(Decomposition::Double(decomposition_first, decomposition_second)), "data = {:064b}", data), + 0 => prop_assert_eq!(c, None, "data = {:064b}", data), + 1 => prop_assert_eq!(c, Some(vec![decomposition_first]), "data = {:064b}", data), + 2 => prop_assert_eq!(c, Some(vec![decomposition_first, decomposition_second]), "data = {:064b}", data), _ => unreachable!(), } - - - - // prop_assert_eq!(a, b, "data = {:064b}", data); } } } diff --git a/crates/smol-uca/Cargo.toml b/crates/u-sort/Cargo.toml similarity index 58% rename from crates/smol-uca/Cargo.toml rename to crates/u-sort/Cargo.toml index 616a771..b71c590 100644 --- a/crates/smol-uca/Cargo.toml +++ b/crates/u-sort/Cargo.toml @@ -1,16 +1,17 @@ [package] -name = "smol-uca" +name = "u-sort" version = "0.1.0" edition = "2021" [dependencies] -ufst = { path = "../ufst" } -unf = { path = "../unf" } +parse = { path = "../parse" } +u-fst = { path = "../u-fst" } +u-norm = { path = "../u-norm" } [build-dependencies] bytemuck = "1.9.1" parse = { path = "../parse" } -ufst = { path = "../ufst" } +u-fst = { path = "../u-fst" } [dev-dependencies] proptest = "1.0.0" diff --git a/crates/smol-uca/README.md b/crates/u-sort/README.md similarity index 100% rename from crates/smol-uca/README.md rename to crates/u-sort/README.md diff --git a/crates/smol-uca/build.rs b/crates/u-sort/build.rs similarity index 99% rename from crates/smol-uca/build.rs rename to crates/u-sort/build.rs index 26107b6..44b3c35 100644 --- a/crates/smol-uca/build.rs +++ b/crates/u-sort/build.rs @@ -3,7 +3,7 @@ use std::fs; use std::path::Path; use parse::uca::allkeys; -use ufst::raw::Builder; +use u_fst::raw::Builder; fn main() { println!("cargo:rerun-if-changed=data/allkeys.txt"); diff --git a/crates/smol-uca/data/CollationTest_NON_IGNORABLE.txt b/crates/u-sort/data/CollationTest_NON_IGNORABLE.txt similarity index 100% rename from crates/smol-uca/data/CollationTest_NON_IGNORABLE.txt rename to crates/u-sort/data/CollationTest_NON_IGNORABLE.txt diff --git a/crates/smol-uca/data/CollationTest_SHIFTED.txt b/crates/u-sort/data/CollationTest_SHIFTED.txt similarity index 100% rename from crates/smol-uca/data/CollationTest_SHIFTED.txt rename to crates/u-sort/data/CollationTest_SHIFTED.txt diff --git a/crates/smol-uca/data/allkeys.txt b/crates/u-sort/data/allkeys.txt similarity index 100% rename from crates/smol-uca/data/allkeys.txt rename to crates/u-sort/data/allkeys.txt diff --git a/crates/smol-uca/src/collator.rs b/crates/u-sort/src/collator.rs similarity index 97% rename from crates/smol-uca/src/collator.rs rename to crates/u-sort/src/collator.rs index 082ea8d..7d174c4 100644 --- a/crates/smol-uca/src/collator.rs +++ b/crates/u-sort/src/collator.rs @@ -52,7 +52,8 @@ impl Collator { // advance to last combining C while tail_index < code_points_len { - let combining_class = unf::table::lookup(code_points[tail_index]).combining_class(); + let combining_class = + u_norm::table::lookup(code_points[tail_index]).combining_class(); if debug { eprintln!( @@ -282,7 +283,7 @@ impl Collator { } pub fn sort_key>(&self, input: S) -> Vec { - let normalized = unf::nfd(input.as_ref()).collect::>(); + let normalized = u_norm::nfd(input.as_ref()).collect::>(); let collation_elements = self.collation_elements(&normalized); self.sort_key_from_collation_elements(&collation_elements) @@ -346,7 +347,7 @@ mod tests { "04D0 | 0020 004A 0024 | 0002 0002 0002 |", fmt(&sort_key), "nfd: {:?}", - unf::nfd(fixture) + u_norm::nfd(fixture) .map(|ch| format!("{:04X}", ch as u32)) .collect::>() .join(" ") @@ -364,7 +365,7 @@ mod tests { "FB00 9D00 0268 | 0020 0020 | 0002 0002 |", fmt(&sort_key), "nfd: {:?}", - unf::nfd(fixture) + u_norm::nfd(fixture) .map(|ch| format!("{:04X}", ch as u32)) .collect::>() .join(" ") diff --git a/crates/smol-uca/src/lib.rs b/crates/u-sort/src/lib.rs similarity index 100% rename from crates/smol-uca/src/lib.rs rename to crates/u-sort/src/lib.rs diff --git a/crates/u-sort/src/main.rs b/crates/u-sort/src/main.rs new file mode 100644 index 0000000..799d46c --- /dev/null +++ b/crates/u-sort/src/main.rs @@ -0,0 +1,52 @@ +use std::cmp; +use std::collections::HashMap; +use std::fs; + +use parse::uca::allkeys; + +fn main() { + let allkeys = { + let data = fs::read_to_string("data/allkeys.txt").unwrap(); + + allkeys::parse(&data) + }; + + let mut l1 = 0; + let mut l2 = 0; + let mut l3 = 0; + let mut l4 = 0; + + let mut count = HashMap::new(); + + for entry in allkeys.entries { + count + .entry(entry.elements.len()) + .and_modify(|x| *x += 1) + .or_insert(1); + + for element in entry.elements { + l1 = cmp::max(l1, element.l1); + l2 = cmp::max(l2, element.l2); + l3 = cmp::max(l3, element.l3); + l4 = cmp::max(l4, element.l4); + } + } + + /* + l1 = 16 bits + l2 = 9 bits + l3 = 5 bits + l4 = 0 bits + variable = 1 bit + + total = 31 bits + */ + + println!("l1: {} - {} bit(s)", l1, u16::BITS - l1.leading_zeros()); + println!("l2: {} - {} bit(s)", l2, u16::BITS - l2.leading_zeros()); + println!("l3: {} - {} bit(s)", l3, u8::BITS - l3.leading_zeros()); + println!("l4: {} - {} bit(s)", l4, u16::BITS - l4.leading_zeros()); + println!("variable: 1 bit(s)"); + println!(); + println!("{:#?}", count); +} diff --git a/crates/smol-uca/src/table.rs b/crates/u-sort/src/table.rs similarity index 99% rename from crates/smol-uca/src/table.rs rename to crates/u-sort/src/table.rs index c0d8e36..53d69e4 100644 --- a/crates/smol-uca/src/table.rs +++ b/crates/u-sort/src/table.rs @@ -1,6 +1,6 @@ use std::fmt::Display; -use ufst::raw::{Fst, Output}; +use u_fst::raw::{Fst, Output}; const TABLE: Fst<&'static [u8]> = Fst::new_unchecked(include_bytes!(concat!(env!("OUT_DIR"), "/table.fst"))); diff --git a/crates/smol-uca/src/weights.rs b/crates/u-sort/src/weights.rs similarity index 100% rename from crates/smol-uca/src/weights.rs rename to crates/u-sort/src/weights.rs diff --git a/crates/smol-uca/tests/collation_test.rs b/crates/u-sort/tests/collation_test.rs similarity index 93% rename from crates/smol-uca/tests/collation_test.rs rename to crates/u-sort/tests/collation_test.rs index 5e964e8..001e97d 100644 --- a/crates/smol-uca/tests/collation_test.rs +++ b/crates/u-sort/tests/collation_test.rs @@ -1,7 +1,7 @@ use std::fs::File; use std::io::{BufRead, BufReader}; -use smol_uca::collator::Collator; +use u_sort::collator::Collator; #[test] fn collation_test_non_ignorable() { @@ -55,7 +55,7 @@ fn collation_test_non_ignorable() { let expected_sort_key = rest.rsplit(['[', ']']).nth(1).expect("sort key"); let sort_key = collator.sort_key(&test_string); - let fmt_sort_key = smol_uca::fmt(&sort_key); + let fmt_sort_key = u_sort::fmt(&sort_key); if let Some(prev_sort_key) = prev_sort_key.take() { if sort_key < prev_sort_key { @@ -76,7 +76,7 @@ fn collation_test_non_ignorable() { eprintln!( "Error at line {}: {:?} expected: [{}], got: [{}] ({})", n + 1, - unf::nfd(&test_string).collect::(), + u_norm::nfd(&test_string).collect::(), expected_sort_key, fmt_sort_key, line