logaritmisk/byte-ngram
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove index from reader.

Browse Source
This commit is contained in:
logaritmisk
2018-01-10 11:41:19 +01:00
parent 86c237f75b
commit 62f29babc7
1 changed files with 5 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/lib.rs
View File

@@ -94,7 +94,6 @@ pub struct ByteNgramReader<R: Read> {
inner: Bytes<R>, inner: Bytes<R>,
token: u64, token: u64,
count: u64, count: u64,
index: u64,
mask: u64, mask: u64,
} }
@@ -104,7 +103,6 @@ impl<R: Read> ByteNgramReader<R> {
inner: inner.bytes(), inner: inner.bytes(),
token: 0, token: 0,
count: 0, count: 0,
index: 0,
mask: 0, mask: 0,
} }
} }
@@ -114,7 +112,7 @@ impl<R: Read> Iterator for ByteNgramReader<R> {
type Item = ByteNgram; type Item = ByteNgram;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
if self.index == 0 { if self.mask == 0 {
if let Some(Ok(byte)) = self.inner.next() { if let Some(Ok(byte)) = self.inner.next() {
self.token += u64::from(byte); self.token += u64::from(byte);
self.token <<= 8; self.token <<= 8;
@@ -122,21 +120,18 @@ impl<R: Read> Iterator for ByteNgramReader<R> {
if self.count < 6 { if self.count < 6 {
self.count += 1; self.count += 1;
self.index = self.count; self.mask = ((256u64.pow(self.count as u32) - 1) << 8) + self.count;
self.mask = (256u64.pow(self.count as u32) - 1) << 8;
} else { } else {
self.index = 7; self.mask = 0xFFFF_FFFF_FFFF_FF07;
self.mask = 0xFFFF_FFFF_FFFF_FF00;
} }
} else { } else {
return None; return None;
} }
} }
let token = ByteNgram((self.token & self.mask & !0xFF) + self.index); let token = ByteNgram((self.token & self.mask & !0xFF) + (self.mask & 0xFF));
self.index -= 1; self.mask = ((self.mask >> 8) & !0xFF) + (self.mask & 0xFF) - 1;
self.mask >>= 8;
Some(token) Some(token)
} }