logaritmisk/byte-ngram
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove index from reader.

Browse Source
This commit is contained in:
logaritmisk
2018-01-10 11:41:19 +01:00
parent 86c237f75b
commit 62f29babc7
1 changed files with 5 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/lib.rs
View File

@@ -94,7 +94,6 @@ pub struct ByteNgramReader<R: Read> {
inner: Bytes<R>,
token: u64,
count: u64,
index: u64,
mask: u64,
}
@@ -104,7 +103,6 @@ impl<R: Read> ByteNgramReader<R> {
inner: inner.bytes(),
token: 0,
count: 0,
index: 0,
mask: 0,
}
}
@@ -114,7 +112,7 @@ impl<R: Read> Iterator for ByteNgramReader<R> {
type Item = ByteNgram;
fn next(&mut self) -> Option<Self::Item> {
if self.index == 0 {
if self.mask == 0 {
if let Some(Ok(byte)) = self.inner.next() {
self.token += u64::from(byte);
self.token <<= 8;
@@ -122,21 +120,18 @@ impl<R: Read> Iterator for ByteNgramReader<R> {
if self.count < 6 {
self.count += 1;
self.index = self.count;
self.mask = (256u64.pow(self.count as u32) - 1) << 8;
self.mask = ((256u64.pow(self.count as u32) - 1) << 8) + self.count;
} else {
self.index = 7;
self.mask = 0xFFFF_FFFF_FFFF_FF00;
self.mask = 0xFFFF_FFFF_FFFF_FF07;
}
} else {
return None;
}
}
let token = ByteNgram((self.token & self.mask & !0xFF) + self.index);
let token = ByteNgram((self.token & self.mask & !0xFF) + (self.mask & 0xFF));
self.index -= 1;
self.mask >>= 8;
self.mask = ((self.mask >> 8) & !0xFF) + (self.mask & 0xFF) - 1;
Some(token)
}