diff --git a/src/lib.rs b/src/lib.rs index 9caee98..9aa1c0a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -94,7 +94,6 @@ pub struct ByteNgramReader { inner: Bytes, token: u64, count: u64, - index: u64, mask: u64, } @@ -104,7 +103,6 @@ impl ByteNgramReader { inner: inner.bytes(), token: 0, count: 0, - index: 0, mask: 0, } } @@ -114,7 +112,7 @@ impl Iterator for ByteNgramReader { type Item = ByteNgram; fn next(&mut self) -> Option { - if self.index == 0 { + if self.mask == 0 { if let Some(Ok(byte)) = self.inner.next() { self.token += u64::from(byte); self.token <<= 8; @@ -122,21 +120,18 @@ impl Iterator for ByteNgramReader { if self.count < 6 { self.count += 1; - self.index = self.count; - self.mask = (256u64.pow(self.count as u32) - 1) << 8; + self.mask = ((256u64.pow(self.count as u32) - 1) << 8) + self.count; } else { - self.index = 7; - self.mask = 0xFFFF_FFFF_FFFF_FF00; + self.mask = 0xFFFF_FFFF_FFFF_FF07; } } else { return None; } } - let token = ByteNgram((self.token & self.mask & !0xFF) + self.index); + let token = ByteNgram((self.token & self.mask & !0xFF) + (self.mask & 0xFF)); - self.index -= 1; - self.mask >>= 8; + self.mask = ((self.mask >> 8) & !0xFF) + (self.mask & 0xFF) - 1; Some(token) }