From 9f9955a84882261caae17d8a7d697329d644aadf Mon Sep 17 00:00:00 2001 From: Anders Olsson Date: Tue, 18 Dec 2018 14:42:46 +0100 Subject: [PATCH] Cache works, but needs a lot more work. --- Cargo.lock | 45 ++++++++++++++--------- Cargo.toml | 3 ++ src/context.rs | 66 ++++++++++++++++++++++++++++++++++ src/main.rs | 32 +++-------------- src/probe.rs | 4 ++- src/probe/eniro.rs | 17 +++++++-- src/probe/hitta.rs | 19 +++++++--- src/probe/konsument_info.rs | 17 +++++++-- src/probe/telefonforsaljare.rs | 17 +++++++-- src/probe/vem_ringde.rs | 17 +++++++-- 10 files changed, 176 insertions(+), 61 deletions(-) create mode 100644 src/context.rs diff --git a/Cargo.lock b/Cargo.lock index 3865f10..7c13c8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5,7 +5,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "arrayvec" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", @@ -20,6 +20,15 @@ dependencies = [ "safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bincode" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bitflags" version = "1.0.4" @@ -84,19 +93,19 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-epoch 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-epoch 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "crossbeam-epoch" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "arrayvec 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "arrayvec 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -506,9 +515,9 @@ dependencies = [ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl 0.10.15 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl 0.10.16 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys 0.9.39 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.40 (registry+https://github.com/rust-lang/crates.io-index)", "schannel 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", "security-framework 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -548,7 +557,7 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.15" +version = "0.10.16" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -556,7 +565,7 @@ dependencies = [ "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys 0.9.39 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -566,7 +575,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "openssl-sys" -version = "0.9.39" +version = "0.9.40" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1163,7 +1172,7 @@ name = "tokio-threadpool" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-deque 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1318,9 +1327,12 @@ dependencies = [ name = "whoareyou" version = "0.1.0" dependencies = [ + "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "directories 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", "scraper 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1363,8 +1375,9 @@ dependencies = [ [metadata] "checksum adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7e522997b529f05601e05166c07ed17789691f562762c7f3b987263d2dedee5c" -"checksum arrayvec 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "f405cc4c21cd8b784f6c8fc2adf9bc00f59558f0049b5ec21517f875963040cc" +"checksum arrayvec 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)" = "d18513977c2d8261c448511c5c53dc66b26dfccbc3d4446672dea1e71a7d8a26" "checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" +"checksum bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9f2fb9e29e72fd6bc12071533d5dc7664cb01480c59406f656d7ac25c7bd8ff7" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "94f88df23a25417badc922ab0f5716cc1330e87f71ddd9203b3a3ccd9cedf75d" "checksum bytes 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)" = "40ade3d27603c2cb345eb0912aec461a6dec7e06a4ae48589904e808335c7afa" @@ -1374,8 +1387,8 @@ dependencies = [ "checksum core-foundation 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "286e0b41c3a20da26536c6000a280585d519fd07b3956b43aed8a79e9edce980" "checksum core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "716c271e8613ace48344f723b60b900a93150271e5be206212d052bbc0883efa" "checksum crc32fast 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e91d5240c6975ef33aeb5f148f35275c25eda8e8a5f95abe421978b05b8bf192" -"checksum crossbeam-deque 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4fe1b6f945f824c7a25afe44f62e25d714c0cc523f8e99d8db5cd1026e1269d3" -"checksum crossbeam-epoch 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2449aaa4ec7ef96e5fb24db16024b935df718e9ae1cec0a1e68feeca2efca7b8" +"checksum crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "05e44b8cf3e1a625844d1750e1f7820da46044ff6d28f4d43e455ba3e5bb2c13" +"checksum crossbeam-epoch 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f10a4f8f409aaac4b16a5474fb233624238fcdeefb9ba50d5ea059aab63ba31c" "checksum crossbeam-utils 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "41ee4864f4797060e52044376f7d107429ce1fb43460021b126424b7180ee21a" "checksum cssparser 0.24.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b200a7193703a615c8d2751fed1ede39b9c4b3905e09d1ec7064a24688c190fc" "checksum cssparser-macros 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b16e382d9b983fdb9ac6a36b37fdeb84ce3ea81f749febfee3463cfa7f24275e" @@ -1425,9 +1438,9 @@ dependencies = [ "checksum new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0cdc457076c78ab54d5e0d6fa7c47981757f1e34dc39ff92787f217dede586c4" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum num_cpus 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5a69d464bdc213aaaff628444e99578ede64e9c854025aa43b9796530afa9238" -"checksum openssl 0.10.15 (registry+https://github.com/rust-lang/crates.io-index)" = "5e1309181cdcbdb51bc3b6bedb33dfac2a83b3d585033d3f6d9e22e8c1928613" +"checksum openssl 0.10.16 (registry+https://github.com/rust-lang/crates.io-index)" = "ec7bd7ca4cce6dbdc77e7c1230682740d307d1218a87fb0349a571272be749f9" "checksum openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" -"checksum openssl-sys 0.9.39 (registry+https://github.com/rust-lang/crates.io-index)" = "278c1ad40a89aa1e741a1eed089a2f60b18fab8089c3139b542140fc7d674106" +"checksum openssl-sys 0.9.40 (registry+https://github.com/rust-lang/crates.io-index)" = "1bb974e77de925ef426b6bc82fce15fd45bdcbeb5728bffcfc7cdeeb7ce1c2d6" "checksum owning_ref 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49a4b8ea2179e6a2e27411d3bca09ca6dd630821cf6894c6c7c8467a8ee7ef13" "checksum parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f0802bff09003b291ba756dc7e79313e51cc31667e94afbe847def490424cde5" "checksum parking_lot_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad7f7e6ebdc79edff6fdcb87a55b620174f7a989e3eb31b65231f4af57f00b8c" diff --git a/Cargo.toml b/Cargo.toml index 0c2d7c5..b903518 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,3 +8,6 @@ edition = "2018" directories = "1.0" reqwest = "0.9" scraper = "0.9" +serde = "1.0" +serde_derive = "1.0" +bincode = "1.0" diff --git a/src/context.rs b/src/context.rs new file mode 100644 index 0000000..a9cd73b --- /dev/null +++ b/src/context.rs @@ -0,0 +1,66 @@ +use std::fs; +use std::io; + +use directories::ProjectDirs; +use serde_derive::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize)] +pub struct Cache { + timestamp: u64, + pub data: Vec, +} + +pub struct Context { + dirs: ProjectDirs, +} + +impl Context { + pub fn new() -> Context { + Context { + dirs: ProjectDirs::from("com", "logaritmisk", "whoareyou").unwrap(), + } + } + + pub fn cache_get(&mut self, bin: &str, key: &str) -> Option { + let cache = self.dirs.cache_dir().join(format!("{}-{}.bin", bin, key)); + + if cache.exists() { + fs::File::open(cache) + .and_then(|file| { + bincode::deserialize_from(&file) + .map_err(|_| io::Error::new(io::ErrorKind::Other, "failed to deserialize cache entry")) + }) + .ok() + } else { + None + } + } + + pub fn cache_set(&mut self, bin: &str, key: &str, data: D) -> Result<(), io::Error> + where + D: AsRef<[u8]>, + { + let entry = Cache { + timestamp: 0, + data: data.as_ref().to_vec(), + }; + + let cache = self.dirs.cache_dir(); + + if !cache.exists() { + fs::create_dir_all(&cache)?; + } + + let cache = cache.join(format!("{}-{}.bin", bin, key)); + + fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(cache) + .and_then(|mut file| { + bincode::serialize_into(&mut file, &entry) + .map_err(|_| io::Error::new(io::ErrorKind::Other, "failed to serialize cache entry")) + }) + } +} diff --git a/src/main.rs b/src/main.rs index a89d685..75e02ed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,36 +1,12 @@ use std::env; -use directories::ProjectDirs; - +mod context; mod probe; +use crate::context::Context; use crate::probe::*; -struct Context { - dirs: ProjectDirs, -} - -impl Context { - fn new() -> Context { - Context { - dirs: ProjectDirs::from("com", "logaritmisk", "whoareyou").unwrap(), - } - } - - fn cache_get(&mut self, key: &str) -> Option> { - None - } - - fn cache_set(&mut self, key: &str) {} -} - fn main() { - if let Some(proj_dirs) = ProjectDirs::from("com", "logaritmisk", "whoareyou") { - println!("{:?}", proj_dirs.cache_dir()); - } - - std::process::exit(0); - let number = env::args() .nth(1) .expect("must specify a number to search for"); @@ -43,7 +19,9 @@ fn main() { Box::new(VemRingde), ]; + let mut ctx = Context::new(); + for probe in &mut probes { - probe.search(&number); + probe.search(&mut ctx, &number); } } diff --git a/src/probe.rs b/src/probe.rs index 5a317a8..ed5ae2c 100644 --- a/src/probe.rs +++ b/src/probe.rs @@ -10,6 +10,8 @@ pub use self::konsument_info::KonsumentInfo; pub use self::telefonforsaljare::Telefonforsaljare; pub use self::vem_ringde::VemRingde; +use crate::context::Context; + pub trait Probe { - fn search(&mut self, _: &str); + fn search(&mut self, _: &mut Context, _: &str); } diff --git a/src/probe/eniro.rs b/src/probe/eniro.rs index 55ecfa1..6ab7cb7 100644 --- a/src/probe/eniro.rs +++ b/src/probe/eniro.rs @@ -1,10 +1,21 @@ -use scraper::{Html, Selector}; - +use crate::context::Context; use crate::probe::Probe; // https://gulasidorna.eniro.se/hitta:{} pub struct Eniro; impl Probe for Eniro { - fn search(&mut self, _: &str) {} + fn search(&mut self, ctx: &mut Context, number: &str) { + let body = if let Some(cache) = ctx.cache_get("eniro", &number) { + String::from_utf8(cache.data) + .unwrap() + } else { + reqwest::get(&format!("https://gulasidorna.eniro.se/hitta:{}", number)) + .unwrap() + .text() + .unwrap() + }; + + ctx.cache_set("eniro", &number, body.as_bytes()).expect("wut?! why not?!"); + } } diff --git a/src/probe/hitta.rs b/src/probe/hitta.rs index 0ce07ad..92d976a 100644 --- a/src/probe/hitta.rs +++ b/src/probe/hitta.rs @@ -1,19 +1,28 @@ use scraper::{Html, Selector}; +use crate::context::Context; use crate::probe::Probe; // https://www.hitta.se/vem-ringde/{} pub struct Hitta; impl Probe for Hitta { - fn search(&mut self, number: &str) { - let body = reqwest::get(&format!("https://www.hitta.se/vem-ringde/{}", number)) - .unwrap() - .text() - .unwrap(); + fn search(&mut self, ctx: &mut Context, number: &str) { + let body = if let Some(cache) = ctx.cache_get("hitta", &number) { + String::from_utf8(cache.data) + .unwrap() + } else { + reqwest::get(&format!("https://www.hitta.se/vem-ringde/{}", number)) + .unwrap() + .text() + .unwrap() + }; + + ctx.cache_set("hitta", &number, body.as_bytes()).expect("wut?! why not?!"); let document = Html::parse_document(&body); + // Header. let selector = Selector::parse(r#"div[class^="Header__WhoCalledHeader"] > h1"#).unwrap(); print!("Hitta.se:"); diff --git a/src/probe/konsument_info.rs b/src/probe/konsument_info.rs index baf3c69..c37d063 100644 --- a/src/probe/konsument_info.rs +++ b/src/probe/konsument_info.rs @@ -1,10 +1,21 @@ -use scraper::{Html, Selector}; - +use crate::context::Context; use crate::probe::Probe; // http://konsumentinfo.se/telefonnummer/sverige/{} pub struct KonsumentInfo; impl Probe for KonsumentInfo { - fn search(&mut self, _: &str) {} + fn search(&mut self, ctx: &mut Context, number: &str) { + let body = if let Some(cache) = ctx.cache_get("konsument_info", &number) { + String::from_utf8(cache.data) + .unwrap() + } else { + reqwest::get(&format!("http://konsumentinfo.se/telefonnummer/sverige/{}", number)) + .unwrap() + .text() + .unwrap() + }; + + ctx.cache_set("konsument_info", &number, body.as_bytes()).expect("wut?! why not?!"); + } } diff --git a/src/probe/telefonforsaljare.rs b/src/probe/telefonforsaljare.rs index 9bbd40c..9e22477 100644 --- a/src/probe/telefonforsaljare.rs +++ b/src/probe/telefonforsaljare.rs @@ -1,10 +1,21 @@ -use scraper::{Html, Selector}; - +use crate::context::Context; use crate::probe::Probe; // http://www.telefonforsaljare.nu/telefonnummer/{}/ pub struct Telefonforsaljare; impl Probe for Telefonforsaljare { - fn search(&mut self, _: &str) {} + fn search(&mut self, ctx: &mut Context, number: &str) { + let body = if let Some(cache) = ctx.cache_get("telefonforsaljare", &number) { + String::from_utf8(cache.data) + .unwrap() + } else { + reqwest::get(&format!("http://www.telefonforsaljare.nu/telefonnummer/{}/", number)) + .unwrap() + .text() + .unwrap() + }; + + ctx.cache_set("telefonforsaljare", &number, body.as_bytes()).expect("wut?! why not?!"); + } } diff --git a/src/probe/vem_ringde.rs b/src/probe/vem_ringde.rs index 72bd545..be72f1f 100644 --- a/src/probe/vem_ringde.rs +++ b/src/probe/vem_ringde.rs @@ -1,10 +1,21 @@ -use scraper::{Html, Selector}; - +use crate::context::Context; use crate::probe::Probe; // http://vemringde.se/?q={} pub struct VemRingde; impl Probe for VemRingde { - fn search(&mut self, _: &str) {} + fn search(&mut self, ctx: &mut Context, number: &str) { + let body = if let Some(cache) = ctx.cache_get("vem_ringde", &number) { + String::from_utf8(cache.data) + .unwrap() + } else { + reqwest::get(&format!("http://vemringde.se/?q={}", number)) + .unwrap() + .text() + .unwrap() + }; + + ctx.cache_set("vem_ringde", &number, body.as_bytes()).expect("wut?! why not?!"); + } }