diff --git a/Cargo.lock b/Cargo.lock index d85c88d..14b1134 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -415,12 +415,15 @@ version = "0.1.0" dependencies = [ "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "mockito 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "termion 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", "tui 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "webbrowser 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -633,6 +636,16 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "mio-uds" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "miow" version = "0.2.1" @@ -1267,13 +1280,18 @@ dependencies = [ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-fs 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-reactor 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-sync 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-timer 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-udp 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1286,6 +1304,16 @@ dependencies = [ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tokio-codec" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tokio-current-thread" version = "0.1.6" @@ -1304,6 +1332,16 @@ dependencies = [ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tokio-fs" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tokio-io" version = "0.1.12" @@ -1381,6 +1419,37 @@ dependencies = [ "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tokio-udp" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-uds" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)", + "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "try-lock" version = "0.2.2" @@ -1651,6 +1720,7 @@ dependencies = [ "checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599" "checksum miniz_oxide 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "304f66c19be2afa56530fa7c39796192eef38618da8d19df725ad7c6d6b2aaae" "checksum mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)" = "83f51996a3ed004ef184e16818edc51fadffe8e7ca68be67f9dee67d84d0ff23" +"checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum mockito 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aee38c301104cc75a6628a4360be706fbdf84290c15a120b7e54eca5881c3450" "checksum native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4b2df1a4c22fd44a62147fd8f13dd0f95c9d8ca7b2610299b2a2f9cf8964274e" @@ -1722,14 +1792,18 @@ dependencies = [ "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" "checksum tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)" = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" "checksum tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46" +"checksum tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5c501eceaf96f0e1793cf26beb63da3d11c738c4a943fdf3746d81d64684c39f" "checksum tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "d16217cad7f1b840c5a97dfb3c43b0c871fef423a6e8d2118c604e843662a443" "checksum tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "0f27ee0e6db01c5f0b2973824547ce7e637b2ed79b891a9677b0de9bd532b6ac" +"checksum tokio-fs 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe6dc22b08d6993916647d108a1a7d15b9cd29c4f4496c62b92c45b5041b7af" "checksum tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "5090db468dad16e1a7a54c8c67280c5e4b544f3d3e018f0b913b400261f85926" "checksum tokio-reactor 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "c56391be9805bc80163151c0b9e5164ee64f4b0200962c346fea12773158f22d" "checksum tokio-sync 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "d06554cce1ae4a50f42fba8023918afa931413aded705b560e29600ccf7c6d76" "checksum tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1d14b10654be682ac43efee27401d792507e30fd8d26389e1da3b185de2e4119" "checksum tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "2bd2c6a3885302581f4401c82af70d792bb9df1700e7437b0aeb4ada94d5388c" "checksum tokio-timer 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "f2106812d500ed25a4f38235b9cae8f78a09edf43203e16e59c3b769a342a60e" +"checksum tokio-udp 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f02298505547f73e60f568359ef0d016d5acd6e830ab9bc7c4a5b3403440121b" +"checksum tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" "checksum try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "283d3b89e1368717881a9d51dad843cc435380d8109c9e47d38780a324698d8b" "checksum tui 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "73b422ff4986065d33272b587907654f918a3fe8702786a8110bf68dede0d8ee" diff --git a/Cargo.toml b/Cargo.toml index 7156b24..6dc2211 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,9 @@ edition = "2018" [dependencies] termion = "1.5.3" reqwest = "0.9.22" +futures = "0.1.26" +tokio = "0.1.16" +lazy_static = "1.4.0" serde = { version = "1.0", features = ["derive"] } num_cpus = "1.10.1" mockito = "0.21.0" diff --git a/src/hn.rs b/src/hn.rs index 18e0ca1..0f7605a 100644 --- a/src/hn.rs +++ b/src/hn.rs @@ -2,19 +2,22 @@ extern crate num_cpus; extern crate reqwest; use chrono::Utc; +use futures::future::join_all; +use futures::Future; +use futures::Stream; +use reqwest::r#async::Client; use serde::Deserialize; -use std::ops::DerefMut; -use std::sync::Arc; -use std::sync::LockResult; -use std::sync::Mutex; -use std::sync::MutexGuard; -use std::thread; - -use self::reqwest::header::CONNECTION; +use tokio::runtime::Runtime; +use tokio::sync::mpsc::channel; + use crate::time; #[cfg(test)] use mockito; +lazy_static! { + pub static ref CLIENT: Client = Client::new(); +} + #[derive(Deserialize, Debug)] pub struct Story { pub id: i64, @@ -35,27 +38,19 @@ impl Story { } } -fn next(cursor: &mut Arc>) -> usize { - let result: LockResult> = cursor.lock(); - let mut guard: MutexGuard = result.unwrap(); - let temp = guard.deref_mut(); - *temp += 1; - *temp -} - -/// Get top stories on Hacker News, +/// Fetch top stories on Hacker News, /// Using /v0/topstories.json and /v0/item/{:id}.json endpoints. /// /// https://github.com/HackerNews/API /// /// # Examples /// ``` -/// let stories = match get_top_stories(10) { +/// let stories = match fetch_top_stories(10) { /// Ok(res) => res, /// Err(e) => println!("{:#?}", e) /// } /// ``` -pub fn get_top_stories(num: usize) -> Result, Box> { +pub fn fetch_top_stories(num: usize) -> Result, reqwest::Error> { #[cfg(not(test))] let hn_url = "https://hacker-news.firebaseio.com"; #[cfg(test)] @@ -68,54 +63,65 @@ pub fn get_top_stories(num: usize) -> Result, Box>> = Vec::new(); - let lock: Arc> = Arc::new(Mutex::new(0)); - - for _ in 0..num_cpus::get() { - let mut lock2 = lock.clone(); - let hn_url2 = hn_url.to_owned(); - let vec2 = vec.clone(); - handles.push(thread::spawn(move || { - let mut stories = Vec::new(); - loop { - let cursor = next(&mut lock2); - - if cursor > vec2.len() { - break; - } - - let story_url = format!("{}/v0/item/{}.json", hn_url2, vec2[cursor - 1],); - let client = reqwest::Client::new(); - if let Ok(mut res) = client - .get(story_url.as_str()) - .header(CONNECTION, "Keep-Alive") - .send() - { - if let Ok(story) = res.json() { - stories.push(story) - } - } - } - stories - })); +fn fetch_stories(ids: Vec) -> Result, reqwest::Error> { + if ids.is_empty() { + return Ok(Vec::new()); } - + let mut core = Runtime::new().unwrap(); + let (tx, rx) = channel(ids.len()); + + let num = ids.len(); + let all = ids.into_iter().enumerate().map(move |(i, id)| { + let mut tx = tx.clone(); + fetch_story(id) + .then(move |x| tx.try_send((i, x))) + .map(|_| ()) + .map_err(|e| println!("{:?}", e)) + }); + core.spawn(join_all(all).map(|_| ())); let mut stories = Vec::new(); - for handle in handles.into_iter() { - let mut res = handle.join().unwrap(); - stories.append(&mut res); - } + match rx.take(num as u64).collect().wait() { + Ok(mut x) => { + x.sort_by(|a, b| { + let (i1, _) = a; + let (i2, _) = b; + i1.cmp(i2) + }); + for s in x { + let (_, story) = s; + if let Ok(st) = story { + stories.push(st) + } + } + } + Err(e) => eprintln!("{:?}", e), + }; Ok(stories) } +fn fetch_story(id: i64) -> impl Future { + #[cfg(not(test))] + let hn_url = "https://hacker-news.firebaseio.com"; + #[cfg(test)] + let hn_url = &mockito::server_url(); + + let url = format!("{}/v0/item/{}.json", hn_url, id); + CLIENT + .get(url.as_str()) + .send() + .and_then(move |mut res| res.json()) +} + #[cfg(test)] mod tests { - use crate::hn::get_top_stories; + use crate::hn::fetch_top_stories; use mockito::mock; #[test] - fn test_get_get_top_stories1() { + fn test_fetch_top_stories1() { let _m1 = mock("GET", "/v0/topstories.json") .with_status(200) .with_header("content-type", "application/json") @@ -129,27 +135,27 @@ mod tests { .create(); assert!( - get_top_stories(1).is_ok(), - "get_top_stories should return top stories" + fetch_top_stories(1).is_ok(), + "fetch_top_stories should return top stories" ); - let stories = get_top_stories(1); + let stories = fetch_top_stories(1); let story = &stories.unwrap()[0]; assert_eq!(story.by, String::from("pg")); assert_eq!(story.id, 1); } #[test] - fn test_get_get_top_stories2() { + fn test_fetch_top_stories2() { let _m1 = mock("GET", "/v0/topstories.json").with_status(500).create(); assert!( - get_top_stories(1).is_err(), - "get_top_stories should return an error" + fetch_top_stories(1).is_err(), + "fetch_top_stories should return an error" ); } #[test] - fn test_get_get_top_stories3() { + fn test_fetch_top_stories3() { let _m1 = mock("GET", "/v0/topstories.json") .with_status(200) .with_header("content-type", "application/json") @@ -165,27 +171,27 @@ mod tests { let _m3 = mock("GET", "/v0/item/2.json").with_status(500).create(); assert!( - get_top_stories(5).is_ok(), - "get_top_stories should return stories." + fetch_top_stories(5).is_ok(), + "fetch_top_stories should return stories." ); - let stories = get_top_stories(1); + let stories = fetch_top_stories(1); let story = &stories.unwrap()[0]; assert_eq!(story.by, String::from("pg")); assert_eq!(story.id, 1); } #[test] - fn test_get_get_top_stories4() { + fn test_fetch_top_stories4() { let _m1 = mock("GET", "/v0/topstories.json") .with_status(200) .with_header("content-type", "application/json") .with_body("[]") .create(); assert!( - get_top_stories(5).is_ok(), - "get_top_stories should return stories." + fetch_top_stories(5).is_ok(), + "fetch_top_stories should return stories." ); - let stories = get_top_stories(1); + let stories = fetch_top_stories(1); assert_eq!(stories.unwrap().len(), 0); } } diff --git a/src/main.rs b/src/main.rs index 5d0f1a7..c8f0624 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,9 @@ extern crate clap; extern crate termion; +#[macro_use] +extern crate lazy_static; + use clap::{App as ClapApp, Arg}; use std::io::{stdin, stdout, Write}; use std::sync::mpsc::channel; @@ -52,7 +55,7 @@ fn main() { .unwrap_or("50") .parse() .unwrap_or(50); - match hn::get_top_stories(num) { + match hn::fetch_top_stories(num) { Ok(mut s) => stories.append(&mut s), Err(e) => println!("{:#?}", e), }; @@ -109,7 +112,7 @@ fn main() { a.start_loading(); ui::draw(&mut terminal, &a).unwrap(); let mut stories: Vec = Vec::new(); - match hn::get_top_stories(num) { + match hn::fetch_top_stories(num) { Ok(mut s) => stories.append(&mut s), Err(e) => println!("{:#?}", e), };