diff --git a/CHANGELOG.md b/CHANGELOG.md index 353b4dee..3c134b59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- smoother large file download&proxy support (#463) ### Fixed - When queriying GitHub for the list of releases, retrieve more items (#462) diff --git a/Cargo.lock b/Cargo.lock index db97d2d2..0f92ecab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -323,6 +323,19 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width 0.1.14", + "windows-sys 0.52.0", +] + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -481,6 +494,12 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -556,12 +575,16 @@ version = "0.13.1-dev" dependencies = [ "assert_cmd", "async-trait", + "bytes", "clap", "clap_complete", "directories", "env_logger", "flate2", "guess_host_triple", + "indicatif", + "indicatif-log-bridge", + "lazy_static", "log", "miette", "openssl", @@ -575,6 +598,7 @@ dependencies = [ "thiserror 2.0.3", "tokio", "tokio-retry", + "tokio-stream", "update-informer", "winapi", "winreg 0.52.0", @@ -662,6 +686,17 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -682,6 +717,7 @@ checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -1127,6 +1163,29 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.17.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf675b85ed934d3c67b5c5469701eec7db22689d0a2139d856e0925fa28b281" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width 0.2.0", + "web-time", +] + +[[package]] +name = "indicatif-log-bridge" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63703cf9069b85dbe6fe26e1c5230d013dee99d3559cd3d02ba39e099ef7ab02" +dependencies = [ + "indicatif", + "log", +] + [[package]] name = "inout" version = "0.1.3" @@ -1178,6 +1237,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.165" @@ -1273,7 +1338,7 @@ dependencies = [ "terminal_size", "textwrap", "thiserror 1.0.69", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -1337,6 +1402,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.36.5" @@ -1495,6 +1566,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +[[package]] +name = "portable-atomic" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + [[package]] name = "powerfmt" version = "0.2.0" @@ -1712,10 +1789,12 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-socks", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "windows-registry", ] @@ -2195,7 +2274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" dependencies = [ "unicode-linebreak", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -2350,6 +2429,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.12" @@ -2418,6 +2508,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "untrusted" version = "0.9.0" @@ -2588,6 +2684,19 @@ version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.72" @@ -2598,6 +2707,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.25.4" diff --git a/Cargo.toml b/Cargo.toml index 70f4ebd1..0c70e6bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,16 +15,20 @@ rust-version = "1.74.1" [dependencies] async-trait = "0.1.83" +bytes = "1.8.0" clap = { version = "4.5.21", features = ["derive", "env"] } clap_complete = "4.5.38" directories = "5.0.1" env_logger = "0.11.5" flate2 = "1.0.35" guess_host_triple = "0.1.4" +indicatif = "0.17.9" +indicatif-log-bridge = "0.2.3" +lazy_static = "1.0" log = "0.4.22" miette = { version = "7.3.0", features = ["fancy"] } regex = "1.11.1" -reqwest = { version = "0.12.9", features = ["blocking", "socks"] } +reqwest = { version = "0.12.9", features = ["blocking", "socks", "stream"] } retry = "2.0.0" serde_json = "1.0.133" strum = { version = "0.26.3", features = ["derive"] } @@ -33,6 +37,7 @@ tempfile = "3.14.0" thiserror = "2.0.3" tokio = { version = "1.41.1", features = ["full"] } tokio-retry = "0.3.0" +tokio-stream = "0.1.17" update-informer = "1.1.0" xz2 = "0.1.7" zip = "2.2.1" diff --git a/src/lib.rs b/src/lib.rs index b8b92cc2..a35e72cc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,9 +8,11 @@ pub mod toolchain; pub mod logging { use env_logger::{Builder, Env, WriteStyle}; + use crate::toolchain::PROCESS_BARS; + /// Initializes the logger pub fn initialize_logger(log_level: &str) { - Builder::from_env(Env::default().default_filter_or(log_level)) + let logger = Builder::from_env(Env::default().default_filter_or(log_level)) .format(|buf, record| { use std::io::Write; writeln!( @@ -21,7 +23,13 @@ pub mod logging { ) }) .write_style(WriteStyle::Always) - .init(); + .build(); + let level = logger.filter(); + // make logging and process bar no longer mixed up + indicatif_log_bridge::LogWrapper::new(PROCESS_BARS.clone(), logger) + .try_init() + .unwrap(); + log::set_max_level(level); } } diff --git a/src/toolchain/mod.rs b/src/toolchain/mod.rs index 4b65c651..69652007 100644 --- a/src/toolchain/mod.rs +++ b/src/toolchain/mod.rs @@ -25,10 +25,12 @@ use std::{ fs::{create_dir_all, remove_file, File}, io::{copy, Write}, path::{Path, PathBuf}, + sync::atomic::{self, AtomicUsize}, }; use tar::Archive; use tokio::{fs::remove_dir_all, sync::mpsc}; use tokio_retry::{strategy::FixedInterval, Retry}; +use tokio_stream::StreamExt; use xz2::read::XzDecoder; use zip::ZipArchive; @@ -36,6 +38,11 @@ pub mod gcc; pub mod llvm; pub mod rust; +lazy_static::lazy_static! { + pub static ref PROCESS_BARS: indicatif::MultiProgress = indicatif::MultiProgress::new(); + pub static ref DOWNLOAD_CNT: AtomicUsize = AtomicUsize::new(0); +} + pub enum InstallMode { Install, Update, @@ -49,6 +56,47 @@ pub trait Installable { fn name(&self) -> String; } +/// Get https proxy from environment variables(if any) +/// +/// sadly there is not standard on the environment variable name for the proxy, but it seems +/// that the most common are: +/// +/// - https_proxy(or http_proxy for http) +/// - HTTPS_PROXY(or HTTP_PROXY for http) +/// - all_proxy +/// - ALL_PROXY +/// +/// hence we will check for all of them +fn https_proxy() -> Option { + for proxy in ["https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"] { + if let Ok(proxy_addr) = std::env::var(proxy) { + info!("Get Proxy from env var: {}={}", proxy, proxy_addr); + return Some(proxy_addr); + } + } + None +} + +/// Build a reqwest client with proxy if env var is set +fn build_proxy_blocking_client() -> Result { + let mut builder = reqwest::blocking::Client::builder(); + if let Some(proxy) = https_proxy() { + builder = builder.proxy(reqwest::Proxy::https(&proxy).unwrap()); + } + let client = builder.build()?; + Ok(client) +} + +/// Build a reqwest client with proxy if env var is set +fn build_proxy_async_client() -> Result { + let mut builder = reqwest::Client::builder(); + if let Some(proxy) = https_proxy() { + builder = builder.proxy(reqwest::Proxy::https(&proxy).unwrap()); + } + let client = builder.build()?; + Ok(client) +} + /// Downloads a file from a URL and uncompresses it, if necesary, to the output directory. pub async fn download_file( url: String, @@ -69,9 +117,49 @@ pub async fn download_file( create_dir_all(output_directory) .map_err(|_| Error::CreateDirectory(output_directory.to_string()))?; } - info!("Downloading '{}'", &file_name); - let resp = reqwest::get(&url).await?; - let bytes = resp.bytes().await?; + + let resp = { + let client = build_proxy_async_client()?; + client.get(&url).send().await? + }; + let bytes = { + let len = resp.content_length(); + + // draw a progress bar + let sty = indicatif::ProgressStyle::with_template( + "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", + ) + .unwrap() + .progress_chars("##-"); + let bar = len + .map(indicatif::ProgressBar::new) + .unwrap_or(indicatif::ProgressBar::no_length()); + let bar = PROCESS_BARS.add(bar); + bar.set_style(sty); + bar.set_message(file_name.to_string()); + DOWNLOAD_CNT.fetch_add(1, atomic::Ordering::Relaxed); + + let mut size_downloaded = 0; + let mut stream = resp.bytes_stream(); + let mut bytes = bytes::BytesMut::new(); + while let Some(chunk_result) = stream.next().await { + let chunk = chunk_result?; + size_downloaded += chunk.len(); + bar.set_position(size_downloaded as u64); + + bytes.extend(&chunk); + } + bar.finish_with_message(format!("{} download complete", file_name)); + // leave the progress bar after completion + if DOWNLOAD_CNT.fetch_sub(1, atomic::Ordering::Relaxed) == 1 { + // clear all progress bars + PROCESS_BARS.clear().unwrap(); + info!("All downloads complete"); + } + // wait while DOWNLOAD_CNT is not zero + + bytes.freeze() + }; if uncompress { let extension = Path::new(file_name).extension().unwrap().to_str().unwrap(); match extension { @@ -286,7 +374,7 @@ pub fn github_query(url: &str) -> Result { .unwrap(), ); } - let client = Client::new(); + let client = build_proxy_blocking_client()?; let json = retry( Fixed::from_millis(100).take(5), || -> Result { diff --git a/src/toolchain/rust.rs b/src/toolchain/rust.rs index 1eff4122..fb5b7678 100644 --- a/src/toolchain/rust.rs +++ b/src/toolchain/rust.rs @@ -73,11 +73,16 @@ pub struct XtensaRust { impl XtensaRust { /// Get the latest version of Xtensa Rust toolchain. pub async fn get_latest_version() -> Result { - let json = github_query(XTENSA_RUST_LATEST_API_URL)?; + let json = tokio::task::spawn_blocking(|| github_query(XTENSA_RUST_LATEST_API_URL)) + .await + .unwrap()?; let mut version = json["tag_name"].to_string(); version.retain(|c| c != 'v' && c != '"'); - Self::parse_version(&version)?; + let borrowed = version.clone(); + tokio::task::spawn_blocking(move || Self::parse_version(&borrowed)) + .await + .expect("Join blocking task error")?; debug!("Latest Xtensa Rust version: {}", version); Ok(version) } @@ -228,6 +233,15 @@ impl Installable for XtensaRust { let tmp_dir = tempdir_in(path)?; let tmp_dir_path = &tmp_dir.path().display().to_string(); + download_file( + self.src_dist_url.clone(), + "rust-src.tar.xz", + tmp_dir_path, + true, + false, + ) + .await?; + download_file( self.dist_url.clone(), "rust.tar.xz", @@ -261,14 +275,6 @@ impl Installable for XtensaRust { return Err(Error::XtensaRust); } - download_file( - self.src_dist_url.clone(), - "rust-src.tar.xz", - tmp_dir_path, - true, - false, - ) - .await?; info!("Installing 'rust-src' component for Xtensa Rust toolchain"); if !Command::new("/usr/bin/env") .arg("bash")