From f2aad308e7559c9a1a5d05b3b209f255708af8cf Mon Sep 17 00:00:00 2001 From: jyn Date: Sat, 15 Apr 2023 07:04:22 -0500 Subject: [PATCH] Add support for `-Zbuild-std` - Update to a version of rustwide that supports `fetch_build_std_targets` - Don't try to download rust-std for tier 3 targets - Add a local test - Don't try to make API calls to crates.io for local crates This shaves 16 seconds off the build, and also avoids some unnecessary network calls. Note that this calls `cargo fetch -Zbuild-std` unconditionally just because it was easier; I doubt it'll cause any trouble but I can change it if you like. --- Cargo.lock | 33 +++++++---- Cargo.toml | 5 +- src/docbuilder/rustwide_builder.rs | 90 +++++++++++++++++++----------- tests/crates/build-std/Cargo.lock | 7 +++ tests/crates/build-std/Cargo.toml | 12 ++++ tests/crates/build-std/src/lib.rs | 14 +++++ 6 files changed, 114 insertions(+), 47 deletions(-) create mode 100644 tests/crates/build-std/Cargo.lock create mode 100644 tests/crates/build-std/Cargo.toml create mode 100644 tests/crates/build-std/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index aa9506553..a0644e1e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1099,9 +1099,9 @@ dependencies = [ [[package]] name = "crates-index" -version = "0.18.11" +version = "0.19.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "599f67b56f40863598cb30450427049935d05de2e36c61d33c050f04d7ec8cf2" +checksum = "65aa5fcd68f892b56202f15a18a53308b2d489b728958dbce48d2d1f3bbaa685" dependencies = [ "git2", "hex", @@ -1114,8 +1114,8 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "smartstring", - "toml 0.5.11", + "smol_str", + "toml 0.7.3", ] [[package]] @@ -1838,9 +1838,9 @@ checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" [[package]] name = "git2" -version = "0.14.4" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0155506aab710a86160ddb504a480d2964d7ab5b9e62419be69e0032bc5931c" +checksum = "8b7905cdfe33d31a88bb2e8419ddd054451f5432d1da9eaf2ac7804ee1ea12d5" dependencies = [ "bitflags 1.3.2", "libc", @@ -2997,9 +2997,9 @@ checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "libgit2-sys" -version = "0.13.5+1.4.5" +version = "0.15.1+1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e5ea06c26926f1002dd553fded6cfcdc9784c1f60feeb58368b4d9b07b6dba" +checksum = "fb4577bde8cdfc7d6a2a4bcb7b049598597de33ffd337276e9c7db6cd4a2cee7" dependencies = [ "cc", "libc", @@ -3022,9 +3022,9 @@ dependencies = [ [[package]] name = "libssh2-sys" -version = "0.2.23" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b094a36eb4b8b8c8a7b4b8ae43b2944502be3e59cd87687595cf6b0a71b3f4ca" +checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" dependencies = [ "cc", "libc", @@ -4364,9 +4364,9 @@ checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" [[package]] name = "rustwide" -version = "0.15.2" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1b0070cdf593a5b66ba6160974bbfb504a837f8c06f3357a623c9a7b6619c64" +checksum = "4f77082ac67ce0958dca987d10f5a994114cd98ee0a81afa6f5bc44fb2e416f0" dependencies = [ "attohttpc", "base64 0.13.1", @@ -4862,6 +4862,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "smol_str" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74212e6bbe9a4352329b2f68ba3130c15a3f26fe88ff22dbdc6cdd58fa85e99c" +dependencies = [ + "serde", +] + [[package]] name = "socket2" version = "0.4.9" diff --git a/Cargo.toml b/Cargo.toml index f1ca8ffdf..1ea46c557 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ edition = "2021" [workspace] exclude = [ "ignored", + "tests", ".workspace", ".rustwide-docker", ] @@ -30,7 +31,7 @@ tracing-subscriber = { version = "0.3.16", default-features = false, features = tracing-log = "0.1.3" regex = "1" clap = { version = "4.0.22", features = [ "derive" ] } -crates-index = { version = "0.18.5", optional = true } +crates-index = { version = "0.19", optional = true } rayon = "1.6.1" num_cpus = "1.15.0" crates-index-diff = { version = "17.0.0", features = [ "max-performance" ]} @@ -52,7 +53,7 @@ toml = "0.7.2" schemamama = "0.3" schemamama_postgres = "0.3" prometheus = { version = "0.13.0", default-features = false } -rustwide = { version = "0.15.0", features = ["unstable-toolchain-ci"] } +rustwide = { version = "0.16.0", features = ["unstable-toolchain-ci", "unstable"] } mime_guess = "2" zstd = "0.12.0" hostname = "0.3.1" diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index d3b519bcf..8df0e3876 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -5,7 +5,6 @@ use crate::db::{ }; use crate::docbuilder::{crates::crates_from_path, Limits}; use crate::error::Result; -use crate::index::api::ReleaseData; use crate::repositories::RepositoryStatsUpdater; use crate::storage::{rustdoc_archive_path, source_archive_path}; use crate::utils::{ @@ -16,7 +15,7 @@ use crate::RUSTDOC_STATIC_STORAGE_PREFIX; use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage}; use crate::{Config, Context, Index, Metrics, Storage}; use anyhow::{anyhow, bail, Context as _, Error}; -use docsrs_metadata::{Metadata, DEFAULT_TARGETS, HOST_TARGET}; +use docsrs_metadata::{BuildTargets, Metadata, DEFAULT_TARGETS, HOST_TARGET}; use failure::Error as FailureError; use postgres::Client; use regex::Regex; @@ -367,6 +366,7 @@ impl RustwideBuilder { let mut build_dir = self.workspace.build_dir(&format!("{name}-{version}")); build_dir.purge().map_err(FailureError::compat)?; + let is_local = matches!(kind, PackageKind::Local(_)); let krate = match kind { PackageKind::Local(path) => Crate::local(path), PackageKind::CratesIo => Crate::crates_io(name, version), @@ -383,16 +383,19 @@ impl RustwideBuilder { let successful = build_dir .build(&self.toolchain, &krate, self.prepare_sandbox(&limits)) .run(|build| { - (|| -> Result { - use docsrs_metadata::BuildTargets; + let metadata = Metadata::from_crate_root(build.host_source_dir())?; + let BuildTargets { + default_target, + other_targets, + } = metadata.targets(self.config.include_default_targets); + let mut targets = vec![default_target]; + targets.extend(&other_targets); + // Fetch this before we enter the sandbox, so networking isn't blocked. + build.fetch_build_std_dependencies(&targets)?; + (|| -> Result { let mut has_docs = false; let mut successful_targets = Vec::new(); - let metadata = Metadata::from_crate_root(build.host_source_dir())?; - let BuildTargets { - default_target, - other_targets, - } = metadata.targets(self.config.include_default_targets); // Perform an initial build let mut res = @@ -488,19 +491,24 @@ impl RustwideBuilder { self.metrics.non_library_builds.inc(); } - let release_data = match self - .index - .api() - .get_release_data(name, version) - .with_context(|| { - format!("could not fetch releases-data for {name}-{version}") - }) { - Ok(data) => data, - Err(err) => { - report_error(&err); - ReleaseData::default() + let release_data = if !is_local { + match self + .index + .api() + .get_release_data(name, version) + .with_context(|| { + format!("could not fetch releases-data for {name}-{version}") + }) { + Ok(data) => Some(data), + Err(err) => { + report_error(&err); + None + } } - }; + } else { + None + } + .unwrap_or_default(); let cargo_metadata = res.cargo_metadata.root(); let repository = self.get_repo(cargo_metadata)?; @@ -530,11 +538,13 @@ impl RustwideBuilder { self.storage.store_one(build_log_path, res.build_log)?; // Some crates.io crate data is mutable, so we proactively update it during a release - match self.index.api().get_crate_data(name) { - Ok(crate_data) => { - update_crate_data_in_database(&mut conn, name, &crate_data)? + if !is_local { + match self.index.api().get_crate_data(name) { + Ok(crate_data) => { + update_crate_data_in_database(&mut conn, name, &crate_data)? + } + Err(err) => warn!("{:#?}", err), } - Err(err) => warn!("{:#?}", err), } if res.result.successful { @@ -725,14 +735,6 @@ impl RustwideBuilder { limits: &Limits, mut rustdoc_flags_extras: Vec, ) -> Result> { - // If the explicit target is not a tier one target, we need to install it. - if !docsrs_metadata::DEFAULT_TARGETS.contains(&target) { - // This is a no-op if the target is already installed. - self.toolchain - .add_target(&self.workspace, target) - .map_err(FailureError::compat)?; - } - // Add docs.rs specific arguments let mut cargo_args = vec![ "--offline".into(), @@ -781,6 +783,18 @@ impl RustwideBuilder { rustdoc_flags_extras.extend(UNCONDITIONAL_ARGS.iter().map(|&s| s.to_owned())); let cargo_args = metadata.cargo_args(&cargo_args, &rustdoc_flags_extras); + // If the explicit target is not a tier one target, we need to install it. + let has_build_std = cargo_args.windows(2).any(|args| { + args[0].starts_with("-Zbuild-std") + || (args[0] == "-Z" && args[1].starts_with("build-std")) + }) || cargo_args.last().unwrap().starts_with("-Zbuild-std"); + if !docsrs_metadata::DEFAULT_TARGETS.contains(&target) && !has_build_std { + // This is a no-op if the target is already installed. + self.toolchain + .add_target(&self.workspace, target) + .map_err(FailureError::compat)?; + } + let mut command = build .cargo() .timeout(Some(limits.timeout())) @@ -1218,4 +1232,14 @@ mod tests { Ok(()) }); } + + #[test] + #[ignore] + fn test_build_std() { + wrapper(|env| { + assert!(RustwideBuilder::init(env)? + .build_local_package(Path::new("tests/crates/build-std"))?); + Ok(()) + }) + } } diff --git a/tests/crates/build-std/Cargo.lock b/tests/crates/build-std/Cargo.lock new file mode 100644 index 000000000..1235124d2 --- /dev/null +++ b/tests/crates/build-std/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "build-std" +version = "0.1.0" diff --git a/tests/crates/build-std/Cargo.toml b/tests/crates/build-std/Cargo.toml new file mode 100644 index 000000000..9ed4a17b7 --- /dev/null +++ b/tests/crates/build-std/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "build-std" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +cargo-args = ["-Zbuild-std=core"] diff --git a/tests/crates/build-std/src/lib.rs b/tests/crates/build-std/src/lib.rs new file mode 100644 index 000000000..7d12d9af8 --- /dev/null +++ b/tests/crates/build-std/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: usize, right: usize) -> usize { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +}