diff --git a/CHANGELOG.md b/CHANGELOG.md index ee22e7bb..bb24683c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +## [0.27.0] - 2024-10-07 + +### Added + +* debug status emitter for when you have problems with ui_test +* cargo features to disable gha or CLI refreshing + +### Fixed + +* CLI refreshing is now reliable and not leaving around fragments +* Can run multiple `Config`s that test the same files in parallel. + +### Changed + * `only`/`ignore` filters now only accept integers, alphabetic characters, `-` and `_` * `only`/ `ignore` filters allow comments by ignoring everything from an `#` onwards +* `OutputConflictHandling` has been replaced by `error_on_output_conflict`, `bless_output_files`, + and `ignore_output_conflict` functions. Custom functions can now be used to implement special + handling of output conflicts. +* `Run` now forwards env vars passed to the compiler to the executable, too ### Removed diff --git a/Cargo.lock b/Cargo.lock index f1131f15..7266326a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -592,7 +592,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.26.5" +version = "0.27.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 8448442e..357bfcc3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ui_test" -version = "0.26.5" +version = "0.27.0" edition = "2021" license = "MIT OR Apache-2.0" description = "A test framework for testing rustc diagnostics output" @@ -23,7 +23,7 @@ rustfix = "0.8.1" cargo-platform = { version = "0.1.2", optional = true } comma = "1.0.0" anyhow = "1.0.6" -indicatif = "0.17.6" +indicatif = { version = "0.17.6", optional = true } prettydiff = { version = "0.7", default-features = false } annotate-snippets = { version = "0.11.2" } levenshtein = "1.0.5" @@ -52,5 +52,6 @@ name = "build_std" test = false [features] -default = ["rustc"] +default = ["rustc", "indicatif", "gha"] +gha = [] rustc = ["dep:cargo-platform", "dep:cargo_metadata"] diff --git a/examples/rustc_basic.rs b/examples/rustc_basic.rs index 7806d79c..b31f546d 100644 --- a/examples/rustc_basic.rs +++ b/examples/rustc_basic.rs @@ -4,6 +4,7 @@ use std::sync::atomic::Ordering; use ui_test::{run_tests, Config}; #[cfg(feature = "rustc")] +#[cfg_attr(test, test)] fn main() -> ui_test::color_eyre::Result<()> { let config = Config::rustc("examples_tests/rustc_basic"); let abort_check = config.abort_check.clone(); diff --git a/examples/rustc_twice_with_different_flags.rs b/examples/rustc_twice_with_different_flags.rs new file mode 100644 index 00000000..35c7a238 --- /dev/null +++ b/examples/rustc_twice_with_different_flags.rs @@ -0,0 +1,29 @@ +#[cfg(feature = "rustc")] +use std::sync::atomic::Ordering; +#[cfg(feature = "rustc")] +use ui_test::{ + default_file_filter, default_per_file_config, run_tests_generic, status_emitter, Config, +}; + +#[cfg(feature = "rustc")] +#[cfg_attr(test, test)] +fn main() -> ui_test::color_eyre::Result<()> { + let config = Config::rustc("examples_tests/rustc_basic"); + let abort_check = config.abort_check.clone(); + ctrlc::set_handler(move || abort_check.store(true, Ordering::Relaxed))?; + + // Compile all `.rs` files in the given directory (relative to your + // Cargo.toml) and compare their output against the corresponding + // `.stderr` files. + run_tests_generic( + vec![config.clone(), config], + default_file_filter, + default_per_file_config, + status_emitter::Text::verbose(), + ) +} + +#[cfg(not(feature = "rustc"))] +fn main() -> ui_test::color_eyre::Result<()> { + Ok(()) +} diff --git a/examples_tests/rustc_basic/aux_derive.stderr b/examples_tests/rustc_basic/aux_derive.stderr index c4f915e6..3e5cb446 100644 --- a/examples_tests/rustc_basic/aux_derive.stderr +++ b/examples_tests/rustc_basic/aux_derive.stderr @@ -20,12 +20,14 @@ error[E0384]: cannot assign twice to immutable variable `x` --> examples_tests/rustc_basic/aux_derive.rs:8:5 | 7 | let x = Foo; - | - - | | - | first assignment to `x` - | help: consider making this binding mutable: `mut x` + | - first assignment to `x` 8 | x = Foo; | ^^^^^^^ cannot assign twice to immutable variable + | +help: consider making this binding mutable + | +7 | let mut x = Foo; + | +++ error: aborting due to 1 previous error; 2 warnings emitted diff --git a/examples_tests/rustc_basic/executable.rs b/examples_tests/rustc_basic/executable.rs index 51a603dc..31ee59c7 100644 --- a/examples_tests/rustc_basic/executable.rs +++ b/examples_tests/rustc_basic/executable.rs @@ -1,4 +1,5 @@ //@run +//@revisions: a b c d e f g h i j k l m n fn main() { std::thread::sleep(std::time::Duration::from_secs(5)); diff --git a/examples_tests/rustc_basic/filtered_out.rs b/examples_tests/rustc_basic/filtered_out.rs index 5e1e6f50..e883f04f 100644 --- a/examples_tests/rustc_basic/filtered_out.rs +++ b/examples_tests/rustc_basic/filtered_out.rs @@ -1,4 +1,4 @@ -//@[foo] only-target-asdfasdf +//@[foo] only-target: asdfasdf //@ revisions: foo bar diff --git a/src/aux_builds.rs b/src/aux_builds.rs index 67a7221a..c595ada1 100644 --- a/src/aux_builds.rs +++ b/src/aux_builds.rs @@ -1,10 +1,6 @@ //! Everything needed to build auxilary files with rustc // lol we can't name this file `aux.rs` on windows -use bstr::ByteSlice; -use spanned::Spanned; -use std::{ffi::OsString, path::PathBuf, process::Command, sync::Arc}; - use crate::{ build_manager::{Build, BuildManager}, custom_flags::Flag, @@ -13,6 +9,9 @@ use crate::{ status_emitter::SilentStatus, CrateType, Error, Errored, }; +use bstr::ByteSlice; +use spanned::Spanned; +use std::{ffi::OsString, path::PathBuf, process::Command, sync::Arc}; impl Flag for AuxBuilder { fn must_be_unique(&self) -> bool { diff --git a/src/build_manager.rs b/src/build_manager.rs index ff0a41ec..945ed7a7 100644 --- a/src/build_manager.rs +++ b/src/build_manager.rs @@ -1,19 +1,17 @@ //! Auxiliary and dependency builder. Extendable to custom builds. -use std::{ - collections::{hash_map::Entry, HashMap}, - ffi::OsString, - sync::{Arc, OnceLock, RwLock}, -}; - -use color_eyre::eyre::Result; -use crossbeam_channel::{bounded, Sender}; - use crate::{ status_emitter::{RevisionStyle, TestStatus}, test_result::TestRun, Config, Errored, }; +use color_eyre::eyre::Result; +use crossbeam_channel::{bounded, Sender}; +use std::{ + collections::{hash_map::Entry, HashMap}, + ffi::OsString, + sync::{Arc, OnceLock, RwLock}, +}; /// A build shared between all tests of the same `BuildManager` pub trait Build { diff --git a/src/cmd.rs b/src/cmd.rs index da658249..5fe6ac23 100644 --- a/src/cmd.rs +++ b/src/cmd.rs @@ -1,11 +1,10 @@ +use crate::display; use std::{ ffi::OsString, path::{Path, PathBuf}, process::Command, }; -use crate::display; - #[derive(Debug, Clone)] /// A command, its args and its environment. Used for /// the main command, the dependency builder and the cfg-reader. diff --git a/src/config.rs b/src/config.rs index 6d489d07..b319843e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,6 +1,3 @@ -use regex::bytes::Regex; -use spanned::Spanned; - #[cfg(feature = "rustc")] use crate::{ aux_builds::AuxBuilder, custom_flags::run::Run, custom_flags::rustfix::RustfixMode, @@ -10,10 +7,12 @@ use crate::{ diagnostics::Diagnostics, parser::CommandParserFunc, per_test_config::{Comments, Condition}, - CommandBuilder, + CommandBuilder, Error, Errors, }; pub use color_eyre; use color_eyre::eyre::Result; +use regex::bytes::Regex; +use spanned::Spanned; use std::{ collections::BTreeMap, num::NonZeroUsize, @@ -40,7 +39,7 @@ pub struct Config { /// The recommended command to bless failing tests. pub bless_command: Option, /// Where to dump files like the binaries compiled from tests. - /// Defaults to `target/ui` in the current directory. + /// Defaults to `target/ui/index_of_config` in the current directory. pub out_dir: PathBuf, /// Skip test files whose names contain any of these entries. pub skip_files: Vec, @@ -68,6 +67,9 @@ pub struct Config { pub abort_check: Arc, } +/// Function that performs the actual output conflict handling. +pub type OutputConflictHandling = fn(&Path, Vec, &mut Errors, &Config); + impl Config { /// Create a blank configuration that doesn't do anything interesting pub fn dummy() -> Self { @@ -78,7 +80,7 @@ impl Config { target: Default::default(), root_dir: Default::default(), program: CommandBuilder::cmd(""), - output_conflict_handling: OutputConflictHandling::Error, + output_conflict_handling: error_on_output_conflict, bless_command: Default::default(), out_dir: Default::default(), skip_files: Default::default(), @@ -117,7 +119,12 @@ impl Config { fn clone_inner(&self) -> Box { Box::new(NeedsAsmSupport) } - fn test_condition(&self, config: &Config) -> bool { + fn test_condition( + &self, + config: &Config, + _comments: &Comments, + _revision: &str, + ) -> bool { let target = config.target.as_ref().unwrap(); static ASM_SUPPORTED_ARCHS: &[&str] = &[ "x86", "x86_64", "arm", "aarch64", "riscv32", @@ -154,7 +161,7 @@ impl Config { target: None, root_dir: root_dir.into(), program: CommandBuilder::rustc(), - output_conflict_handling: OutputConflictHandling::Error, + output_conflict_handling: error_on_output_conflict, bless_command: None, out_dir: std::env::var_os("CARGO_TARGET_DIR") .map(PathBuf::from) @@ -194,7 +201,14 @@ impl Config { config.custom_comments.insert("run", |parser, args, span| { let set = |exit_code| { - parser.set_custom_once("run", Run { exit_code }, args.span()); + parser.set_custom_once( + "run", + Run { + exit_code, + output_conflict_handling: None, + }, + args.span(), + ); parser.exit_status = Spanned::new(0, span.clone()).into(); parser.require_annotations = Spanned::new(false, span.clone()).into(); @@ -268,9 +282,9 @@ impl Config { self.list = list; if check { - self.output_conflict_handling = OutputConflictHandling::Error; + self.output_conflict_handling = error_on_output_conflict; } else if bless { - self.output_conflict_handling = OutputConflictHandling::Bless; + self.output_conflict_handling = bless_output_files; } } @@ -411,9 +425,12 @@ impl Config { return self.run_only_ignored; } if comments.for_revision(revision).any(|r| { - r.custom - .values() - .any(|flags| flags.content.iter().any(|flag| flag.test_condition(self))) + r.custom.values().any(|flags| { + flags + .content + .iter() + .any(|flag| flag.test_condition(self, comments, revision)) + }) }) { return self.run_only_ignored; } @@ -429,16 +446,41 @@ impl Config { } } -#[derive(Debug, Clone)] -/// The different options for what to do when stdout/stderr files differ from the actual output. -pub enum OutputConflictHandling { - /// Fail the test when mismatches are found, if provided the command string - /// in [`Config::bless_command`] will be suggested as a way to bless the - /// test. - Error, - /// Ignore mismatches in the stderr/stdout files. - Ignore, - /// Instead of erroring if the stderr/stdout differs from the expected - /// automatically replace it with the found output (after applying filters). - Bless, +/// Fail the test when mismatches are found, if provided the command string +/// in [`Config::bless_command`] will be suggested as a way to bless the +/// test. +pub fn error_on_output_conflict( + path: &Path, + actual: Vec, + errors: &mut Errors, + config: &Config, +) { + let expected = std::fs::read(path).unwrap_or_default(); + if actual != expected { + errors.push(Error::OutputDiffers { + path: path.to_path_buf(), + actual, + expected, + bless_command: config.bless_command.clone(), + }); + } +} + +/// Ignore mismatches in the stderr/stdout files. +pub fn ignore_output_conflict( + _path: &Path, + _actual: Vec, + _errors: &mut Errors, + _config: &Config, +) { +} + +/// Instead of erroring if the stderr/stdout differs from the expected +/// automatically replace it with the found output (after applying filters). +pub fn bless_output_files(path: &Path, actual: Vec, _errors: &mut Errors, _config: &Config) { + if actual.is_empty() { + let _ = std::fs::remove_file(path); + } else { + std::fs::write(path, &actual).unwrap(); + } } diff --git a/src/config/args.rs b/src/config/args.rs index 101c77d0..89da6d7d 100644 --- a/src/config/args.rs +++ b/src/config/args.rs @@ -1,9 +1,8 @@ //! Default argument processing when `ui_test` is used //! as a test driver. -use std::{borrow::Cow, num::NonZeroUsize}; - use color_eyre::eyre::{bail, ensure, Result}; +use std::{borrow::Cow, num::NonZeroUsize}; /// Plain arguments if `ui_test` is used as a binary. #[derive(Debug, Default)] diff --git a/src/custom_flags.rs b/src/custom_flags.rs index 8b408dff..3b97bc98 100644 --- a/src/custom_flags.rs +++ b/src/custom_flags.rs @@ -1,12 +1,13 @@ //! Define custom test flags not natively supported by ui_test +use crate::{ + build_manager::BuildManager, parser::Comments, per_test_config::TestConfig, Config, Errored, +}; use std::{ panic::{RefUnwindSafe, UnwindSafe}, process::{Command, Output}, }; -use crate::{build_manager::BuildManager, per_test_config::TestConfig, Config, Errored}; - #[cfg(feature = "rustc")] pub mod edition; #[cfg(feature = "rustc")] @@ -29,7 +30,7 @@ pub trait Flag: Send + Sync + UnwindSafe + RefUnwindSafe + std::fmt::Debug { } /// Whether this flag causes a test to be filtered out - fn test_condition(&self, _config: &Config) -> bool { + fn test_condition(&self, _config: &Config, _comments: &Comments, _revision: &str) -> bool { false } diff --git a/src/custom_flags/edition.rs b/src/custom_flags/edition.rs index 147b1ff0..aa516adc 100644 --- a/src/custom_flags/edition.rs +++ b/src/custom_flags/edition.rs @@ -1,28 +1,27 @@ -//! Custom flag for setting the edition for all tests - -use crate::{build_manager::BuildManager, per_test_config::TestConfig, Errored}; - -use super::Flag; - -#[derive(Debug)] -/// Set the edition of the tests -pub struct Edition(pub String); - -impl Flag for Edition { - fn must_be_unique(&self) -> bool { - true - } - fn clone_inner(&self) -> Box { - Box::new(Edition(self.0.clone())) - } - - fn apply( - &self, - cmd: &mut std::process::Command, - _config: &TestConfig, - _build_manager: &BuildManager, - ) -> Result<(), Errored> { - cmd.arg("--edition").arg(&self.0); - Ok(()) - } -} +//! Custom flag for setting the edition for all tests + +use super::Flag; +use crate::{build_manager::BuildManager, per_test_config::TestConfig, Errored}; + +#[derive(Debug)] +/// Set the edition of the tests +pub struct Edition(pub String); + +impl Flag for Edition { + fn must_be_unique(&self) -> bool { + true + } + fn clone_inner(&self) -> Box { + Box::new(Edition(self.0.clone())) + } + + fn apply( + &self, + cmd: &mut std::process::Command, + _config: &TestConfig, + _build_manager: &BuildManager, + ) -> Result<(), Errored> { + cmd.arg("--edition").arg(&self.0); + Ok(()) + } +} diff --git a/src/custom_flags/run.rs b/src/custom_flags/run.rs index fbdc41d6..57a9acc7 100644 --- a/src/custom_flags/run.rs +++ b/src/custom_flags/run.rs @@ -1,19 +1,22 @@ //! Types used for running tests after they pass compilation -use bstr::ByteSlice; -use spanned::Spanned; -use std::process::{Command, Output}; - +use super::Flag; use crate::{ build_manager::BuildManager, display, per_test_config::TestConfig, - status_emitter::RevisionStyle, Error, Errored, TestOk, TestRun, + status_emitter::RevisionStyle, CommandBuilder, Error, Errored, OutputConflictHandling, TestOk, + TestRun, }; - -use super::Flag; +use bstr::ByteSlice; +use spanned::Spanned; +use std::{path::Path, process::Output}; #[derive(Debug, Copy, Clone)] -pub(crate) struct Run { +/// Run a test after successfully compiling it +pub struct Run { + /// The exit code that the test is expected to emit. pub exit_code: i32, + /// How to handle output conflicts + pub output_conflict_handling: Option, } impl Flag for Run { @@ -33,12 +36,15 @@ impl Flag for Run { let mut cmd = config.build_command(build_manager)?; let exit_code = self.exit_code; let revision = config.extension("run"); - let config = TestConfig { + let mut config = TestConfig { config: config.config.clone(), comments: config.comments.clone(), aux_dir: config.aux_dir.clone(), status: config.status.for_revision(&revision, RevisionStyle::Show), }; + if let Some(och) = self.output_conflict_handling { + config.config.output_conflict_handling = och; + } build_manager.add_new_job(move || { cmd.arg("--print").arg("file-names"); let output = cmd.output().unwrap(); @@ -55,8 +61,12 @@ impl Flag for Run { let file = files.next().unwrap(); assert_eq!(files.next(), None); let file = std::str::from_utf8(file).unwrap(); - let exe_file = config.config.out_dir.join(file); - let mut exe = Command::new(&exe_file); + let mut envs = std::mem::take(&mut config.config.program.envs); + config.config.program = CommandBuilder::cmd(config.config.out_dir.join(file)); + envs.extend(config.envs().map(|(k, v)| (k.into(), Some(v.into())))); + config.config.program.envs = envs; + + let mut exe = config.config.program.build(Path::new("")); let stdin = config .status .path() @@ -64,9 +74,12 @@ impl Flag for Run { if stdin.exists() { exe.stdin(std::fs::File::open(stdin).unwrap()); } - let output = exe - .output() - .unwrap_or_else(|err| panic!("exe file: {}: {err}", display(&exe_file))); + let output = exe.output().unwrap_or_else(|err| { + panic!( + "exe file: {}: {err}", + display(&config.config.program.program) + ) + }); if config.aborted() { return TestRun { diff --git a/src/custom_flags/rustfix.rs b/src/custom_flags/rustfix.rs index d03a462b..2021c304 100644 --- a/src/custom_flags/rustfix.rs +++ b/src/custom_flags/rustfix.rs @@ -1,15 +1,6 @@ //! All the logic needed to run rustfix on a test that failed compilation -use std::{ - collections::HashSet, - path::{Path, PathBuf}, - process::Output, - sync::Arc, -}; - -use rustfix::{CodeFix, Filter, Suggestion}; -use spanned::{Span, Spanned}; - +use super::Flag; use crate::{ build_manager::BuildManager, display, @@ -17,8 +8,14 @@ use crate::{ per_test_config::{Comments, Revisioned, TestConfig}, Error, Errored, TestOk, TestRun, }; - -use super::Flag; +use rustfix::{CodeFix, Filter, Suggestion}; +use spanned::{Span, Spanned}; +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + process::Output, + sync::Arc, +}; /// When to run rustfix on tests #[derive(Copy, Clone, Debug, PartialEq, Eq)] diff --git a/src/dependencies.rs b/src/dependencies.rs index 7417143a..72e392bf 100644 --- a/src/dependencies.rs +++ b/src/dependencies.rs @@ -1,5 +1,12 @@ //! Use `cargo` to build dependencies and make them available in your tests +use crate::{ + build_manager::{Build, BuildManager}, + custom_flags::Flag, + per_test_config::TestConfig, + test_result::Errored, + CommandBuilder, Config, +}; use bstr::ByteSlice; use cargo_metadata::{camino::Utf8PathBuf, BuildScript, DependencyKind}; use cargo_platform::Cfg; @@ -11,14 +18,6 @@ use std::{ str::FromStr, }; -use crate::{ - build_manager::{Build, BuildManager}, - custom_flags::Flag, - per_test_config::TestConfig, - test_result::Errored, - CommandBuilder, Config, OutputConflictHandling, -}; - #[derive(Default, Debug)] /// Describes where to find the binaries built for the dependencies pub struct Dependencies { @@ -100,12 +99,11 @@ fn build_dependencies_inner( // Reusable closure for setting up the environment both for artifact generation and `cargo_metadata` let set_locking = |cmd: &mut Command| { - if let OutputConflictHandling::Error = config.output_conflict_handling { + if !info.bless_lockfile { cmd.arg("--locked"); } }; - set_locking(&mut build); build.arg("--message-format=json"); let output = match build.output() { @@ -365,6 +363,8 @@ pub struct DependencyBuilder { /// Build with [`build-std`](https://doc.rust-lang.org/1.78.0/cargo/reference/unstable.html#build-std), /// which requires the nightly toolchain. The [`String`] can contain the standard library crates to build. pub build_std: Option, + /// Whether the lockfile can be overwritten + pub bless_lockfile: bool, } impl Default for DependencyBuilder { @@ -373,6 +373,7 @@ impl Default for DependencyBuilder { crate_manifest_path: PathBuf::from("Cargo.toml"), program: CommandBuilder::cargo(), build_std: None, + bless_lockfile: false, } } } diff --git a/src/filter.rs b/src/filter.rs index f9e8c68f..6e349881 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -1,13 +1,12 @@ //! Datastructures and operations used for normalizing test output. +use crate::display; use bstr::ByteSlice; use regex::bytes::{Captures, Regex}; use std::borrow::Cow; use std::path::Path; use std::sync::OnceLock; -use crate::display; - /// A filter's match rule. #[derive(Clone, Debug)] pub enum Match { diff --git a/src/lib.rs b/src/lib.rs index 39e94ce4..e96b82e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,7 @@ #![deny(missing_docs)] #![doc = include_str!("../README.md")] +use crate::parser::Comments; use build_manager::BuildManager; use build_manager::NewJob; pub use color_eyre; @@ -22,8 +23,10 @@ pub use filter::Match; use per_test_config::TestConfig; use spanned::Spanned; use status_emitter::RevisionStyle; +use status_emitter::SilentStatus; use status_emitter::{StatusEmitter, TestStatus}; use std::collections::VecDeque; +use std::panic::AssertUnwindSafe; use std::path::Path; #[cfg(feature = "rustc")] use std::process::Command; @@ -32,8 +35,6 @@ use std::sync::Arc; use test_result::TestRun; pub use test_result::{Errored, TestOk}; -use crate::parser::Comments; - pub mod aux_builds; pub mod build_manager; mod cmd; @@ -46,6 +47,7 @@ pub mod diagnostics; mod diff; mod error; pub mod filter; +#[cfg(feature = "gha")] pub mod github_actions; mod mode; pub mod nextest; @@ -62,7 +64,6 @@ mod tests; pub use cmd::*; pub use config::*; pub use error::*; - pub use spanned; /// Run all tests as described in the config argument. @@ -76,6 +77,7 @@ pub fn run_tests(mut config: Config) -> Result<()> { ); } + #[cfg(feature = "gha")] let name = display(&config.root_dir); let text = match args.format { @@ -88,7 +90,11 @@ pub fn run_tests(mut config: Config) -> Result<()> { vec![config], default_file_filter, default_per_file_config, - (text, status_emitter::Gha:: { name }), + ( + text, + #[cfg(feature = "gha")] + status_emitter::Gha:: { name }, + ), ) } @@ -180,8 +186,9 @@ pub fn run_tests_generic( return Ok(()); } - for config in &mut configs { + for (i, config) in configs.iter_mut().enumerate() { config.fill_host_and_target()?; + config.out_dir.push(i.to_string()) } let mut results = vec![]; @@ -230,25 +237,26 @@ pub fn run_tests_generic( } } else if let Some(matched) = file_filter(&path, build_manager.config()) { if matched { - let status = status_emitter.register_test(path); + let status = status_emitter.register_test(path.clone()); // Forward .rs files to the test workers. submit .send(Box::new(move |finished_files_sender: &Sender| { - let path = status.path(); - let file_contents = Spanned::read_from_file(path).unwrap(); + let file_contents = Spanned::read_from_file(&path).unwrap(); let mut config = build_manager.config().clone(); let abort_check = config.abort_check.clone(); per_file_config(&mut config, &file_contents); + let status = AssertUnwindSafe(status); let result = match std::panic::catch_unwind(|| { + let status = status; parse_and_test_file( build_manager, - &status, + status.0, config, file_contents, ) }) { Ok(Ok(res)) => res, - Ok(Err(err)) => { + Ok(Err((status, err))) => { finished_files_sender.send(TestRun { result: Err(err), status, @@ -271,7 +279,10 @@ pub fn run_tests_generic( stderr: vec![], stdout: vec![], }), - status, + status: Box::new(SilentStatus { + revision: String::new(), + path, + }), abort_check, })?; return Ok(()); @@ -360,44 +371,65 @@ pub fn run_tests_generic( fn parse_and_test_file( build_manager: Arc, - status: &dyn TestStatus, + status: Box, config: Config, file_contents: Spanned>, -) -> Result, Errored> { - let comments = Comments::parse(file_contents.as_ref(), &config) - .map_err(|errors| Errored::new(errors, "parse comments"))?; +) -> Result, (Box, Errored)> { + let comments = match Comments::parse(file_contents.as_ref(), &config) { + Ok(t) => t, + Err(errors) => return Err((status, Errored::new(errors, "parse comments"))), + }; let comments = Arc::new(comments); - const EMPTY: &[String] = &[String::new()]; // Run the test for all revisions - let revisions = comments.revisions.as_deref().unwrap_or(EMPTY); let mut runs = vec![]; - for revision in revisions { - let status = status.for_revision(revision, RevisionStyle::Show); - // Ignore file if only/ignore rules do (not) apply - if !config.test_file_conditions(&comments, revision) { - runs.push(TestRun { - result: Ok(TestOk::Ignored), - status, - abort_check: config.abort_check.clone(), - }); - continue; + for i in 0.. { + match comments.revisions.as_deref() { + Some(revisions) => { + let Some(revision) = revisions.get(i) else { + status.done(&Ok(TestOk::Ok), build_manager.aborted()); + break; + }; + let status = status.for_revision(revision, RevisionStyle::Show); + test_file(&config, &comments, status, &mut runs, &build_manager) + } + None => { + test_file(&config, &comments, status, &mut runs, &build_manager); + break; + } } + } + Ok(runs) +} - let mut test_config = TestConfig { - config: config.clone(), - comments: comments.clone(), - aux_dir: status.path().parent().unwrap().join("auxiliary"), - status, - }; - - let result = test_config.run_test(&build_manager); +fn test_file( + config: &Config, + comments: &Arc, + status: Box, + runs: &mut Vec, + build_manager: &Arc, +) { + if !config.test_file_conditions(comments, status.revision()) { runs.push(TestRun { - result, - status: test_config.status, - abort_check: test_config.config.abort_check, - }) + result: Ok(TestOk::Ignored), + status, + abort_check: config.abort_check.clone(), + }); + return; } - Ok(runs) + let mut test_config = TestConfig { + config: config.clone(), + comments: comments.clone(), + aux_dir: status.path().parent().unwrap().join("auxiliary"), + status, + }; + let result = test_config.run_test(build_manager); + // Ignore file if only/ignore rules do (not) apply + + runs.push(TestRun { + result, + status: test_config.status, + abort_check: test_config.config.abort_check, + }); } fn display(path: &Path) -> String { diff --git a/src/mode.rs b/src/mode.rs index 782e5f5e..762d98be 100644 --- a/src/mode.rs +++ b/src/mode.rs @@ -1,8 +1,6 @@ -use spanned::Spanned; - -use crate::{per_test_config::TestConfig, Errored}; - use super::Error; +use crate::{per_test_config::TestConfig, Errored}; +use spanned::Spanned; use std::process::ExitStatus; impl TestConfig { diff --git a/src/parser.rs b/src/parser.rs index e0f755f8..b3d4afd8 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,18 +1,14 @@ -use std::{ - collections::{BTreeMap, HashMap}, - num::NonZeroUsize, -}; - -use bstr::{ByteSlice, Utf8Error}; -use regex::bytes::Regex; - use crate::{ custom_flags::Flag, diagnostics::Level, filter::Match, test_result::Errored, Config, Error, }; - +use bstr::{ByteSlice, Utf8Error}; use color_eyre::eyre::Result; - +use regex::bytes::Regex; pub(crate) use spanned::*; +use std::{ + collections::{BTreeMap, HashMap}, + num::NonZeroUsize, +}; mod spanned; #[cfg(test)] diff --git a/src/parser/tests.rs b/src/parser/tests.rs index 51c30f73..e7112c2f 100644 --- a/src/parser/tests.rs +++ b/src/parser/tests.rs @@ -1,13 +1,10 @@ -use std::path::PathBuf; - -use spanned::{Span, Spanned}; - +use super::Comments; use crate::{ parser::{Condition, ErrorMatchKind, Pattern}, Config, Error, }; - -use super::Comments; +use spanned::{Span, Spanned}; +use std::path::PathBuf; macro_rules! line { ($thing:expr, $s:expr) => {{ diff --git a/src/per_test_config.rs b/src/per_test_config.rs index 1a9b47d6..b262b33c 100644 --- a/src/per_test_config.rs +++ b/src/per_test_config.rs @@ -3,15 +3,6 @@ //! in the files. These comments still overwrite the defaults, although //! some boolean settings have no way to disable them. -use std::collections::btree_map::Entry; -use std::collections::BTreeMap; -use std::num::NonZeroUsize; -use std::path::PathBuf; -use std::process::{Command, Output}; -use std::sync::Arc; - -use spanned::Spanned; - use crate::build_manager::BuildManager; use crate::custom_flags::Flag; pub use crate::diagnostics::Level; @@ -20,7 +11,14 @@ pub use crate::parser::{Comments, Condition, Revisioned}; use crate::parser::{ErrorMatch, ErrorMatchKind, OptWithLine}; use crate::status_emitter::{SilentStatus, TestStatus}; use crate::test_result::{Errored, TestOk, TestResult}; -use crate::{core::strip_path_prefix, Config, Error, Errors, OutputConflictHandling}; +use crate::{core::strip_path_prefix, Config, Error, Errors}; +use spanned::Spanned; +use std::collections::btree_map::Entry; +use std::collections::BTreeMap; +use std::num::NonZeroUsize; +use std::path::PathBuf; +use std::process::{Command, Output}; +use std::sync::Arc; /// All information needed to run a single test pub struct TestConfig { @@ -136,6 +134,7 @@ impl TestConfig { cmd.arg(self.status.path()); if !self.status.revision().is_empty() { cmd.arg(format!("--cfg={}", self.status.revision())); + cmd.arg(format!("-Cextra-filename={}", self.status.revision())); } for r in self.comments() { cmd.args(&r.compile_flags); @@ -152,14 +151,7 @@ impl TestConfig { } } - // False positive in miri, our `map` uses a ref pattern to get the references to the tuple fields instead - // of a reference to a tuple - #[allow(clippy::map_identity)] - cmd.envs( - self.comments() - .flat_map(|r| r.env_vars.iter()) - .map(|(k, v)| (k, v)), - ); + cmd.envs(self.envs()); Ok(cmd) } @@ -199,27 +191,7 @@ impl TestConfig { pub(crate) fn check_output(&self, output: &[u8], errors: &mut Errors, kind: &str) -> PathBuf { let output = self.normalize(output, kind); let path = self.output_path(kind); - match &self.config.output_conflict_handling { - OutputConflictHandling::Error => { - let expected_output = std::fs::read(&path).unwrap_or_default(); - if output != expected_output { - errors.push(Error::OutputDiffers { - path: path.clone(), - actual: output.clone(), - expected: expected_output, - bless_command: self.config.bless_command.clone(), - }); - } - } - OutputConflictHandling::Bless => { - if output.is_empty() { - let _ = std::fs::remove_file(&path); - } else { - std::fs::write(&path, &output).unwrap(); - } - } - OutputConflictHandling::Ignore => {} - } + (self.config.output_conflict_handling)(&path, output, errors, &self.config); path } @@ -445,4 +417,11 @@ impl TestConfig { pub(crate) fn aborted(&self) -> bool { self.config.aborted() } + + /// All the environment variables set for the given revision + pub fn envs(&self) -> impl Iterator { + self.comments() + .flat_map(|r| r.env_vars.iter()) + .map(|(k, v)| (k.as_ref(), v.as_ref())) + } } diff --git a/src/rustc_stderr.rs b/src/rustc_stderr.rs index 17e1082d..34a62d28 100644 --- a/src/rustc_stderr.rs +++ b/src/rustc_stderr.rs @@ -1,10 +1,9 @@ +use crate::diagnostics::{Diagnostics, Message}; use bstr::ByteSlice; use cargo_metadata::diagnostic::{Diagnostic, DiagnosticSpan}; use regex::Regex; use std::path::{Path, PathBuf}; -use crate::diagnostics::{Diagnostics, Message}; - fn diag_line(diag: &Diagnostic, file: &Path) -> Option<(spanned::Span, usize)> { let span = |primary| { diag.spans diff --git a/src/status_emitter.rs b/src/status_emitter.rs index fd5eb8ee..ced095f8 100644 --- a/src/status_emitter.rs +++ b/src/status_emitter.rs @@ -1,36 +1,25 @@ -//! Variaous schemes for reporting messages during testing or after testing is done. +//! Various schemes for reporting messages during testing or after testing is done. -use annotate_snippets::{Renderer, Snippet}; -use bstr::ByteSlice; -use colored::Colorize; -use crossbeam_channel::{Sender, TryRecvError}; -use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle}; -use spanned::{Span, Spanned}; +use crate::{test_result::TestResult, Errors}; -use crate::{ - diagnostics::{Level, Message}, - display, github_actions, - parser::Pattern, - test_result::{Errored, TestOk, TestResult}, - Error, Errors, Format, -}; use std::{ - collections::HashMap, - fmt::{Debug, Display, Write as _}, - io::Write as _, - num::NonZeroUsize, + fmt::Debug, panic::RefUnwindSafe, path::{Path, PathBuf}, - sync::{Arc, Mutex}, - thread::JoinHandle, - time::Duration, }; +pub use text::*; +pub mod debug; +mod text; +#[cfg(feature = "gha")] +pub use gha::*; +#[cfg(feature = "gha")] +mod gha; /// A generic way to handle the output of this crate. pub trait StatusEmitter: Sync + RefUnwindSafe { /// Invoked the moment we know a test will later be run. /// Useful for progress bars and such. - fn register_test(&self, path: PathBuf) -> Box; + fn register_test(&self, path: PathBuf) -> Box; /// Create a report about the entire test run at the end. #[allow(clippy::type_complexity)] @@ -152,1014 +141,6 @@ impl TestStatus for SilentStatus { } } -#[derive(Clone, Copy)] -enum OutputVerbosity { - Progress, - DiffOnly, - Full, -} - -/// A human readable output emitter. -#[derive(Clone)] -pub struct Text { - sender: Sender, - progress: OutputVerbosity, - handle: Arc, -} - -struct JoinOnDrop(Mutex>>); -impl From> for JoinOnDrop { - fn from(handle: JoinHandle<()>) -> Self { - Self(Mutex::new(Some(handle))) - } -} -impl Drop for JoinOnDrop { - fn drop(&mut self) { - self.join(); - } -} - -impl JoinOnDrop { - fn join(&self) { - let Ok(Some(handle)) = self.0.try_lock().map(|mut g| g.take()) else { - return; - }; - let _ = handle.join(); - } -} - -#[derive(Debug)] -enum Msg { - Pop { - msg: String, - new_leftover_msg: String, - parent: String, - }, - Push { - parent: String, - msg: String, - }, - Finish, - Abort, -} - -impl Text { - fn start_thread(progress: OutputVerbosity) -> Self { - let (sender, receiver) = crossbeam_channel::unbounded(); - let handle = std::thread::spawn(move || { - let bars = MultiProgress::new(); - let progress = match progress { - OutputVerbosity::Progress => bars.add(ProgressBar::new(0)), - OutputVerbosity::DiffOnly | OutputVerbosity::Full => { - ProgressBar::with_draw_target(Some(0), ProgressDrawTarget::hidden()) - } - }; - - struct ProgressHandler { - // The bools signal whether the progress bar is done (used for sanity assertions only) - threads: HashMap>, - aborted: bool, - bars: MultiProgress, - progress: ProgressBar, - } - - impl ProgressHandler { - fn pop(&mut self, msg: String, new_leftover_msg: String, parent: String) { - let Some(children) = self.threads.get_mut(&parent) else { - // This can happen when a test was not run at all, because it failed directly during - // comment parsing. - return; - }; - self.progress.inc(1); - let Some((spinner, done)) = children.get_mut(&msg) else { - panic!("pop: {parent}({msg}): {children:#?}") - }; - *done = true; - let spinner = spinner.clone(); - spinner.finish_with_message(new_leftover_msg); - let parent = children[""].0.clone(); - if children.values().all(|&(_, done)| done) { - self.bars.remove(&parent); - if self.progress.is_hidden() { - self.bars - .println(format!("{} {}", parent.prefix(), parent.message())) - .unwrap(); - } - for (msg, (child, _)) in children.iter() { - if !msg.is_empty() { - self.bars.remove(child); - if self.progress.is_hidden() { - self.bars - .println(format!( - " {} {}", - child.prefix(), - child.message() - )) - .unwrap(); - } - } - } - } - } - - fn push(&mut self, parent: String, msg: String) { - self.progress.inc_length(1); - let children = self.threads.entry(parent.clone()).or_default(); - if !msg.is_empty() { - let parent = &children - .entry(String::new()) - .or_insert_with(|| { - let spinner = self - .bars - .add(ProgressBar::new_spinner().with_prefix(parent)); - spinner.set_style( - ProgressStyle::with_template("{prefix} {msg}").unwrap(), - ); - (spinner, true) - }) - .0; - let spinner = self.bars.insert_after( - parent, - ProgressBar::new_spinner().with_prefix(msg.clone()), - ); - spinner.set_style( - ProgressStyle::with_template(" {prefix} {spinner} {msg}").unwrap(), - ); - children.insert(msg, (spinner, false)); - } else { - let spinner = self - .bars - .add(ProgressBar::new_spinner().with_prefix(parent)); - spinner.set_style( - ProgressStyle::with_template("{prefix} {spinner} {msg}").unwrap(), - ); - children.insert(msg, (spinner, false)); - }; - } - - fn tick(&self) { - for children in self.threads.values() { - for (spinner, done) in children.values() { - if !done { - spinner.tick(); - } - } - } - } - } - - impl Drop for ProgressHandler { - fn drop(&mut self) { - for (key, children) in self.threads.iter() { - for (sub_key, (_child, done)) in children { - assert!(done, "{key} ({sub_key}) not finished"); - } - } - if self.aborted { - self.progress.abandon(); - } else { - assert_eq!( - Some(self.progress.position()), - self.progress.length(), - "{:#?}", - self.threads - ); - self.progress.finish(); - } - } - } - - let mut handler = ProgressHandler { - threads: Default::default(), - aborted: false, - bars, - progress, - }; - - 'outer: loop { - std::thread::sleep(Duration::from_millis(100)); - loop { - match receiver.try_recv() { - Ok(val) => match val { - Msg::Pop { - msg, - new_leftover_msg, - parent, - } => { - handler.pop(msg, new_leftover_msg, parent); - } - - Msg::Push { parent, msg } => { - handler.push(parent, msg); - } - Msg::Finish => break 'outer, - Msg::Abort => handler.aborted = true, - }, - // Sender panicked, skip asserts - Err(TryRecvError::Disconnected) => return, - Err(TryRecvError::Empty) => break, - } - } - handler.tick() - } - }); - Self { - sender, - progress, - handle: Arc::new(handle.into()), - } - } - - /// Print one line per test that gets run. - pub fn verbose() -> Self { - Self::start_thread(OutputVerbosity::Full) - } - /// Print one line per test that gets run. - pub fn diff() -> Self { - Self::start_thread(OutputVerbosity::DiffOnly) - } - /// Print a progress bar. - pub fn quiet() -> Self { - Self::start_thread(OutputVerbosity::Progress) - } - - fn is_full_output(&self) -> bool { - matches!(self.progress, OutputVerbosity::Full) - } -} - -impl From for Text { - fn from(format: Format) -> Self { - match format { - Format::Terse => Text::quiet(), - Format::Pretty => Text::verbose(), - } - } -} - -struct TextTest { - text: Text, - parent: String, - path: PathBuf, - revision: String, - style: RevisionStyle, -} - -impl TestStatus for TextTest { - fn done(&self, result: &TestResult, aborted: bool) { - if aborted { - self.text.sender.send(Msg::Abort).unwrap(); - } - let result = match result { - _ if aborted => "aborted".white(), - Ok(TestOk::Ok) => "ok".green(), - Err(Errored { .. }) => "FAILED".bright_red().bold(), - Ok(TestOk::Ignored) => "ignored (in-test comment)".yellow(), - }; - let new_leftover_msg = format!("... {result}"); - if ProgressDrawTarget::stdout().is_hidden() { - match self.style { - RevisionStyle::Separate => println!("{} {new_leftover_msg}", self.revision), - RevisionStyle::Show => { - let revision = if self.revision.is_empty() { - String::new() - } else { - format!(" (revision `{}`)", self.revision) - }; - println!("{}{revision} {new_leftover_msg}", display(&self.path)); - } - } - std::io::stdout().flush().unwrap(); - } - self.text - .sender - .send(Msg::Pop { - msg: if self.revision.is_empty() && display(&self.path) != self.parent { - display(&self.path) - } else { - self.revision.clone() - }, - new_leftover_msg, - parent: self.parent.clone(), - }) - .unwrap(); - } - - fn failed_test<'a>( - &self, - cmd: &str, - stderr: &'a [u8], - stdout: &'a [u8], - ) -> Box { - let maybe_revision = if self.revision.is_empty() { - String::new() - } else { - format!(" (revision `{}`)", self.revision) - }; - let text = format!( - "{} {}{}", - "FAILED TEST:".bright_red(), - display(&self.path), - maybe_revision - ); - - println!(); - println!("{}", text.bold().underline()); - println!("command: {cmd}"); - println!(); - - if self.text.is_full_output() { - #[derive(Debug)] - struct Guard<'a> { - stderr: &'a [u8], - stdout: &'a [u8], - } - impl<'a> Drop for Guard<'a> { - fn drop(&mut self) { - println!("{}", "full stderr:".bold()); - std::io::stdout().write_all(self.stderr).unwrap(); - println!(); - println!("{}", "full stdout:".bold()); - std::io::stdout().write_all(self.stdout).unwrap(); - println!(); - println!(); - } - } - Box::new(Guard { stderr, stdout }) - } else { - Box::new(()) - } - } - - fn path(&self) -> &Path { - &self.path - } - - fn for_revision(&self, revision: &str, style: RevisionStyle) -> Box { - let text = Self { - text: self.text.clone(), - path: self.path.clone(), - parent: self.parent.clone(), - revision: revision.to_owned(), - style, - }; - self.text - .sender - .send(Msg::Push { - parent: self.parent.clone(), - msg: if revision.is_empty() && display(&self.path) != self.parent { - display(&self.path) - } else { - text.revision.clone() - }, - }) - .unwrap(); - - Box::new(text) - } - - fn for_path(&self, path: &Path) -> Box { - let text = Self { - text: self.text.clone(), - path: path.to_path_buf(), - parent: self.parent.clone(), - revision: String::new(), - style: RevisionStyle::Show, - }; - - self.text - .sender - .send(Msg::Push { - parent: self.parent.clone(), - msg: display(path), - }) - .unwrap(); - Box::new(text) - } - - fn revision(&self) -> &str { - &self.revision - } -} - -impl StatusEmitter for Text { - fn register_test(&self, path: PathBuf) -> Box { - Box::new(TextTest { - text: self.clone(), - parent: display(&path), - path, - revision: String::new(), - style: RevisionStyle::Show, - }) - } - - fn finalize( - &self, - _failures: usize, - succeeded: usize, - ignored: usize, - filtered: usize, - aborted: bool, - ) -> Box { - self.sender.send(Msg::Finish).unwrap(); - - self.handle.join(); - if !ProgressDrawTarget::stdout().is_hidden() { - // The progress bars do not have a trailing newline, so let's - // add it here. - println!(); - } - // Print all errors in a single thread to show reliable output - struct Summarizer { - failures: Vec, - succeeded: usize, - ignored: usize, - filtered: usize, - aborted: bool, - } - - impl Summary for Summarizer { - fn test_failure(&mut self, status: &dyn TestStatus, errors: &Errors) { - for error in errors { - print_error(error, status.path()); - } - - self.failures.push(if status.revision().is_empty() { - format!(" {}", display(status.path())) - } else { - format!( - " {} (revision {})", - display(status.path()), - status.revision() - ) - }); - } - } - - impl Drop for Summarizer { - fn drop(&mut self) { - if self.failures.is_empty() { - println!(); - if self.aborted { - print!("test result: cancelled."); - } else { - print!("test result: {}.", "ok".green()); - } - } else { - println!("{}", "FAILURES:".bright_red().underline().bold()); - for line in &self.failures { - println!("{line}"); - } - println!(); - print!("test result: {}.", "FAIL".bright_red()); - print!(" {} failed", self.failures.len().to_string().green()); - if self.succeeded > 0 || self.ignored > 0 || self.filtered > 0 { - print!(";"); - } - } - if self.succeeded > 0 { - print!(" {} passed", self.succeeded.to_string().green()); - if self.ignored > 0 || self.filtered > 0 { - print!(";"); - } - } - if self.ignored > 0 { - print!(" {} ignored", self.ignored.to_string().yellow()); - if self.filtered > 0 { - print!(";"); - } - } - if self.filtered > 0 { - print!(" {} filtered out", self.filtered.to_string().yellow()); - } - println!(); - println!(); - } - } - Box::new(Summarizer { - failures: vec![], - succeeded, - ignored, - filtered, - aborted, - }) - } -} - -fn print_error(error: &Error, path: &Path) { - /// Every error starts with a header like that, to make them all easy to find. - /// It is made to look like the headers printed for spanned errors. - fn print_error_header(msg: impl Display) { - let text = format!("{} {msg}", "error:".bright_red()); - println!("{}", text.bold()); - } - - match error { - Error::ExitStatus { - status, - expected, - reason, - } => { - // `status` prints as `exit status: N`. - create_error( - format!("test got {status}, but expected {expected}"), - &[&[(reason, reason.span.clone())]], - path, - ) - } - Error::Command { kind, status } => { - // `status` prints as `exit status: N`. - print_error_header(format_args!("{kind} failed with {status}")); - } - Error::PatternNotFound { - pattern, - expected_line, - } => { - let line = match expected_line { - Some(line) => format!("on line {line}"), - None => format!("outside the testfile"), - }; - let msg = match &**pattern { - Pattern::SubString(s) => { - format!("`{s}` not found in diagnostics {line}") - } - Pattern::Regex(r) => { - format!("`/{r}/` does not match diagnostics {line}",) - } - }; - // This will print a suitable error header. - create_error( - msg, - &[&[("expected because of this pattern", pattern.span())]], - path, - ); - } - Error::CodeNotFound { - code, - expected_line, - } => { - let line = match expected_line { - Some(line) => format!("on line {line}"), - None => format!("outside the testfile"), - }; - create_error( - format!("diagnostic code `{}` not found {line}", &**code), - &[&[("expected because of this pattern", code.span())]], - path, - ); - } - Error::NoPatternsFound => { - print_error_header("expected error patterns, but found none"); - } - Error::PatternFoundInPassTest { mode, span } => { - let annot = [("expected because of this annotation", span.clone())]; - let mut lines: Vec<&[_]> = vec![&annot]; - let annot = [("expected because of this mode change", mode.clone())]; - if !mode.is_dummy() { - lines.push(&annot) - } - // This will print a suitable error header. - create_error("error pattern found in pass test", &lines, path); - } - Error::OutputDiffers { - path: output_path, - actual, - expected, - bless_command, - } => { - print_error_header("actual output differed from expected"); - if let Some(bless_command) = bless_command { - println!( - "Execute `{}` to update `{}` to the actual output", - bless_command, - display(output_path) - ); - } - println!("{}", format!("--- {}", display(output_path)).red()); - println!( - "{}", - format!( - "+++ <{} output>", - output_path.extension().unwrap().to_str().unwrap() - ) - .green() - ); - crate::diff::print_diff(expected, actual); - } - Error::ErrorsWithoutPattern { path, msgs } => { - if let Some((path, _)) = path.as_ref() { - let msgs = msgs - .iter() - .map(|msg| { - let text = match (&msg.code, msg.level) { - (Some(code), Level::Error) => { - format!("Error[{code}]: {}", msg.message) - } - _ => format!("{:?}: {}", msg.level, msg.message), - }; - (text, msg.span.clone().unwrap_or_default()) - }) - .collect::>(); - // This will print a suitable error header. - create_error( - format!("there were {} unmatched diagnostics", msgs.len()), - &[&msgs - .iter() - .map(|(msg, lc)| (msg.as_ref(), lc.clone())) - .collect::>()], - path, - ); - } else { - print_error_header(format_args!( - "there were {} unmatched diagnostics that occurred outside the testfile and had no pattern", - msgs.len(), - )); - for Message { - level, - message, - line: _, - code: _, - span: _, - } in msgs - { - println!(" {level:?}: {message}") - } - } - } - Error::InvalidComment { msg, span } => { - // This will print a suitable error header. - create_error(msg, &[&[("", span.clone())]], path) - } - Error::MultipleRevisionsWithResults { kind, lines } => { - let title = format!("multiple {kind} found"); - // This will print a suitable error header. - create_error( - title, - &lines.iter().map(|_line| &[] as &[_]).collect::>(), - path, - ) - } - Error::Bug(msg) => { - print_error_header("a bug in `ui_test` occurred"); - println!("{msg}"); - } - Error::Aux { - path: aux_path, - errors, - } => { - create_error( - "aux build failed", - &[&[(&path.display().to_string(), aux_path.span.clone())]], - &aux_path.span.file, - ); - for error in errors { - print_error(error, aux_path); - } - } - Error::Rustfix(error) => { - print_error_header(format_args!( - "failed to apply suggestions for {} with rustfix", - display(path) - )); - println!("{error}"); - println!("Add //@no-rustfix to the test file to ignore rustfix suggestions"); - } - Error::ConfigError(msg) => println!("{msg}"), - } - println!(); -} - -#[allow(clippy::type_complexity)] -fn create_error(s: impl AsRef, lines: &[&[(&str, Span)]], file: &Path) { - let source = std::fs::read_to_string(file).unwrap(); - let file = display(file); - let mut msg = annotate_snippets::Level::Error.title(s.as_ref()); - for &label in lines { - let annotations = label - .iter() - .filter(|(_, span)| !span.is_dummy()) - .map(|(label, span)| { - annotate_snippets::Level::Error - .span(span.bytes.clone()) - .label(label) - }) - .collect::>(); - if !annotations.is_empty() { - let snippet = Snippet::source(&source) - .fold(true) - .origin(&file) - .annotations(annotations); - msg = msg.snippet(snippet); - } - let footer = label - .iter() - .filter(|(_, span)| span.is_dummy()) - .map(|(label, _)| annotate_snippets::Level::Note.title(label)); - msg = msg.footers(footer); - } - let renderer = if colored::control::SHOULD_COLORIZE.should_colorize() { - Renderer::styled() - } else { - Renderer::plain() - }; - println!("{}", renderer.render(msg)); -} - -fn gha_error(error: &Error, test_path: &str, revision: &str) { - let file = Spanned::read_from_file(test_path).unwrap(); - let line = |span: &Span| { - let line = file - .lines() - .position(|line| line.span.bytes.contains(&span.bytes.start)) - .unwrap(); - NonZeroUsize::new(line + 1).unwrap() - }; - match error { - Error::ExitStatus { - status, - expected, - reason, - } => { - let mut err = github_actions::error( - test_path, - format!("test{revision} got {status}, but expected {expected}"), - ); - err.write_str(reason).unwrap(); - } - Error::Command { kind, status } => { - github_actions::error(test_path, format!("{kind}{revision} failed with {status}")); - } - Error::PatternNotFound { pattern, .. } => { - github_actions::error(test_path, format!("Pattern not found{revision}")) - .line(line(&pattern.span)); - } - Error::CodeNotFound { code, .. } => { - github_actions::error(test_path, format!("Diagnostic code not found{revision}")) - .line(line(&code.span)); - } - Error::NoPatternsFound => { - github_actions::error( - test_path, - format!("expexted error patterns, but found none{revision}"), - ); - } - Error::PatternFoundInPassTest { .. } => { - github_actions::error( - test_path, - format!("error pattern found in pass test{revision}"), - ); - } - Error::OutputDiffers { - path: output_path, - actual, - expected, - bless_command, - } => { - if expected.is_empty() { - let mut err = github_actions::error( - test_path, - "test generated output, but there was no output file", - ); - if let Some(bless_command) = bless_command { - writeln!( - err, - "you likely need to bless the tests with `{bless_command}`" - ) - .unwrap(); - } - return; - } - - let mut line = 1; - for r in - prettydiff::diff_lines(expected.to_str().unwrap(), actual.to_str().unwrap()).diff() - { - use prettydiff::basic::DiffOp::*; - match r { - Equal(s) => { - line += s.len(); - continue; - } - Replace(l, r) => { - let mut err = github_actions::error( - display(output_path), - "actual output differs from expected", - ) - .line(NonZeroUsize::new(line + 1).unwrap()); - writeln!(err, "this line was expected to be `{}`", r[0]).unwrap(); - line += l.len(); - } - Remove(l) => { - let mut err = github_actions::error( - display(output_path), - "extraneous lines in output", - ) - .line(NonZeroUsize::new(line + 1).unwrap()); - writeln!( - err, - "remove this line and possibly later ones by blessing the test" - ) - .unwrap(); - line += l.len(); - } - Insert(r) => { - let mut err = - github_actions::error(display(output_path), "missing line in output") - .line(NonZeroUsize::new(line + 1).unwrap()); - writeln!(err, "bless the test to create a line containing `{}`", r[0]) - .unwrap(); - // Do not count these lines, they don't exist in the original file and - // would thus mess up the line number. - } - } - } - } - Error::ErrorsWithoutPattern { path, msgs } => { - if let Some((path, line)) = path.as_ref() { - let path = display(path); - let mut err = - github_actions::error(path, format!("Unmatched diagnostics{revision}")) - .line(*line); - for Message { - level, - message, - line: _, - span: _, - code: _, - } in msgs - { - writeln!(err, "{level:?}: {message}").unwrap(); - } - } else { - let mut err = github_actions::error( - test_path, - format!("Unmatched diagnostics outside the testfile{revision}"), - ); - for Message { - level, - message, - line: _, - span: _, - code: _, - } in msgs - { - writeln!(err, "{level:?}: {message}").unwrap(); - } - } - } - Error::InvalidComment { msg, span } => { - let mut err = github_actions::error(test_path, format!("Could not parse comment")) - .line(line(span)); - writeln!(err, "{msg}").unwrap(); - } - Error::MultipleRevisionsWithResults { kind, lines } => { - github_actions::error(test_path, format!("multiple {kind} found")) - .line(line(&lines[0])); - } - Error::Bug(_) => {} - Error::Aux { - path: aux_path, - errors, - } => { - github_actions::error(test_path, format!("Aux build failed")) - .line(line(&aux_path.span)); - for error in errors { - gha_error(error, &display(aux_path), "") - } - } - Error::Rustfix(error) => { - github_actions::error( - test_path, - format!("failed to apply suggestions with rustfix: {error}"), - ); - } - Error::ConfigError(msg) => { - github_actions::error(test_path, msg.clone()); - } - } -} - -/// Emits Github Actions Workspace commands to show the failures directly in the github diff view. -/// If the const generic `GROUP` boolean is `true`, also emit `::group` commands. -pub struct Gha { - /// Show a specific name for the final summary. - pub name: String, -} - -#[derive(Clone)] -struct PathAndRev { - path: PathBuf, - revision: String, -} - -impl TestStatus for PathAndRev { - fn path(&self) -> &Path { - &self.path - } - - fn for_revision(&self, revision: &str, _style: RevisionStyle) -> Box { - Box::new(Self { - path: self.path.clone(), - revision: revision.to_owned(), - }) - } - - fn for_path(&self, path: &Path) -> Box { - Box::new(Self { - path: path.to_path_buf(), - revision: self.revision.clone(), - }) - } - - fn failed_test(&self, _cmd: &str, _stderr: &[u8], _stdout: &[u8]) -> Box { - if GROUP { - Box::new(github_actions::group(format_args!( - "{}:{}", - display(&self.path), - self.revision - ))) - } else { - Box::new(()) - } - } - - fn revision(&self) -> &str { - &self.revision - } -} - -impl StatusEmitter for Gha { - fn register_test(&self, path: PathBuf) -> Box { - Box::new(PathAndRev:: { - path, - revision: String::new(), - }) - } - - fn finalize( - &self, - _failures: usize, - succeeded: usize, - ignored: usize, - filtered: usize, - // Can't aborted on gha - _aborted: bool, - ) -> Box { - struct Summarizer { - failures: Vec, - succeeded: usize, - ignored: usize, - filtered: usize, - name: String, - } - - impl Summary for Summarizer { - fn test_failure(&mut self, status: &dyn TestStatus, errors: &Errors) { - let revision = if status.revision().is_empty() { - "".to_string() - } else { - format!(" (revision: {})", status.revision()) - }; - for error in errors { - gha_error(error, &display(status.path()), &revision); - } - self.failures - .push(format!("{}{revision}", display(status.path()))); - } - } - impl Drop for Summarizer { - fn drop(&mut self) { - if let Some(mut file) = github_actions::summary() { - writeln!(file, "### {}", self.name).unwrap(); - for line in &self.failures { - writeln!(file, "* {line}").unwrap(); - } - writeln!(file).unwrap(); - writeln!(file, "| failed | passed | ignored | filtered out |").unwrap(); - writeln!(file, "| --- | --- | --- | --- |").unwrap(); - writeln!( - file, - "| {} | {} | {} | {} |", - self.failures.len(), - self.succeeded, - self.ignored, - self.filtered, - ) - .unwrap(); - } - } - } - - Box::new(Summarizer:: { - failures: vec![], - succeeded, - ignored, - filtered, - name: self.name.clone(), - }) - } -} - impl TestStatus for (T, U) { fn done(&self, result: &TestResult, aborted: bool) { self.0.done(result, aborted); @@ -1227,6 +208,26 @@ impl StatusEmitter for (T, U) { } } +/// Forwards directly to `T`, exists only so that tuples can be used with `cfg` to filter +/// out individual fields +impl StatusEmitter for (T,) { + fn register_test(&self, path: PathBuf) -> Box { + self.0.register_test(path.clone()) + } + + fn finalize( + &self, + failures: usize, + succeeded: usize, + ignored: usize, + filtered: usize, + aborted: bool, + ) -> Box { + self.0 + .finalize(failures, succeeded, ignored, filtered, aborted) + } +} + impl TestStatus for Box { fn done(&self, result: &TestResult, aborted: bool) { (**self).done(result, aborted); diff --git a/src/status_emitter/debug.rs b/src/status_emitter/debug.rs new file mode 100644 index 00000000..99787f23 --- /dev/null +++ b/src/status_emitter/debug.rs @@ -0,0 +1,95 @@ +//! A debug emitter for when things get stuck. Mostly useful for debugging of ui_test itself + +use crate::Error; +use std::path::{Path, PathBuf}; + +/// Very verbose status emitter +pub struct StatusEmitter; + +impl super::StatusEmitter for StatusEmitter { + fn register_test(&self, path: PathBuf) -> Box { + eprintln!("START {}", path.display()); + Box::new(TestStatus(path, String::new())) + } + + fn finalize( + &self, + failed: usize, + succeeded: usize, + ignored: usize, + filtered: usize, + aborted: bool, + ) -> Box { + eprintln!("{failed}, {succeeded}, {ignored}, {filtered}, {aborted}"); + Box::new(Summary) + } +} + +struct Summary; + +impl super::Summary for Summary { + fn test_failure(&mut self, status: &dyn super::TestStatus, errors: &Vec) { + eprintln!( + "FAILED: {} ({})", + status.path().display(), + status.revision() + ); + eprintln!("{errors:#?}"); + } +} + +struct TestStatus(PathBuf, String); + +impl super::TestStatus for TestStatus { + fn for_revision( + &self, + revision: &str, + _style: super::RevisionStyle, + ) -> Box { + eprintln!( + "REVISION {}: {} (old: {})", + self.0.display(), + revision, + self.1 + ); + Box::new(TestStatus(self.0.clone(), revision.to_string())) + } + + fn for_path(&self, path: &Path) -> Box { + eprintln!( + "PATH {} (old: {} ({}))", + path.display(), + self.0.display(), + self.1 + ); + Box::new(TestStatus(path.to_owned(), String::new())) + } + + fn failed_test<'a>( + &'a self, + cmd: &'a str, + stderr: &'a [u8], + stdout: &'a [u8], + ) -> Box { + eprintln!("failed {} ({})", self.0.display(), self.1); + eprintln!("{cmd}"); + eprintln!("{}", std::str::from_utf8(stderr).unwrap()); + eprintln!("{}", std::str::from_utf8(stdout).unwrap()); + eprintln!(); + Box::new(()) + } + + fn path(&self) -> &Path { + &self.0 + } + + fn revision(&self) -> &str { + &self.1 + } +} + +impl Drop for TestStatus { + fn drop(&mut self) { + eprintln!("DONE {} ({})", self.0.display(), self.1); + } +} diff --git a/src/status_emitter/gha.rs b/src/status_emitter/gha.rs new file mode 100644 index 00000000..c1db16df --- /dev/null +++ b/src/status_emitter/gha.rs @@ -0,0 +1,307 @@ +use crate::{diagnostics::Message, display, Error, Errors}; + +use crate::github_actions; +use bstr::ByteSlice; +use spanned::{Span, Spanned}; +use std::{ + fmt::{Debug, Write as _}, + io::Write as _, + num::NonZeroUsize, + path::{Path, PathBuf}, +}; + +use super::{RevisionStyle, StatusEmitter, Summary, TestStatus}; +fn gha_error(error: &Error, test_path: &str, revision: &str) { + let file = Spanned::read_from_file(test_path).unwrap(); + let line = |span: &Span| { + let line = file + .lines() + .position(|line| line.span.bytes.contains(&span.bytes.start)) + .unwrap(); + NonZeroUsize::new(line + 1).unwrap() + }; + match error { + Error::ExitStatus { + status, + expected, + reason, + } => { + let mut err = github_actions::error( + test_path, + format!("test{revision} got {status}, but expected {expected}"), + ); + err.write_str(reason).unwrap(); + } + Error::Command { kind, status } => { + github_actions::error(test_path, format!("{kind}{revision} failed with {status}")); + } + Error::PatternNotFound { pattern, .. } => { + github_actions::error(test_path, format!("Pattern not found{revision}")) + .line(line(&pattern.span)); + } + Error::CodeNotFound { code, .. } => { + github_actions::error(test_path, format!("Diagnostic code not found{revision}")) + .line(line(&code.span)); + } + Error::NoPatternsFound => { + github_actions::error( + test_path, + format!("expexted error patterns, but found none{revision}"), + ); + } + Error::PatternFoundInPassTest { .. } => { + github_actions::error( + test_path, + format!("error pattern found in pass test{revision}"), + ); + } + Error::OutputDiffers { + path: output_path, + actual, + expected, + bless_command, + } => { + if expected.is_empty() { + let mut err = github_actions::error( + test_path, + "test generated output, but there was no output file", + ); + if let Some(bless_command) = bless_command { + writeln!( + err, + "you likely need to bless the tests with `{bless_command}`" + ) + .unwrap(); + } + return; + } + + let mut line = 1; + for r in + prettydiff::diff_lines(expected.to_str().unwrap(), actual.to_str().unwrap()).diff() + { + use prettydiff::basic::DiffOp::*; + match r { + Equal(s) => { + line += s.len(); + continue; + } + Replace(l, r) => { + let mut err = github_actions::error( + display(output_path), + "actual output differs from expected", + ) + .line(NonZeroUsize::new(line + 1).unwrap()); + writeln!(err, "this line was expected to be `{}`", r[0]).unwrap(); + line += l.len(); + } + Remove(l) => { + let mut err = github_actions::error( + display(output_path), + "extraneous lines in output", + ) + .line(NonZeroUsize::new(line + 1).unwrap()); + writeln!( + err, + "remove this line and possibly later ones by blessing the test" + ) + .unwrap(); + line += l.len(); + } + Insert(r) => { + let mut err = + github_actions::error(display(output_path), "missing line in output") + .line(NonZeroUsize::new(line + 1).unwrap()); + writeln!(err, "bless the test to create a line containing `{}`", r[0]) + .unwrap(); + // Do not count these lines, they don't exist in the original file and + // would thus mess up the line number. + } + } + } + } + Error::ErrorsWithoutPattern { path, msgs } => { + if let Some((path, line)) = path.as_ref() { + let path = display(path); + let mut err = + github_actions::error(path, format!("Unmatched diagnostics{revision}")) + .line(*line); + for Message { + level, + message, + line: _, + span: _, + code: _, + } in msgs + { + writeln!(err, "{level:?}: {message}").unwrap(); + } + } else { + let mut err = github_actions::error( + test_path, + format!("Unmatched diagnostics outside the testfile{revision}"), + ); + for Message { + level, + message, + line: _, + span: _, + code: _, + } in msgs + { + writeln!(err, "{level:?}: {message}").unwrap(); + } + } + } + Error::InvalidComment { msg, span } => { + let mut err = github_actions::error(test_path, format!("Could not parse comment")) + .line(line(span)); + writeln!(err, "{msg}").unwrap(); + } + Error::MultipleRevisionsWithResults { kind, lines } => { + github_actions::error(test_path, format!("multiple {kind} found")) + .line(line(&lines[0])); + } + Error::Bug(_) => {} + Error::Aux { + path: aux_path, + errors, + } => { + github_actions::error(test_path, format!("Aux build failed")) + .line(line(&aux_path.span)); + for error in errors { + gha_error(error, &display(aux_path), "") + } + } + Error::Rustfix(error) => { + github_actions::error( + test_path, + format!("failed to apply suggestions with rustfix: {error}"), + ); + } + Error::ConfigError(msg) => { + github_actions::error(test_path, msg.clone()); + } + } +} + +/// Emits Github Actions Workspace commands to show the failures directly in the github diff view. +/// If the const generic `GROUP` boolean is `true`, also emit `::group` commands. +pub struct Gha { + /// Show a specific name for the final summary. + pub name: String, +} + +#[derive(Clone)] +struct PathAndRev { + path: PathBuf, + revision: String, +} + +impl TestStatus for PathAndRev { + fn path(&self) -> &Path { + &self.path + } + + fn for_revision(&self, revision: &str, _style: RevisionStyle) -> Box { + Box::new(Self { + path: self.path.clone(), + revision: revision.to_owned(), + }) + } + + fn for_path(&self, path: &Path) -> Box { + Box::new(Self { + path: path.to_path_buf(), + revision: self.revision.clone(), + }) + } + + fn failed_test(&self, _cmd: &str, _stderr: &[u8], _stdout: &[u8]) -> Box { + if GROUP { + Box::new(github_actions::group(format_args!( + "{}:{}", + display(&self.path), + self.revision + ))) + } else { + Box::new(()) + } + } + + fn revision(&self) -> &str { + &self.revision + } +} + +impl StatusEmitter for Gha { + fn register_test(&self, path: PathBuf) -> Box { + Box::new(PathAndRev:: { + path, + revision: String::new(), + }) + } + + fn finalize( + &self, + _failures: usize, + succeeded: usize, + ignored: usize, + filtered: usize, + // Can't aborted on gha + _aborted: bool, + ) -> Box { + struct Summarizer { + failures: Vec, + succeeded: usize, + ignored: usize, + filtered: usize, + name: String, + } + + impl Summary for Summarizer { + fn test_failure(&mut self, status: &dyn TestStatus, errors: &Errors) { + let revision = if status.revision().is_empty() { + "".to_string() + } else { + format!(" (revision: {})", status.revision()) + }; + for error in errors { + gha_error(error, &display(status.path()), &revision); + } + self.failures + .push(format!("{}{revision}", display(status.path()))); + } + } + impl Drop for Summarizer { + fn drop(&mut self) { + if let Some(mut file) = github_actions::summary() { + writeln!(file, "### {}", self.name).unwrap(); + for line in &self.failures { + writeln!(file, "* {line}").unwrap(); + } + writeln!(file).unwrap(); + writeln!(file, "| failed | passed | ignored | filtered out |").unwrap(); + writeln!(file, "| --- | --- | --- | --- |").unwrap(); + writeln!( + file, + "| {} | {} | {} | {} |", + self.failures.len(), + self.succeeded, + self.ignored, + self.filtered, + ) + .unwrap(); + } + } + } + + Box::new(Summarizer:: { + failures: vec![], + succeeded, + ignored, + filtered, + name: self.name.clone(), + }) + } +} diff --git a/src/status_emitter/text.rs b/src/status_emitter/text.rs new file mode 100644 index 00000000..d5606a47 --- /dev/null +++ b/src/status_emitter/text.rs @@ -0,0 +1,847 @@ +use super::RevisionStyle; +use super::StatusEmitter; +use super::Summary; +use super::TestStatus; +use crate::diagnostics::Level; +use crate::diagnostics::Message; +use crate::display; +use crate::parser::Pattern; +use crate::test_result::Errored; +use crate::test_result::TestOk; +use crate::test_result::TestResult; +use crate::Error; +use crate::Errors; +use crate::Format; +use annotate_snippets::Renderer; +use annotate_snippets::Snippet; +use colored::Colorize; +#[cfg(feature = "indicatif")] +use crossbeam_channel::{Sender, TryRecvError}; +#[cfg(feature = "indicatif")] +use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle}; +use spanned::Span; +use std::fmt::{Debug, Display}; +use std::io::Write as _; +use std::path::Path; +use std::path::PathBuf; + +#[cfg(feature = "indicatif")] +use std::{ + sync::{atomic::AtomicUsize, atomic::Ordering, Arc, Mutex}, + thread::JoinHandle, + time::Duration, +}; + +#[derive(Clone, Copy)] +enum OutputVerbosity { + Progress, + DiffOnly, + Full, +} + +/// A human readable output emitter. +#[derive(Clone)] +pub struct Text { + #[cfg(feature = "indicatif")] + sender: Sender, + progress: OutputVerbosity, + #[cfg(feature = "indicatif")] + handle: Arc, +} + +#[cfg(feature = "indicatif")] +struct JoinOnDrop(Mutex>>); +#[cfg(feature = "indicatif")] +impl From> for JoinOnDrop { + fn from(handle: JoinHandle<()>) -> Self { + Self(Mutex::new(Some(handle))) + } +} +#[cfg(feature = "indicatif")] +impl Drop for JoinOnDrop { + fn drop(&mut self) { + self.join(); + } +} + +#[cfg(feature = "indicatif")] +impl JoinOnDrop { + fn join(&self) { + let Ok(Some(handle)) = self.0.try_lock().map(|mut g| g.take()) else { + return; + }; + let _ = handle.join(); + } +} + +#[cfg(feature = "indicatif")] +#[derive(Debug)] +enum Msg { + Pop { + new_leftover_msg: String, + id: usize, + }, + Push { + id: usize, + parent: usize, + msg: String, + }, + Finish, + Abort, +} + +impl Text { + fn start_thread(progress: OutputVerbosity) -> Self { + #[cfg(feature = "indicatif")] + let (sender, receiver) = crossbeam_channel::unbounded(); + #[cfg(feature = "indicatif")] + let handle = std::thread::spawn(move || { + let bars = MultiProgress::new(); + let progress = match progress { + OutputVerbosity::Progress => bars.add(ProgressBar::new(0)), + OutputVerbosity::DiffOnly | OutputVerbosity::Full => { + ProgressBar::with_draw_target(Some(0), ProgressDrawTarget::hidden()) + } + }; + + struct Thread { + parent: usize, + spinner: ProgressBar, + /// Used for sanity assertions only + done: bool, + } + + impl Debug for Thread { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Thread") + .field("parent", &self.parent) + .field( + "spinner", + &format_args!("{}: {}", self.spinner.prefix(), self.spinner.message()), + ) + .field("done", &self.done) + .finish() + } + } + + struct ProgressHandler { + threads: Vec>, + aborted: bool, + bars: MultiProgress, + } + + impl ProgressHandler { + fn parents(&self, mut id: usize) -> impl Iterator + '_ { + std::iter::from_fn(move || { + let parent = self.threads[id].as_ref().unwrap().parent; + if parent == 0 { + None + } else { + id = parent; + Some(parent) + } + }) + } + + fn root(&self, id: usize) -> usize { + self.parents(id).last().unwrap_or(id) + } + + fn tree(&self, id: usize) -> impl Iterator { + let root = self.root(id); + // No need to look at the entries before `root`, as child nodes + // are always after parent nodes. + self.threads + .iter() + .filter_map(|t| t.as_ref()) + .enumerate() + .skip(root - 1) + .filter(move |&(i, t)| { + root == if t.parent == 0 { + i + } else { + self.root(t.parent) + } + }) + } + + fn tree_done(&self, id: usize) -> bool { + self.tree(id).all(|(_, t)| t.done) + } + + fn pop(&mut self, new_leftover_msg: String, id: usize) { + assert_ne!(id, 0); + let Some(Some(thread)) = self.threads.get_mut(id) else { + // This can happen when a test was not run at all, because it failed directly during + // comment parsing. + return; + }; + thread.done = true; + let spinner = thread.spinner.clone(); + spinner.finish_with_message(new_leftover_msg); + let progress = &self.threads[0].as_ref().unwrap().spinner; + progress.inc(1); + if self.tree_done(id) { + for (_, thread) in self.tree(id) { + self.bars.remove(&thread.spinner); + if progress.is_hidden() { + self.bars + .println(format!( + "{} {}", + thread.spinner.prefix(), + thread.spinner.message() + )) + .unwrap(); + } + } + } + } + + fn push(&mut self, parent: usize, id: usize, mut msg: String) { + assert!(parent < id); + self.threads[0].as_mut().unwrap().spinner.inc_length(1); + if self.threads.len() <= id { + self.threads.resize_with(id + 1, || None); + } + let parents = if parent == 0 { + 0 + } else { + self.parents(parent).count() + 1 + }; + for _ in 0..parents { + msg.insert_str(0, " "); + } + let spinner = ProgressBar::new_spinner().with_prefix(msg); + let spinner = if parent == 0 { + self.bars.add(spinner) + } else { + let last = self + .threads + .iter() + .enumerate() + .rev() + .filter_map(|(i, t)| Some((i, t.as_ref()?))) + .find(|&(i, _)| self.parents(i).any(|p| p == parent)) + .map(|(_, thread)| thread) + .unwrap_or_else(|| self.threads[parent].as_ref().unwrap()); + self.bars.insert_after(&last.spinner, spinner) + }; + spinner.set_style( + ProgressStyle::with_template("{prefix} {spinner}{msg}").unwrap(), + ); + let thread = &mut self.threads[id]; + assert!(thread.is_none()); + let _ = thread.insert(Thread { + parent, + spinner, + done: false, + }); + } + + fn tick(&self) { + for thread in self.threads.iter().flatten() { + if !thread.done { + thread.spinner.tick(); + } + } + } + } + + impl Drop for ProgressHandler { + fn drop(&mut self) { + let progress = self.threads[0].as_ref().unwrap(); + for (key, thread) in self.threads.iter().skip(1).enumerate() { + if let Some(thread) = thread { + assert!( + thread.done, + "{key} ({}: {}) not finished", + thread.spinner.prefix(), + thread.spinner.message() + ); + } + } + if self.aborted { + progress.spinner.abandon(); + } else { + assert_eq!( + Some(progress.spinner.position()), + progress.spinner.length(), + "{:?}", + self.threads + ); + progress.spinner.finish(); + } + } + } + + let mut handler = ProgressHandler { + threads: vec![Some(Thread { + parent: 0, + spinner: progress, + done: false, + })], + aborted: false, + bars, + }; + + 'outer: loop { + std::thread::sleep(Duration::from_millis(100)); + loop { + match receiver.try_recv() { + Ok(val) => match val { + Msg::Pop { + id, + new_leftover_msg, + } => { + handler.pop(new_leftover_msg, id); + } + + Msg::Push { parent, msg, id } => { + handler.push(parent, id, msg); + } + Msg::Finish => break 'outer, + Msg::Abort => handler.aborted = true, + }, + // Sender panicked, skip asserts + Err(TryRecvError::Disconnected) => return, + Err(TryRecvError::Empty) => break, + } + } + handler.tick() + } + }); + Self { + #[cfg(feature = "indicatif")] + sender, + progress, + #[cfg(feature = "indicatif")] + handle: Arc::new(handle.into()), + } + } + + /// Print one line per test that gets run. + pub fn verbose() -> Self { + Self::start_thread(OutputVerbosity::Full) + } + /// Print one line per test that gets run. + pub fn diff() -> Self { + Self::start_thread(OutputVerbosity::DiffOnly) + } + /// Print a progress bar. + pub fn quiet() -> Self { + Self::start_thread(OutputVerbosity::Progress) + } + + fn is_full_output(&self) -> bool { + matches!(self.progress, OutputVerbosity::Full) + } +} + +impl From for Text { + fn from(format: Format) -> Self { + match format { + Format::Terse => Text::quiet(), + Format::Pretty => Text::verbose(), + } + } +} + +struct TextTest { + text: Text, + #[cfg(feature = "indicatif")] + parent: usize, + #[cfg(feature = "indicatif")] + id: usize, + path: PathBuf, + revision: String, + style: RevisionStyle, +} + +#[cfg(feature = "indicatif")] +static ID_GENERATOR: AtomicUsize = AtomicUsize::new(1); + +impl TestStatus for TextTest { + fn done(&self, result: &TestResult, aborted: bool) { + #[cfg(feature = "indicatif")] + if aborted { + self.text.sender.send(Msg::Abort).unwrap(); + } + let result = match result { + _ if aborted => "aborted".white(), + Ok(TestOk::Ok) => "ok".green(), + Err(Errored { .. }) => "FAILED".bright_red().bold(), + Ok(TestOk::Ignored) => "ignored (in-test comment)".yellow(), + }; + let new_leftover_msg = format!("... {result}"); + #[cfg(feature = "indicatif")] + let print_immediately = ProgressDrawTarget::stdout().is_hidden(); + #[cfg(not(feature = "indicatif"))] + let print_immediately = true; + if print_immediately { + match self.style { + RevisionStyle::Separate => println!("{} {new_leftover_msg}", self.revision), + RevisionStyle::Show => { + let revision = if self.revision.is_empty() { + String::new() + } else { + format!(" (revision `{}`)", self.revision) + }; + println!("{}{revision} {new_leftover_msg}", display(&self.path)); + } + } + std::io::stdout().flush().unwrap(); + } + #[cfg(feature = "indicatif")] + self.text + .sender + .send(Msg::Pop { + id: self.id, + new_leftover_msg, + }) + .unwrap(); + } + + fn failed_test<'a>( + &self, + cmd: &str, + stderr: &'a [u8], + stdout: &'a [u8], + ) -> Box { + let maybe_revision = if self.revision.is_empty() { + String::new() + } else { + format!(" (revision `{}`)", self.revision) + }; + let text = format!( + "{} {}{}", + "FAILED TEST:".bright_red(), + display(&self.path), + maybe_revision + ); + + println!(); + println!("{}", text.bold().underline()); + println!("command: {cmd}"); + println!(); + + if self.text.is_full_output() { + #[derive(Debug)] + struct Guard<'a> { + stderr: &'a [u8], + stdout: &'a [u8], + } + impl<'a> Drop for Guard<'a> { + fn drop(&mut self) { + println!("{}", "full stderr:".bold()); + std::io::stdout().write_all(self.stderr).unwrap(); + println!(); + println!("{}", "full stdout:".bold()); + std::io::stdout().write_all(self.stdout).unwrap(); + println!(); + println!(); + } + } + Box::new(Guard { stderr, stdout }) + } else { + Box::new(()) + } + } + + fn path(&self) -> &Path { + &self.path + } + + fn for_revision(&self, revision: &str, style: RevisionStyle) -> Box { + let text = Self { + text: self.text.clone(), + path: self.path.clone(), + #[cfg(feature = "indicatif")] + parent: self.id, + #[cfg(feature = "indicatif")] + id: ID_GENERATOR.fetch_add(1, Ordering::Relaxed), + revision: revision.to_owned(), + style, + }; + // We already created the base entry + #[cfg(feature = "indicatif")] + if !revision.is_empty() { + self.text + .sender + .send(Msg::Push { + parent: text.parent, + id: text.id, + msg: text.revision.clone(), + }) + .unwrap(); + } + + Box::new(text) + } + + fn for_path(&self, path: &Path) -> Box { + let text = Self { + text: self.text.clone(), + path: path.to_path_buf(), + #[cfg(feature = "indicatif")] + parent: self.id, + #[cfg(feature = "indicatif")] + id: ID_GENERATOR.fetch_add(1, Ordering::Relaxed), + revision: String::new(), + style: RevisionStyle::Show, + }; + + #[cfg(feature = "indicatif")] + self.text + .sender + .send(Msg::Push { + id: text.id, + parent: text.parent, + msg: display(path), + }) + .unwrap(); + Box::new(text) + } + + fn revision(&self) -> &str { + &self.revision + } +} + +impl StatusEmitter for Text { + fn register_test(&self, path: PathBuf) -> Box { + #[cfg(feature = "indicatif")] + let id = ID_GENERATOR.fetch_add(1, Ordering::Relaxed); + #[cfg(feature = "indicatif")] + self.sender + .send(Msg::Push { + id, + parent: 0, + msg: display(&path), + }) + .unwrap(); + Box::new(TextTest { + text: self.clone(), + #[cfg(feature = "indicatif")] + parent: 0, + #[cfg(feature = "indicatif")] + id, + path, + revision: String::new(), + style: RevisionStyle::Show, + }) + } + + fn finalize( + &self, + _failures: usize, + succeeded: usize, + ignored: usize, + filtered: usize, + aborted: bool, + ) -> Box { + #[cfg(feature = "indicatif")] + self.sender.send(Msg::Finish).unwrap(); + + #[cfg(feature = "indicatif")] + self.handle.join(); + #[cfg(feature = "indicatif")] + if !ProgressDrawTarget::stdout().is_hidden() { + // The progress bars do not have a trailing newline, so let's + // add it here. + println!(); + } + // Print all errors in a single thread to show reliable output + struct Summarizer { + failures: Vec, + succeeded: usize, + ignored: usize, + filtered: usize, + aborted: bool, + } + + impl Summary for Summarizer { + fn test_failure(&mut self, status: &dyn TestStatus, errors: &Errors) { + for error in errors { + print_error(error, status.path()); + } + + self.failures.push(if status.revision().is_empty() { + format!(" {}", display(status.path())) + } else { + format!( + " {} (revision {})", + display(status.path()), + status.revision() + ) + }); + } + } + + impl Drop for Summarizer { + fn drop(&mut self) { + if self.failures.is_empty() { + println!(); + if self.aborted { + print!("test result: cancelled."); + } else { + print!("test result: {}.", "ok".green()); + } + } else { + println!("{}", "FAILURES:".bright_red().underline().bold()); + for line in &self.failures { + println!("{line}"); + } + println!(); + print!("test result: {}.", "FAIL".bright_red()); + print!(" {} failed", self.failures.len().to_string().green()); + if self.succeeded > 0 || self.ignored > 0 || self.filtered > 0 { + print!(";"); + } + } + if self.succeeded > 0 { + print!(" {} passed", self.succeeded.to_string().green()); + if self.ignored > 0 || self.filtered > 0 { + print!(";"); + } + } + if self.ignored > 0 { + print!(" {} ignored", self.ignored.to_string().yellow()); + if self.filtered > 0 { + print!(";"); + } + } + if self.filtered > 0 { + print!(" {} filtered out", self.filtered.to_string().yellow()); + } + println!(); + println!(); + } + } + Box::new(Summarizer { + failures: vec![], + succeeded, + ignored, + filtered, + aborted, + }) + } +} + +fn print_error(error: &Error, path: &Path) { + /// Every error starts with a header like that, to make them all easy to find. + /// It is made to look like the headers printed for spanned errors. + fn print_error_header(msg: impl Display) { + let text = format!("{} {msg}", "error:".bright_red()); + println!("{}", text.bold()); + } + + match error { + Error::ExitStatus { + status, + expected, + reason, + } => { + // `status` prints as `exit status: N`. + create_error( + format!("test got {status}, but expected {expected}"), + &[&[(reason, reason.span.clone())]], + path, + ) + } + Error::Command { kind, status } => { + // `status` prints as `exit status: N`. + print_error_header(format_args!("{kind} failed with {status}")); + } + Error::PatternNotFound { + pattern, + expected_line, + } => { + let line = match expected_line { + Some(line) => format!("on line {line}"), + None => format!("outside the testfile"), + }; + let msg = match &**pattern { + Pattern::SubString(s) => { + format!("`{s}` not found in diagnostics {line}") + } + Pattern::Regex(r) => { + format!("`/{r}/` does not match diagnostics {line}",) + } + }; + // This will print a suitable error header. + create_error( + msg, + &[&[("expected because of this pattern", pattern.span())]], + path, + ); + } + Error::CodeNotFound { + code, + expected_line, + } => { + let line = match expected_line { + Some(line) => format!("on line {line}"), + None => format!("outside the testfile"), + }; + create_error( + format!("diagnostic code `{}` not found {line}", &**code), + &[&[("expected because of this pattern", code.span())]], + path, + ); + } + Error::NoPatternsFound => { + print_error_header("expected error patterns, but found none"); + } + Error::PatternFoundInPassTest { mode, span } => { + let annot = [("expected because of this annotation", span.clone())]; + let mut lines: Vec<&[_]> = vec![&annot]; + let annot = [("expected because of this mode change", mode.clone())]; + if !mode.is_dummy() { + lines.push(&annot) + } + // This will print a suitable error header. + create_error("error pattern found in pass test", &lines, path); + } + Error::OutputDiffers { + path: output_path, + actual, + expected, + bless_command, + } => { + print_error_header("actual output differed from expected"); + if let Some(bless_command) = bless_command { + println!( + "Execute `{}` to update `{}` to the actual output", + bless_command, + display(output_path) + ); + } + println!("{}", format!("--- {}", display(output_path)).red()); + println!( + "{}", + format!( + "+++ <{} output>", + output_path.extension().unwrap().to_str().unwrap() + ) + .green() + ); + crate::diff::print_diff(expected, actual); + } + Error::ErrorsWithoutPattern { path, msgs } => { + if let Some((path, _)) = path.as_ref() { + let msgs = msgs + .iter() + .map(|msg| { + let text = match (&msg.code, msg.level) { + (Some(code), Level::Error) => { + format!("Error[{code}]: {}", msg.message) + } + _ => format!("{:?}: {}", msg.level, msg.message), + }; + (text, msg.span.clone().unwrap_or_default()) + }) + .collect::>(); + // This will print a suitable error header. + create_error( + format!("there were {} unmatched diagnostics", msgs.len()), + &[&msgs + .iter() + .map(|(msg, lc)| (msg.as_ref(), lc.clone())) + .collect::>()], + path, + ); + } else { + print_error_header(format_args!( + "there were {} unmatched diagnostics that occurred outside the testfile and had no pattern", + msgs.len(), + )); + for Message { + level, + message, + line: _, + code: _, + span: _, + } in msgs + { + println!(" {level:?}: {message}") + } + } + } + Error::InvalidComment { msg, span } => { + // This will print a suitable error header. + create_error(msg, &[&[("", span.clone())]], path) + } + Error::MultipleRevisionsWithResults { kind, lines } => { + let title = format!("multiple {kind} found"); + // This will print a suitable error header. + create_error( + title, + &lines.iter().map(|_line| &[] as &[_]).collect::>(), + path, + ) + } + Error::Bug(msg) => { + print_error_header("a bug in `ui_test` occurred"); + println!("{msg}"); + } + Error::Aux { + path: aux_path, + errors, + } => { + create_error( + "aux build failed", + &[&[(&path.display().to_string(), aux_path.span.clone())]], + &aux_path.span.file, + ); + for error in errors { + print_error(error, aux_path); + } + } + Error::Rustfix(error) => { + print_error_header(format_args!( + "failed to apply suggestions for {} with rustfix", + display(path) + )); + println!("{error}"); + println!("Add //@no-rustfix to the test file to ignore rustfix suggestions"); + } + Error::ConfigError(msg) => println!("{msg}"), + } + println!(); +} + +#[allow(clippy::type_complexity)] +fn create_error(s: impl AsRef, lines: &[&[(&str, Span)]], file: &Path) { + let source = std::fs::read_to_string(file).unwrap(); + let file = display(file); + let mut msg = annotate_snippets::Level::Error.title(s.as_ref()); + for &label in lines { + let annotations = label + .iter() + .filter(|(_, span)| !span.is_dummy()) + .map(|(label, span)| { + annotate_snippets::Level::Error + .span(span.bytes.clone()) + .label(label) + }) + .collect::>(); + if !annotations.is_empty() { + let snippet = Snippet::source(&source) + .fold(true) + .origin(&file) + .annotations(annotations); + msg = msg.snippet(snippet); + } + let footer = label + .iter() + .filter(|(_, span)| span.is_dummy()) + .map(|(label, _)| annotate_snippets::Level::Note.title(label)); + msg = msg.footers(footer); + } + let renderer = if colored::control::SHOULD_COLORIZE.should_colorize() { + Renderer::styled() + } else { + Renderer::plain() + }; + println!("{}", renderer.render(msg)); +} diff --git a/src/test_result.rs b/src/test_result.rs index a9c85ff3..56710db3 100644 --- a/src/test_result.rs +++ b/src/test_result.rs @@ -1,9 +1,8 @@ //! Various data structures used for carrying information about test success or failure -use std::sync::{atomic::AtomicBool, Arc}; - use crate::{status_emitter::TestStatus, Error}; use color_eyre::eyre::Result; +use std::sync::{atomic::AtomicBool, Arc}; /// The possible non-failure results a single test can have. #[derive(Debug)] diff --git a/src/tests.rs b/src/tests.rs index be2d78c6..e5ddae3c 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,11 +1,8 @@ -use std::path::PathBuf; - -use spanned::{Span, Spanned}; - +use super::*; use crate::diagnostics::Level; use crate::diagnostics::Message; - -use super::*; +use spanned::{Span, Spanned}; +use std::path::PathBuf; fn config() -> Config { Config { diff --git a/tests/integration.rs b/tests/integration.rs index c74eede8..9993960d 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -1,5 +1,4 @@ use std::path::Path; - use ui_test::{spanned::Spanned, *}; fn main() -> Result<()> { @@ -73,7 +72,10 @@ fn main() -> Result<()> { config.filter(" +[0-9]+: .*\n", ""); config.filter(" +at \\.?/.*\n", ""); config.filter(" running on .*", ""); - config.stdout_filter("/target/[^/]+/[^/]+/debug", "/target/$$TMP/$$TRIPLE/debug"); + config.stdout_filter( + "/target/[^/]+/[0-9]+/[^/]+/debug", + "/target/$$TMP/$$TRIPLE/debug", + ); config.stdout_filter("/target/.tmp[^/ \"]+", "/target/$$TMP"); // Normalize proc macro filenames on windows to their linux repr config.stdout_filter("/([^/\\.]+)\\.dll", "/lib$1.so"); @@ -138,6 +140,7 @@ fn main() -> Result<()> { |_, _| {}, ( text, + #[cfg(feature = "gha")] ui_test::status_emitter::Gha:: { name: "integration tests".into(), }, diff --git a/tests/integrations/basic-bin/Cargo.lock b/tests/integrations/basic-bin/Cargo.lock index c5450714..95a01d31 100644 --- a/tests/integrations/basic-bin/Cargo.lock +++ b/tests/integrations/basic-bin/Cargo.lock @@ -716,7 +716,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.26.5" +version = "0.27.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/basic-bin/tests/ui_tests.rs b/tests/integrations/basic-bin/tests/ui_tests.rs index dff2d2bb..852d0d25 100644 --- a/tests/integrations/basic-bin/tests/ui_tests.rs +++ b/tests/integrations/basic-bin/tests/ui_tests.rs @@ -4,9 +4,9 @@ fn main() -> ui_test::color_eyre::Result<()> { let path = "../../../target"; let mut config = Config { output_conflict_handling: if std::env::var_os("BLESS").is_some() { - OutputConflictHandling::Bless + ui_test::bless_output_files } else { - OutputConflictHandling::Error + ui_test::error_on_output_conflict }, bless_command: Some("cargo test".to_string()), ..Config::rustc("tests/actual_tests") @@ -19,7 +19,7 @@ fn main() -> ui_test::color_eyre::Result<()> { config.stdout_filter("in ([0-9]m )?[0-9\\.]+s", ""); config.stderr_filter(r"[^ ]*/\.?cargo/registry/.*/", "$$CARGO_REGISTRY"); config.stderr_filter(r"\.exe", ""); - config.stderr_filter("/target/[^/]+/[^/]+/debug", "/target/$$TMP/$$TRIPLE/debug"); + config.stderr_filter("/target/[^/]+/[0-9]+/[^/]+/debug", "/target/$$TMP/$$TRIPLE/debug"); config.path_stderr_filter(&std::path::Path::new(path), "$DIR"); // hide binaries generated for successfully passing tests diff --git a/tests/integrations/basic-fail-mode/Cargo.lock b/tests/integrations/basic-fail-mode/Cargo.lock index f4b9d007..7edb74dd 100644 --- a/tests/integrations/basic-fail-mode/Cargo.lock +++ b/tests/integrations/basic-fail-mode/Cargo.lock @@ -716,7 +716,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.26.5" +version = "0.27.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/basic-fail-mode/tests/ui_tests.rs b/tests/integrations/basic-fail-mode/tests/ui_tests.rs index de831532..92b14795 100644 --- a/tests/integrations/basic-fail-mode/tests/ui_tests.rs +++ b/tests/integrations/basic-fail-mode/tests/ui_tests.rs @@ -4,9 +4,9 @@ fn main() -> ui_test::color_eyre::Result<()> { let path = "../../../target"; let mut config = Config { output_conflict_handling: if std::env::var_os("BLESS").is_some() { - OutputConflictHandling::Bless + ui_test::bless_output_files } else { - OutputConflictHandling::Error + ui_test::error_on_output_conflict }, bless_command: Some("cargo test".to_string()), ..Config::rustc("tests/actual_tests") diff --git a/tests/integrations/basic-fail/Cargo.lock b/tests/integrations/basic-fail/Cargo.lock index 6783110d..4943732b 100644 --- a/tests/integrations/basic-fail/Cargo.lock +++ b/tests/integrations/basic-fail/Cargo.lock @@ -716,7 +716,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.26.5" +version = "0.27.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/basic-fail/Cargo.stderr b/tests/integrations/basic-fail/Cargo.stderr index 610e17aa..f875c62f 100644 --- a/tests/integrations/basic-fail/Cargo.stderr +++ b/tests/integrations/basic-fail/Cargo.stderr @@ -5,7 +5,7 @@ Location: error: test failed, to rerun pass `--test ui_tests` Caused by: - process didn't exit successfully: `$DIR/target/ui/tests/integrations/basic-fail/debug/deps/ui_tests-HASH` (exit status: 1) + process didn't exit successfully: `$DIR/target/ui/1/tests/integrations/basic-fail/debug/deps/ui_tests-HASH` (exit status: 1) Error: tests failed Location: @@ -13,7 +13,7 @@ Location: error: test failed, to rerun pass `--test ui_tests_diff_only` Caused by: - process didn't exit successfully: `$DIR/target/ui/tests/integrations/basic-fail/debug/deps/ui_tests_diff_only-HASH` (exit status: 1) + process didn't exit successfully: `$DIR/target/ui/1/tests/integrations/basic-fail/debug/deps/ui_tests_diff_only-HASH` (exit status: 1) Error: failed to parse rustc version info: invalid_foobarlaksdfalsdfj Caused by: @@ -23,7 +23,7 @@ Location: error: test failed, to rerun pass `--test ui_tests_invalid_program` Caused by: - process didn't exit successfully: `$DIR/target/ui/tests/integrations/basic-fail/debug/deps/ui_tests_invalid_program-HASH` (exit status: 1) + process didn't exit successfully: `$DIR/target/ui/1/tests/integrations/basic-fail/debug/deps/ui_tests_invalid_program-HASH` (exit status: 1) Error: tests failed Location: @@ -31,7 +31,7 @@ Location: error: test failed, to rerun pass `--test ui_tests_invalid_program2` Caused by: - process didn't exit successfully: `$DIR/target/ui/tests/integrations/basic-fail/debug/deps/ui_tests_invalid_program2-HASH` (exit status: 1) + process didn't exit successfully: `$DIR/target/ui/1/tests/integrations/basic-fail/debug/deps/ui_tests_invalid_program2-HASH` (exit status: 1) error: 4 targets failed: `--test ui_tests` `--test ui_tests_diff_only` diff --git a/tests/integrations/basic-fail/Cargo.stdout b/tests/integrations/basic-fail/Cargo.stdout index 221d77e2..9cd779d1 100644 --- a/tests/integrations/basic-fail/Cargo.stdout +++ b/tests/integrations/basic-fail/Cargo.stdout @@ -478,28 +478,37 @@ tests/actual_tests_bless/normalization_override.rs ... ok tests/actual_tests_bless/pass.rs ... ok tests/actual_tests_bless/pass_with_annotation.rs ... FAILED tests/actual_tests_bless/revised_revision.rs ... FAILED +tests/actual_tests_bless/revisioned_executable.rs ... ok tests/actual_tests_bless/revisioned_executable.rs (revision `run`) ... ok tests/actual_tests_bless/revisioned_executable.rs (revision `panic`) ... ok tests/actual_tests_bless/revisioned_executable.rs (revision `run.run`) ... ok tests/actual_tests_bless/revisioned_executable.rs (revision `panic.run`) ... FAILED +tests/actual_tests_bless/revisioned_executable_panic.rs ... ok tests/actual_tests_bless/revisioned_executable_panic.rs (revision `run`) ... ok tests/actual_tests_bless/revisioned_executable_panic.rs (revision `panic`) ... ok tests/actual_tests_bless/revisioned_executable_panic.rs (revision `run.run`) ... FAILED tests/actual_tests_bless/revisioned_executable_panic.rs (revision `panic.run`) ... ok +tests/actual_tests_bless/revisions.rs ... ok tests/actual_tests_bless/revisions.rs (revision `foo`) ... ok tests/actual_tests_bless/revisions.rs (revision `bar`) ... ok +tests/actual_tests_bless/revisions_bad.rs ... ok tests/actual_tests_bless/revisions_bad.rs (revision `foo`) ... ok tests/actual_tests_bless/revisions_bad.rs (revision `bar`) ... FAILED +tests/actual_tests_bless/revisions_filter.rs ... ok tests/actual_tests_bless/revisions_filter.rs (revision `foo`) ... ignored (in-test comment) tests/actual_tests_bless/revisions_filter.rs (revision `bar`) ... ignored (in-test comment) +tests/actual_tests_bless/revisions_filter2.rs ... ok tests/actual_tests_bless/revisions_filter2.rs (revision `foo`) ... ignored (in-test comment) tests/actual_tests_bless/revisions_filter2.rs (revision `bar`) ... ok +tests/actual_tests_bless/revisions_multiple_per_annotation.rs ... ok tests/actual_tests_bless/revisions_multiple_per_annotation.rs (revision `foo`) ... ok tests/actual_tests_bless/revisions_multiple_per_annotation.rs (revision `bar`) ... ok +tests/actual_tests_bless/revisions_same_everywhere.rs ... ok tests/actual_tests_bless/revisions_same_everywhere.rs (revision `foo`) ... ok tests/actual_tests_bless/revisions_same_everywhere.rs (revision `bar`) ... ok tests/actual_tests_bless/run_panic.rs ... ok tests/actual_tests_bless/run_panic.rs (revision `run`) ... ok +tests/actual_tests_bless/rustfix-fail-revisions.rs ... ok tests/actual_tests_bless/rustfix-fail-revisions.rs (revision `a`) ... ok tests/actual_tests_bless/rustfix-fail-revisions.rs (revision `b`) ... ok tests/actual_tests_bless/rustfix-fail-revisions.a.fixed ... FAILED @@ -549,7 +558,7 @@ full stdout: FAILED TEST: tests/actual_tests_bless/aux_proc_macro_no_main.rs -command: "rustc" "--error-format=json" "--crate-type=lib" "--out-dir" "$TMP "tests/actual_tests_bless/aux_proc_macro_no_main.rs" "--extern" "the_proc_macro=$DIR/tests/integrations/basic-fail/../../../target/$TMP/tests/actual_tests_bless/auxiliary/libthe_proc_macro.so" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/tests/actual_tests_bless/auxiliary" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021" +command: "rustc" "--error-format=json" "--crate-type=lib" "--out-dir" "$TMP "tests/actual_tests_bless/aux_proc_macro_no_main.rs" "--extern" "the_proc_macro=$DIR/tests/integrations/basic-fail/../../../target/$TMP/0/tests/actual_tests_bless/auxiliary/libthe_proc_macro.so" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/0/tests/actual_tests_bless/auxiliary" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021" error: there were 1 unmatched diagnostics --> tests/actual_tests_bless/aux_proc_macro_no_main.rs:7:8 @@ -817,7 +826,7 @@ full stdout: FAILED TEST: tests/actual_tests_bless/revisions_bad.rs (revision `bar`) -command: "rustc" "--error-format=json" "--out-dir" "$TMP "tests/actual_tests_bless/revisions_bad.rs" "--cfg=bar" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021" +command: "rustc" "--error-format=json" "--out-dir" "$TMP "tests/actual_tests_bless/revisions_bad.rs" "--cfg=bar" "-Cextra-filename=bar" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021" error: ``main` function not found in crate `revisions_bad`` not found in diagnostics outside the testfile --> tests/actual_tests_bless/revisions_bad.rs:4:31 @@ -1083,6 +1092,7 @@ FAILURES: test result: FAIL. 23 failed; 26 passed; 3 ignored Building dependencies ... ok +tests/actual_tests_bless_yolo/revisions_bad.rs ... ok tests/actual_tests_bless_yolo/revisions_bad.rs (revision `foo`) ... ok tests/actual_tests_bless_yolo/revisions_bad.rs (revision `bar`) ... ok tests/actual_tests_bless_yolo/rustfix-maybe-incorrect.rs ... ok diff --git a/tests/integrations/basic-fail/Cargo.toml b/tests/integrations/basic-fail/Cargo.toml index b9ca9521..585c7946 100644 --- a/tests/integrations/basic-fail/Cargo.toml +++ b/tests/integrations/basic-fail/Cargo.toml @@ -28,3 +28,5 @@ harness = false [[test]] name = "ui_tests_bless" harness = false + +#@normalize-stdout-test: "(command: canonicalizing path `tests/actual_tests_bless/auxiliary/aasdflkjasdlfjasdlfkjasdf`\n\nfull stderr:\n).*" -> "${1}No such file or directory" diff --git a/tests/integrations/basic-fail/tests/ui_tests.rs b/tests/integrations/basic-fail/tests/ui_tests.rs index eff61ae4..2432f749 100644 --- a/tests/integrations/basic-fail/tests/ui_tests.rs +++ b/tests/integrations/basic-fail/tests/ui_tests.rs @@ -4,7 +4,7 @@ fn main() -> ui_test::color_eyre::Result<()> { let path = "../../../target"; let mut config = Config { // Never bless integrations-fail tests, we want to see stderr mismatches - output_conflict_handling: OutputConflictHandling::Error, + output_conflict_handling: ui_test::error_on_output_conflict, bless_command: Some("DO NOT BLESS. These are meant to fail".to_string()), ..Config::rustc("tests/actual_tests") }; diff --git a/tests/integrations/basic-fail/tests/ui_tests_bless.rs b/tests/integrations/basic-fail/tests/ui_tests_bless.rs index 867b6827..2fb5e96f 100644 --- a/tests/integrations/basic-fail/tests/ui_tests_bless.rs +++ b/tests/integrations/basic-fail/tests/ui_tests_bless.rs @@ -14,9 +14,9 @@ fn main() -> ui_test::color_eyre::Result<()> { let mut config = Config { output_conflict_handling: if std::env::var_os("BLESS").is_some() { - OutputConflictHandling::Bless + ui_test::bless_output_files } else { - OutputConflictHandling::Error + ui_test::error_on_output_conflict }, bless_command: Some("cargo test".to_string()), ..Config::rustc(root_dir) diff --git a/tests/integrations/basic-fail/tests/ui_tests_diff_only.rs b/tests/integrations/basic-fail/tests/ui_tests_diff_only.rs index 041cbfb7..022eb3f2 100644 --- a/tests/integrations/basic-fail/tests/ui_tests_diff_only.rs +++ b/tests/integrations/basic-fail/tests/ui_tests_diff_only.rs @@ -3,7 +3,7 @@ use ui_test::*; fn main() -> ui_test::color_eyre::Result<()> { let config = Config { // Never bless integrations-fail tests, we want to see stderr mismatches - output_conflict_handling: OutputConflictHandling::Error, + output_conflict_handling: ui_test::error_on_output_conflict, bless_command: Some("DO NOT BLESS. These are meant to fail".to_string()), ..Config::rustc("tests/actual_tests") }; diff --git a/tests/integrations/basic-fail/tests/ui_tests_invalid_program.rs b/tests/integrations/basic-fail/tests/ui_tests_invalid_program.rs index db81901c..d0aac471 100644 --- a/tests/integrations/basic-fail/tests/ui_tests_invalid_program.rs +++ b/tests/integrations/basic-fail/tests/ui_tests_invalid_program.rs @@ -4,7 +4,7 @@ fn main() -> ui_test::color_eyre::Result<()> { let config = Config { program: CommandBuilder::cmd("invalid_foobarlaksdfalsdfj"), // Never bless integrations-fail tests, we want to see stderr mismatches - output_conflict_handling: OutputConflictHandling::Error, + output_conflict_handling: ui_test::error_on_output_conflict, bless_command: Some("DO NOT BLESS. These are meant to fail".to_string()), ..Config::rustc("tests/actual_tests") }; diff --git a/tests/integrations/basic-fail/tests/ui_tests_invalid_program2.rs b/tests/integrations/basic-fail/tests/ui_tests_invalid_program2.rs index b1bd1ea3..e3d072e0 100644 --- a/tests/integrations/basic-fail/tests/ui_tests_invalid_program2.rs +++ b/tests/integrations/basic-fail/tests/ui_tests_invalid_program2.rs @@ -4,7 +4,7 @@ fn main() -> ui_test::color_eyre::Result<()> { let config = Config { program: CommandBuilder::cmd("invalid_foobarlaksdfalsdfj"), // Never bless integrations-fail tests, we want to see stderr mismatches - output_conflict_handling: OutputConflictHandling::Error, + output_conflict_handling: ui_test::error_on_output_conflict, bless_command: Some("DO NOT BLESS. These are meant to fail".to_string()), host: Some("foo".into()), ..Config::rustc("tests/actual_tests") diff --git a/tests/integrations/basic/Cargo.lock b/tests/integrations/basic/Cargo.lock index df5689af..402be35f 100644 --- a/tests/integrations/basic/Cargo.lock +++ b/tests/integrations/basic/Cargo.lock @@ -639,7 +639,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.26.5" +version = "0.27.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/basic/tests/ui_tests.rs b/tests/integrations/basic/tests/ui_tests.rs index dc387201..9d51f43e 100644 --- a/tests/integrations/basic/tests/ui_tests.rs +++ b/tests/integrations/basic/tests/ui_tests.rs @@ -4,9 +4,9 @@ fn main() -> ui_test::color_eyre::Result<()> { let path = "../../../target"; let mut config = Config { output_conflict_handling: if std::env::var_os("BLESS").is_some() { - OutputConflictHandling::Bless + ui_test::bless_output_files } else { - OutputConflictHandling::Error + ui_test::error_on_output_conflict }, bless_command: Some("cargo test".to_string()), ..Config::rustc("tests/actual_tests") @@ -26,7 +26,7 @@ fn main() -> ui_test::color_eyre::Result<()> { if let Ok(target) = std::env::var("UITEST_TEST_TARGET") { config.target = Some(target); - config.output_conflict_handling = OutputConflictHandling::Ignore; + config.output_conflict_handling = ui_test::ignore_output_conflict; } // hide binaries generated for successfully passing tests diff --git a/tests/integrations/cargo-run/Cargo.lock b/tests/integrations/cargo-run/Cargo.lock index c5450714..95a01d31 100644 --- a/tests/integrations/cargo-run/Cargo.lock +++ b/tests/integrations/cargo-run/Cargo.lock @@ -716,7 +716,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.26.5" +version = "0.27.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/cargo-run/tests/ui_tests.rs b/tests/integrations/cargo-run/tests/ui_tests.rs index 57d09d75..4efd0ec2 100644 --- a/tests/integrations/cargo-run/tests/ui_tests.rs +++ b/tests/integrations/cargo-run/tests/ui_tests.rs @@ -3,9 +3,9 @@ use ui_test::{spanned::Spanned, *}; fn main() -> ui_test::color_eyre::Result<()> { let mut config = Config { output_conflict_handling: if std::env::var_os("BLESS").is_some() { - OutputConflictHandling::Bless + ui_test::bless_output_files } else { - OutputConflictHandling::Error + ui_test::error_on_output_conflict }, bless_command: Some("cargo test".to_string()), ..Config::cargo("tests/actual_tests") diff --git a/tests/integrations/dep-fail/Cargo.lock b/tests/integrations/dep-fail/Cargo.lock index 4d65ed83..8439c434 100644 --- a/tests/integrations/dep-fail/Cargo.lock +++ b/tests/integrations/dep-fail/Cargo.lock @@ -565,7 +565,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.26.5" +version = "0.27.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/dep-fail/Cargo.stderr b/tests/integrations/dep-fail/Cargo.stderr index e1902214..a9e6fbb3 100644 --- a/tests/integrations/dep-fail/Cargo.stderr +++ b/tests/integrations/dep-fail/Cargo.stderr @@ -5,6 +5,6 @@ Location: error: test failed, to rerun pass `--test ui` Caused by: - process didn't exit successfully: `$DIR/target/ui/tests/integrations/dep-fail/debug/deps/ui-HASH` (exit status: 1) + process didn't exit successfully: `$DIR/target/ui/1/tests/integrations/dep-fail/debug/deps/ui-HASH` (exit status: 1) error: 1 target failed: `--test ui` diff --git a/tests/integrations/dep-fail/Cargo.stdout b/tests/integrations/dep-fail/Cargo.stdout index c053179c..370874ff 100644 --- a/tests/integrations/dep-fail/Cargo.stdout +++ b/tests/integrations/dep-fail/Cargo.stdout @@ -7,7 +7,7 @@ Building dependencies ... FAILED tests/ui/basic_test.rs ... FAILED FAILED TEST: tests/ui/basic_test.rs -command: "$CMD" "build" "--color=never" "--quiet" "--jobs" "1" "--target-dir" "$DIR/tests/integrations/dep-fail/target/ui" "--manifest-path" "tested_crate/Cargo.toml" "--message-format=json" +command: "$CMD" "build" "--color=never" "--quiet" "--jobs" "1" "--target-dir" "$DIR/tests/integrations/dep-fail/target/ui/0" "--manifest-path" "tested_crate/Cargo.toml" "--message-format=json" full stderr: error: could not compile `tested_crate` (lib) due to 2 previous errors diff --git a/tests/integrations/dep-fail/tests/ui.rs b/tests/integrations/dep-fail/tests/ui.rs index bad5401b..447a963a 100644 --- a/tests/integrations/dep-fail/tests/ui.rs +++ b/tests/integrations/dep-fail/tests/ui.rs @@ -1,11 +1,11 @@ use ui_test::{ default_file_filter, default_per_file_config, dependencies::DependencyBuilder, - run_tests_generic, status_emitter::Text, Args, Config, OutputConflictHandling, + run_tests_generic, status_emitter::Text, Args, Config, }; fn main() -> ui_test::Result<()> { let mut config = Config { - output_conflict_handling: OutputConflictHandling::Ignore, + output_conflict_handling: ui_test::ignore_output_conflict, ..Config::rustc("tests/ui") }; config.comment_defaults.base().set_custom( diff --git a/tests/integrations/ui_test_dep_bug/Cargo.lock b/tests/integrations/ui_test_dep_bug/Cargo.lock index 51a379ee..5ec27509 100644 --- a/tests/integrations/ui_test_dep_bug/Cargo.lock +++ b/tests/integrations/ui_test_dep_bug/Cargo.lock @@ -570,7 +570,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.26.5" +version = "0.27.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/ui_test_dep_bug/tests/ui.rs b/tests/integrations/ui_test_dep_bug/tests/ui.rs index ea2b3d34..8a304e53 100644 --- a/tests/integrations/ui_test_dep_bug/tests/ui.rs +++ b/tests/integrations/ui_test_dep_bug/tests/ui.rs @@ -1,11 +1,11 @@ use ui_test::{ default_file_filter, default_per_file_config, dependencies::DependencyBuilder, - run_tests_generic, status_emitter::Text, Args, Config, OutputConflictHandling, + run_tests_generic, status_emitter::Text, Args, Config, }; fn main() -> ui_test::Result<()> { let mut config = Config { - output_conflict_handling: OutputConflictHandling::Ignore, + output_conflict_handling: ui_test::ignore_output_conflict, ..Config::rustc("tests/ui") }; config