Merge pull request #283 from oli-obk/miri

Various changes needed for miri

CHANGELOG.md

@@ -13,8 +13,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+## [0.27.0] - 2024-10-07
+
+### Added
+
+* debug status emitter for when you have problems with ui_test
+* cargo features to disable gha or CLI refreshing 
+
+### Fixed
+
+* CLI refreshing is now reliable and not leaving around fragments
+* Can run multiple `Config`s that test the same files in parallel.
+
+### Changed
+
 * `only`/`ignore` filters now only accept integers, alphabetic characters, `-` and `_`
 * `only`/ `ignore` filters allow comments by ignoring everything from an `#` onwards
+* `OutputConflictHandling` has been replaced by `error_on_output_conflict`, `bless_output_files`,
+    and `ignore_output_conflict` functions. Custom functions can now be used to implement special
+    handling of output conflicts.
+* `Run` now forwards env vars passed to the compiler to the executable, too
 
 ### Removed
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ui_test"
-version = "0.26.5"
+version = "0.27.0"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 description = "A test framework for testing rustc diagnostics output"
@@ -23,7 +23,7 @@ rustfix = "0.8.1"
 cargo-platform = { version = "0.1.2", optional = true }
 comma = "1.0.0"
 anyhow = "1.0.6"
-indicatif = "0.17.6"
+indicatif = { version = "0.17.6", optional = true }
 prettydiff = { version = "0.7", default-features = false }
 annotate-snippets = { version = "0.11.2" }
 levenshtein = "1.0.5"
@@ -52,5 +52,6 @@ name = "build_std"
 test = false
 
 [features]
-default = ["rustc"]
+default = ["rustc", "indicatif", "gha"]
+gha = []
 rustc = ["dep:cargo-platform", "dep:cargo_metadata"]

examples/rustc_basic.rs

@@ -4,6 +4,7 @@ use std::sync::atomic::Ordering;
 use ui_test::{run_tests, Config};
 
 #[cfg(feature = "rustc")]
+#[cfg_attr(test, test)]
 fn main() -> ui_test::color_eyre::Result<()> {
     let config = Config::rustc("examples_tests/rustc_basic");
     let abort_check = config.abort_check.clone();

examples/rustc_twice_with_different_flags.rs

@@ -0,0 +1,29 @@
+#[cfg(feature = "rustc")]
+use std::sync::atomic::Ordering;
+#[cfg(feature = "rustc")]
+use ui_test::{
+    default_file_filter, default_per_file_config, run_tests_generic, status_emitter, Config,
+};
+
+#[cfg(feature = "rustc")]
+#[cfg_attr(test, test)]
+fn main() -> ui_test::color_eyre::Result<()> {
+    let config = Config::rustc("examples_tests/rustc_basic");
+    let abort_check = config.abort_check.clone();
+    ctrlc::set_handler(move || abort_check.store(true, Ordering::Relaxed))?;
+
+    // Compile all `.rs` files in the given directory (relative to your
+    // Cargo.toml) and compare their output against the corresponding
+    // `.stderr` files.
+    run_tests_generic(
+        vec![config.clone(), config],
+        default_file_filter,
+        default_per_file_config,
+        status_emitter::Text::verbose(),
+    )
+}
+
+#[cfg(not(feature = "rustc"))]
+fn main() -> ui_test::color_eyre::Result<()> {
+    Ok(())
+}

examples_tests/rustc_basic/aux_derive.stderr

@@ -20,12 +20,14 @@ error[E0384]: cannot assign twice to immutable variable `x`
  --> examples_tests/rustc_basic/aux_derive.rs:8:5
   |
 7 |     let x = Foo;
-  |         -
-  |         |
-  |         first assignment to `x`
-  |         help: consider making this binding mutable: `mut x`
+  |         - first assignment to `x`
 8 |     x = Foo;
   |     ^^^^^^^ cannot assign twice to immutable variable
+  |
+help: consider making this binding mutable
+  |
+7 |     let mut x = Foo;
+  |         +++
 
 error: aborting due to 1 previous error; 2 warnings emitted
 

examples_tests/rustc_basic/executable.rs

@@ -1,4 +1,5 @@
 //@run
+//@revisions: a b c d e f g h i j k l m n
 
 fn main() {
     std::thread::sleep(std::time::Duration::from_secs(5));

examples_tests/rustc_basic/filtered_out.rs

@@ -1,4 +1,4 @@
-//@[foo] only-target-asdfasdf
+//@[foo] only-target: asdfasdf
 
 //@ revisions: foo bar
 

src/aux_builds.rs

@@ -1,10 +1,6 @@
 //! Everything needed to build auxilary files with rustc
 // lol we can't name this file `aux.rs` on windows
 
-use bstr::ByteSlice;
-use spanned::Spanned;
-use std::{ffi::OsString, path::PathBuf, process::Command, sync::Arc};
-
 use crate::{
     build_manager::{Build, BuildManager},
     custom_flags::Flag,
@@ -13,6 +9,9 @@ use crate::{
     status_emitter::SilentStatus,
     CrateType, Error, Errored,
 };
+use bstr::ByteSlice;
+use spanned::Spanned;
+use std::{ffi::OsString, path::PathBuf, process::Command, sync::Arc};
 
 impl Flag for AuxBuilder {
     fn must_be_unique(&self) -> bool {

src/build_manager.rs

@@ -1,19 +1,17 @@
 //! Auxiliary and dependency builder. Extendable to custom builds.
 
-use std::{
-    collections::{hash_map::Entry, HashMap},
-    ffi::OsString,
-    sync::{Arc, OnceLock, RwLock},
-};
-
-use color_eyre::eyre::Result;
-use crossbeam_channel::{bounded, Sender};
-
 use crate::{
     status_emitter::{RevisionStyle, TestStatus},
     test_result::TestRun,
     Config, Errored,
 };
+use color_eyre::eyre::Result;
+use crossbeam_channel::{bounded, Sender};
+use std::{
+    collections::{hash_map::Entry, HashMap},
+    ffi::OsString,
+    sync::{Arc, OnceLock, RwLock},
+};
 
 /// A build shared between all tests of the same `BuildManager`
 pub trait Build {

src/cmd.rs

@@ -1,11 +1,10 @@
+use crate::display;
 use std::{
     ffi::OsString,
     path::{Path, PathBuf},
     process::Command,
 };
 
-use crate::display;
-
 #[derive(Debug, Clone)]
 /// A command, its args and its environment. Used for
 /// the main command, the dependency builder and the cfg-reader.

src/config.rs

@@ -1,6 +1,3 @@
-use regex::bytes::Regex;
-use spanned::Spanned;
-
 #[cfg(feature = "rustc")]
 use crate::{
     aux_builds::AuxBuilder, custom_flags::run::Run, custom_flags::rustfix::RustfixMode,
@@ -10,10 +7,12 @@ use crate::{
     diagnostics::Diagnostics,
     parser::CommandParserFunc,
     per_test_config::{Comments, Condition},
-    CommandBuilder,
+    CommandBuilder, Error, Errors,
 };
 pub use color_eyre;
 use color_eyre::eyre::Result;
+use regex::bytes::Regex;
+use spanned::Spanned;
 use std::{
     collections::BTreeMap,
     num::NonZeroUsize,
@@ -40,7 +39,7 @@ pub struct Config {
     /// The recommended command to bless failing tests.
     pub bless_command: Option<String>,
     /// Where to dump files like the binaries compiled from tests.
-    /// Defaults to `target/ui` in the current directory.
+    /// Defaults to `target/ui/index_of_config` in the current directory.
     pub out_dir: PathBuf,
     /// Skip test files whose names contain any of these entries.
     pub skip_files: Vec<String>,
@@ -68,6 +67,9 @@ pub struct Config {
     pub abort_check: Arc<AtomicBool>,
 }
 
+/// Function that performs the actual output conflict handling.
+pub type OutputConflictHandling = fn(&Path, Vec<u8>, &mut Errors, &Config);
+
 impl Config {
     /// Create a blank configuration that doesn't do anything interesting
     pub fn dummy() -> Self {
@@ -78,7 +80,7 @@ impl Config {
             target: Default::default(),
             root_dir: Default::default(),
             program: CommandBuilder::cmd(""),
-            output_conflict_handling: OutputConflictHandling::Error,
+            output_conflict_handling: error_on_output_conflict,
             bless_command: Default::default(),
             out_dir: Default::default(),
             skip_files: Default::default(),
@@ -117,7 +119,12 @@ impl Config {
             fn clone_inner(&self) -> Box<dyn Flag> {
                 Box::new(NeedsAsmSupport)
             }
-            fn test_condition(&self, config: &Config) -> bool {
+            fn test_condition(
+                &self,
+                config: &Config,
+                _comments: &Comments,
+                _revision: &str,
+            ) -> bool {
                 let target = config.target.as_ref().unwrap();
                 static ASM_SUPPORTED_ARCHS: &[&str] = &[
                     "x86", "x86_64", "arm", "aarch64", "riscv32",
@@ -154,7 +161,7 @@ impl Config {
             target: None,
             root_dir: root_dir.into(),
             program: CommandBuilder::rustc(),
-            output_conflict_handling: OutputConflictHandling::Error,
+            output_conflict_handling: error_on_output_conflict,
             bless_command: None,
             out_dir: std::env::var_os("CARGO_TARGET_DIR")
                 .map(PathBuf::from)
@@ -194,7 +201,14 @@ impl Config {
 
         config.custom_comments.insert("run", |parser, args, span| {
             let set = |exit_code| {
-                parser.set_custom_once("run", Run { exit_code }, args.span());
+                parser.set_custom_once(
+                    "run",
+                    Run {
+                        exit_code,
+                        output_conflict_handling: None,
+                    },
+                    args.span(),
+                );
                 parser.exit_status = Spanned::new(0, span.clone()).into();
                 parser.require_annotations = Spanned::new(false, span.clone()).into();
 
@@ -268,9 +282,9 @@ impl Config {
         self.list = list;
 
         if check {
-            self.output_conflict_handling = OutputConflictHandling::Error;
+            self.output_conflict_handling = error_on_output_conflict;
         } else if bless {
-            self.output_conflict_handling = OutputConflictHandling::Bless;
+            self.output_conflict_handling = bless_output_files;
         }
     }
 
@@ -411,9 +425,12 @@ impl Config {
             return self.run_only_ignored;
         }
         if comments.for_revision(revision).any(|r| {
-            r.custom
-                .values()
-                .any(|flags| flags.content.iter().any(|flag| flag.test_condition(self)))
+            r.custom.values().any(|flags| {
+                flags
+                    .content
+                    .iter()
+                    .any(|flag| flag.test_condition(self, comments, revision))
+            })
         }) {
             return self.run_only_ignored;
         }
@@ -429,16 +446,41 @@ impl Config {
     }
 }
 
-#[derive(Debug, Clone)]
-/// The different options for what to do when stdout/stderr files differ from the actual output.
-pub enum OutputConflictHandling {
-    /// Fail the test when mismatches are found, if provided the command string
-    /// in [`Config::bless_command`] will be suggested as a way to bless the
-    /// test.
-    Error,
-    /// Ignore mismatches in the stderr/stdout files.
-    Ignore,
-    /// Instead of erroring if the stderr/stdout differs from the expected
-    /// automatically replace it with the found output (after applying filters).
-    Bless,
+/// Fail the test when mismatches are found, if provided the command string
+/// in [`Config::bless_command`] will be suggested as a way to bless the
+/// test.
+pub fn error_on_output_conflict(
+    path: &Path,
+    actual: Vec<u8>,
+    errors: &mut Errors,
+    config: &Config,
+) {
+    let expected = std::fs::read(path).unwrap_or_default();
+    if actual != expected {
+        errors.push(Error::OutputDiffers {
+            path: path.to_path_buf(),
+            actual,
+            expected,
+            bless_command: config.bless_command.clone(),
+        });
+    }
+}
+
+/// Ignore mismatches in the stderr/stdout files.
+pub fn ignore_output_conflict(
+    _path: &Path,
+    _actual: Vec<u8>,
+    _errors: &mut Errors,
+    _config: &Config,
+) {
+}
+
+/// Instead of erroring if the stderr/stdout differs from the expected
+/// automatically replace it with the found output (after applying filters).
+pub fn bless_output_files(path: &Path, actual: Vec<u8>, _errors: &mut Errors, _config: &Config) {
+    if actual.is_empty() {
+        let _ = std::fs::remove_file(path);
+    } else {
+        std::fs::write(path, &actual).unwrap();
+    }
 }

src/config/args.rs

@@ -1,9 +1,8 @@
 //! Default argument processing when `ui_test` is used
 //! as a test driver.
 
-use std::{borrow::Cow, num::NonZeroUsize};
-
 use color_eyre::eyre::{bail, ensure, Result};
+use std::{borrow::Cow, num::NonZeroUsize};
 
 /// Plain arguments if `ui_test` is used as a binary.
 #[derive(Debug, Default)]

src/custom_flags.rs

@@ -1,12 +1,13 @@
 //! Define custom test flags not natively supported by ui_test
 
+use crate::{
+    build_manager::BuildManager, parser::Comments, per_test_config::TestConfig, Config, Errored,
+};
 use std::{
     panic::{RefUnwindSafe, UnwindSafe},
     process::{Command, Output},
 };
 
-use crate::{build_manager::BuildManager, per_test_config::TestConfig, Config, Errored};
-
 #[cfg(feature = "rustc")]
 pub mod edition;
 #[cfg(feature = "rustc")]
@@ -29,7 +30,7 @@ pub trait Flag: Send + Sync + UnwindSafe + RefUnwindSafe + std::fmt::Debug {
     }
 
     /// Whether this flag causes a test to be filtered out
-    fn test_condition(&self, _config: &Config) -> bool {
+    fn test_condition(&self, _config: &Config, _comments: &Comments, _revision: &str) -> bool {
         false
     }