From ad897e7cdb8f49e7943290a007bdd5c401e14eb9 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sat, 3 Dec 2022 09:52:15 -0800 Subject: [PATCH 01/10] WIP: Write separate diff files --- README.md | 4 +++- src/lab.rs | 3 ++- src/mutate.rs | 5 ++++- src/outcome.rs | 11 +++++++---- src/output.rs | 20 ++++++++++++++++++++ tests/cli/main.rs | 10 ++++++++++ 6 files changed, 46 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d0837ee3..cd23f0b1 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,8 @@ A `mutants.out` directory is created in the source directory, or whichever direc unmutated case. The log contains the diff of the mutation plus the output from cargo. +* A `diff/` directory, with one diff file for each mutation. + * A `lock.json`, on which an [fs2 lock](https://docs.rs/fs2) is held while cargo-mutants is running, to avoid two tasks trying to write to the same directory at the same time. The lock contains the start time, cargo-mutants @@ -270,7 +272,7 @@ A `mutants.out` directory is created in the source directory, or whichever direc * A `mutants.json` file describing all the generated mutants. -* An `outcomes.json` file describing the results of all tests. +* An `outcomes.json` file describing the results of all tests, including the path of the log and diff files. ### Hangs and timeouts diff --git a/src/lab.rs b/src/lab.rs index ad9ac6cb..4f610d6d 100644 --- a/src/lab.rs +++ b/src/lab.rs @@ -169,8 +169,9 @@ fn test_scenario( mutant.apply(build_dir)?; } console.scenario_started(scenario, log_file.path()); + let diff_filename = output_mutex.lock().unwrap().write_diff_file(scenario)?; - let mut outcome = ScenarioOutcome::new(&log_file, scenario.clone()); + let mut outcome = ScenarioOutcome::new(&log_file, diff_filename, scenario.clone()); let phases: &[Phase] = if options.check_only { &[Phase::Check] } else { diff --git a/src/mutate.rs b/src/mutate.rs index 776f5ce5..e2e88ac5 100644 --- a/src/mutate.rs +++ b/src/mutate.rs @@ -191,10 +191,13 @@ impl Mutant { .with_context(|| format!("failed to write mutated code to {:?}", path)) } + /// Return a filename part, without slashes or extension, that can be used for log and diff files. pub fn log_file_name_base(&self) -> String { + // TODO: Also include a unique number so that they can't collide, even + // with similar mutants on the same line? format!( "{}_line_{}", - self.source_file.tree_relative_slashes(), + self.source_file.tree_relative_slashes().replace('/', "__"), self.span.start.line ) } diff --git a/src/outcome.rs b/src/outcome.rs index 589180ca..a124169d 100644 --- a/src/outcome.rs +++ b/src/outcome.rs @@ -144,6 +144,7 @@ pub struct ScenarioOutcome { /// A file holding the text output from running this test. // TODO: Maybe this should be a log object? log_path: Utf8PathBuf, + diff_path: Utf8PathBuf, /// What kind of scenario was being built? pub scenario: Scenario, /// For each phase, the duration and the cargo result. @@ -155,20 +156,22 @@ impl Serialize for ScenarioOutcome { where S: Serializer, { - // custom serialize to omit inessential info - let mut ss = serializer.serialize_struct("Outcome", 4)?; + // custom serialize to omit inessential info and to inline a summary. + let mut ss = serializer.serialize_struct("Outcome", 5)?; ss.serialize_field("scenario", &self.scenario)?; - ss.serialize_field("log_path", &self.log_path)?; ss.serialize_field("summary", &self.summary())?; + ss.serialize_field("log_path", &self.log_path)?; + ss.serialize_field("diff_path", &self.diff_path)?; ss.serialize_field("phase_results", &self.phase_results)?; ss.end() } } impl ScenarioOutcome { - pub fn new(log_file: &LogFile, scenario: Scenario) -> ScenarioOutcome { + pub fn new(log_file: &LogFile, diff_path: Utf8PathBuf, scenario: Scenario) -> ScenarioOutcome { ScenarioOutcome { log_path: log_file.path().to_owned(), + diff_path, scenario, phase_results: Vec::new(), } diff --git a/src/output.rs b/src/output.rs index 0ea8e8ba..1190bbff 100644 --- a/src/output.rs +++ b/src/output.rs @@ -89,6 +89,7 @@ impl LockFile { pub struct OutputDir { path: Utf8PathBuf, log_dir: Utf8PathBuf, + diff_dir: Utf8PathBuf, #[allow(unused)] // Lifetime controls the file lock lock_file: File, /// A file holding a list of missed mutants as text, one per line. @@ -136,6 +137,8 @@ impl OutputDir { .context("create lock.json lock file")?; let log_dir = output_dir.join("log"); fs::create_dir(&log_dir).with_context(|| format!("create log directory {:?}", &log_dir))?; + let diff_dir = output_dir.join("diff"); + fs::create_dir(&diff_dir).context("create diff dir")?; // Create text list files. let mut list_file_options = OpenOptions::new(); @@ -156,6 +159,7 @@ impl OutputDir { path: output_dir, lab_outcome: LabOutcome::new(), log_dir, + diff_dir, lock_file, missed_list, caught_list, @@ -207,6 +211,21 @@ impl OutputDir { Ok(()) } + pub fn write_diff_file(&self, scenario: &Scenario) -> Result { + // TODO: Unify with the cleaning/uniquifying code in LogFile... + let diff_filename = self + .diff_dir + .join(format!("{}.diff", scenario.log_file_name_base())); + let diff = if let Scenario::Mutant(mutant) = scenario { + mutant.diff() + } else { + String::new() + }; + fs::write(&diff_filename, diff.as_bytes()) + .with_context(|| format!("write diff file {diff_filename:?}"))?; + Ok(diff_filename) + } + pub fn open_debug_log(&self) -> Result { let debug_log_path = self.path.join("debug.log"); OpenOptions::new() @@ -287,6 +306,7 @@ version = "0.0.0" "Cargo.toml", "mutants.out", "mutants.out/caught.txt", + "mutants.out/diff", "mutants.out/lock.json", "mutants.out/log", "mutants.out/missed.txt", diff --git a/tests/cli/main.rs b/tests/cli/main.rs index 3fba6cd8..a99881d5 100644 --- a/tests/cli/main.rs +++ b/tests/cli/main.rs @@ -2,6 +2,7 @@ //! Tests for cargo-mutants CLI layer. +use std::collections::HashSet; use std::fmt::Write; use std::fs::{self, read_dir}; use std::io::Read; @@ -11,6 +12,7 @@ use std::thread::sleep; use std::time::Duration; use assert_cmd::prelude::OutputAssertExt; +use camino::Utf8Path; use itertools::Itertools; // use assert_cmd::prelude::*; // use assert_cmd::Command; @@ -524,6 +526,14 @@ fn workspace_tree_is_well_tested() { ["test", "--workspace"] ); } + + // The outcomes all have `diff_path` keys and they all identify files. + let mut all_diffs = HashSet::new(); + for outcome_json in json["outcomes"].as_array().unwrap() { + let diff_path = outcome_json["diff_path"].as_str().unwrap(); + assert!(Utf8Path::new(diff_path).is_file()); + assert!(all_diffs.insert(diff_path)); + } } #[test] From fbdbc35f1573c73d044484886058f99ad986a8d3 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sat, 24 Aug 2024 13:12:28 -0700 Subject: [PATCH 02/10] Add ScenarioOutput concept --- src/lab.rs | 11 ++--- src/log_file.rs | 35 +++++----------- src/output.rs | 105 +++++++++++++++++++++++++++++++----------------- src/scenario.rs | 7 ---- tests/main.rs | 9 ++++- 5 files changed, 92 insertions(+), 75 deletions(-) diff --git a/src/lab.rs b/src/lab.rs index 172c5549..95444c92 100644 --- a/src/lab.rs +++ b/src/lab.rs @@ -11,6 +11,7 @@ use std::thread; use std::time::Instant; use itertools::Itertools; +use output::ScenarioOutput; use tracing::{debug, debug_span, error, trace, warn}; use crate::cargo::run_cargo; @@ -209,13 +210,13 @@ fn test_scenario( options: &Options, console: &Console, ) -> Result { - let mut log_file = output_mutex + let scenario_output = output_mutex .lock() - .expect("lock output_dir to create log") - .create_log(scenario)?; + .expect("lock output_dir to start scenario") + .start_scenario(scenario)?; + let ScenarioOutput { mut log_file, .. } = scenario_output; log_file.message(&scenario.to_string()); console.scenario_started(build_dir.path().as_ref(), scenario, log_file.path())?; - let diff_filename = output_mutex.lock().unwrap().write_diff_file(scenario)?; let phases: &[Phase] = if options.check_only { &[Phase::Check] @@ -234,7 +235,7 @@ fn test_scenario( let dir: &Path = build_dir.path().as_ref(); console.scenario_started(dir, scenario, log_file.path())?; - let mut outcome = ScenarioOutcome::new(&log_file, &diff_filename, scenario.clone()); + let mut outcome = ScenarioOutcome::new(&log_file, &scenario_output.diff_path, scenario.clone()); for &phase in phases { console.scenario_phase_started(dir, phase); let timeout = match phase { diff --git a/src/log_file.rs b/src/log_file.rs index e7db7a99..f078d36d 100644 --- a/src/log_file.rs +++ b/src/log_file.rs @@ -4,7 +4,7 @@ //! and test cases, mixed with commentary from cargo-mutants. use std::fs::{File, OpenOptions}; -use std::io::{self, Write}; +use std::io::Write; use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; @@ -22,30 +22,15 @@ pub struct LogFile { } impl LogFile { - pub fn create_in(log_dir: &Utf8Path, scenario_name: &str) -> Result { - let basename = clean_filename(scenario_name); - for i in 0..1000 { - let t = if i == 0 { - format!("{basename}.log") - } else { - format!("{basename}_{i:03}.log") - }; - let path = log_dir.join(t); - match OpenOptions::new() - .read(true) - .append(true) - .create_new(true) - .open(&path) - { - Ok(write_to) => return Ok(LogFile { path, write_to }), - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, - Err(e) => return Err(anyhow::Error::from(e).context("create test log file")), - } - } - unreachable!( - "couldn't create any test log in {:?} for {:?}", - log_dir, scenario_name, - ); + pub fn create_in(log_dir: &Utf8Path, basename: &str) -> Result { + let path = log_dir.join(format!("{basename}.log")); + let write_to = OpenOptions::new() + .create_new(true) + .read(true) + .append(true) + .open(&path) + .with_context(|| format!("create test log file {path:?}"))?; + Ok(LogFile { path, write_to }) } /// Open the log file to append more content. diff --git a/src/output.rs b/src/output.rs index 962fc88e..e047f1aa 100644 --- a/src/output.rs +++ b/src/output.rs @@ -2,6 +2,8 @@ //! A `mutants.out` directory holding logs and other output. +use std::collections::hash_map::Entry; +use std::collections::HashMap; use std::fs::{create_dir, remove_dir_all, rename, write, File, OpenOptions}; use std::io::{BufWriter, Write}; use std::path::Path; @@ -87,7 +89,6 @@ impl LockFile { pub struct OutputDir { path: Utf8PathBuf, log_dir: Utf8PathBuf, - diff_dir: Utf8PathBuf, #[allow(unused)] // Lifetime controls the file lock lock_file: File, /// A file holding a list of missed mutants as text, one per line. @@ -99,6 +100,12 @@ pub struct OutputDir { unviable_list: File, /// The accumulated overall lab outcome. pub lab_outcome: LabOutcome, + /// Incrementing sequence numbers for each scenario, so that they can each have a unique + /// filename. + pub scenario_index: usize, + /// Log filenames which have already been used, and the number of times that each + /// basename has been used. + used_log_names: HashMap, } impl OutputDir { @@ -120,7 +127,7 @@ impl OutputDir { if output_dir.exists() { LockFile::acquire_lock(output_dir.as_ref())?; // Now release the lock for a bit while we move the directory. This might be - // slightly racy. + // slightly racy. Maybe we should move the lock outside the directory. let rotated = in_dir.join(ROTATED_NAME); if rotated.exists() { @@ -157,24 +164,54 @@ impl OutputDir { path: output_dir, lab_outcome: LabOutcome::new(), log_dir, - diff_dir, lock_file, missed_list, caught_list, timeout_list, unviable_list, + scenario_index: 0, + used_log_names: HashMap::new(), }) } - /// Create a new log for a given scenario. - /// - /// Returns the [File] to which subprocess output should be sent, and a LogFile to read it - /// later. - pub fn create_log(&self, scenario: &Scenario) -> Result { - LogFile::create_in(&self.log_dir, &scenario.log_file_name_base()) + /// Allocate a sequence number and the output files for a scenario. + pub fn start_scenario(&mut self, scenario: &Scenario) -> Result { + let scenario_name = match scenario { + Scenario::Baseline => "baseline".into(), + Scenario::Mutant(mutant) => mutant.log_file_name_base(), + }; + let basename = match self.used_log_names.entry(scenario_name.clone()) { + Entry::Occupied(mut e) => { + let index = e.get_mut(); + *index += 1; + format!("{scenario_name}_{index:03}") + } + Entry::Vacant(e) => { + e.insert(0); + scenario_name + } + }; + // TODO: Maybe store pathse relative to the output directory; it would be more useful + // if the whole directory is later archived and moved. + let log_file = LogFile::create_in(&self.log_dir, &basename)?; + // TODO: Don't write a diff for the baseline? + let diff_path = Utf8PathBuf::from(format!("diff/{basename}.diff")); + let diff = if let Scenario::Mutant(mutant) = scenario { + // TODO: This calculates the mutated text again, and perhaps we could do it + // only once in the caller. + mutant.diff() + } else { + String::new() + }; + let full_diff_path = self.path().join(&diff_path); + write(&full_diff_path, diff.as_bytes()) + .with_context(|| format!("write diff file {full_diff_path:?}"))?; + Ok(ScenarioOutput { + log_file, + diff_path, + }) } - #[allow(dead_code)] /// Return the path of the `mutants.out` directory. pub fn path(&self) -> &Utf8Path { &self.path @@ -209,23 +246,6 @@ impl OutputDir { Ok(()) } - pub fn write_diff_file(&self, scenario: &Scenario) -> Result { - // TODO: Unify with code to calculate log file names; maybe OutputDir should assign unique - // names? - let diff_filename = self - .diff_dir - .join(format!("{}.diff", scenario.log_file_name_base())); - let diff = if let Scenario::Mutant(mutant) = scenario { - // TODO: Calculate the mutant text only once? - mutant.diff() - } else { - String::new() - }; - write(&diff_filename, diff.as_bytes()) - .with_context(|| format!("write diff file {diff_filename:?}"))?; - Ok(diff_filename) - } - pub fn open_debug_log(&self) -> Result { let debug_log_path = self.path.join("debug.log"); OpenOptions::new() @@ -286,6 +306,20 @@ pub fn load_previously_caught(output_parent_dir: &Utf8Path) -> Result &Utf8Path { + self.log_file.path() + } +} + #[cfg(test)] mod test { use std::fs::write; @@ -366,17 +400,14 @@ mod test { let temp_dir_path = Utf8Path::from_path(temp_dir.path()).unwrap(); // Create an initial output dir with one log. - let output_dir = OutputDir::new(temp_dir_path).unwrap(); - output_dir.create_log(&Scenario::Baseline).unwrap(); - assert!(temp_dir - .path() - .join("mutants.out/log/baseline.log") - .is_file()); + let mut output_dir = OutputDir::new(temp_dir_path).unwrap(); + let _scenario_output = output_dir.start_scenario(&Scenario::Baseline).unwrap(); + assert!(temp_dir_path.join("mutants.out/log/baseline.log").is_file()); drop(output_dir); // release the lock. // The second time we create it in the same directory, the old one is moved away. - let output_dir = OutputDir::new(temp_dir_path).unwrap(); - output_dir.create_log(&Scenario::Baseline).unwrap(); + let mut output_dir = OutputDir::new(temp_dir_path).unwrap(); + output_dir.start_scenario(&Scenario::Baseline).unwrap(); assert!(temp_dir .path() .join("mutants.out.old/log/baseline.log") @@ -388,8 +419,8 @@ mod test { drop(output_dir); // The third time (and later), the .old directory is removed. - let output_dir = OutputDir::new(temp_dir_path).unwrap(); - output_dir.create_log(&Scenario::Baseline).unwrap(); + let mut output_dir = OutputDir::new(temp_dir_path).unwrap(); + output_dir.start_scenario(&Scenario::Baseline).unwrap(); assert!(temp_dir .path() .join("mutants.out/log/baseline.log") diff --git a/src/scenario.rs b/src/scenario.rs index 413fdc8c..ea616d91 100644 --- a/src/scenario.rs +++ b/src/scenario.rs @@ -35,11 +35,4 @@ impl Scenario { Scenario::Mutant(mutant) => Some(mutant), } } - - pub fn log_file_name_base(&self) -> String { - match self { - Scenario::Baseline => "baseline".into(), - Scenario::Mutant(mutant) => mutant.log_file_name_base(), - } - } } diff --git a/tests/main.rs b/tests/main.rs index 25e95255..4f8b67e4 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -94,7 +94,14 @@ fn tree_with_child_directories_is_well_tested() { let mut all_diffs = HashSet::new(); for outcome_json in json["outcomes"].as_array().unwrap() { let diff_path = outcome_json["diff_path"].as_str().unwrap(); - assert!(Utf8Path::new(diff_path).is_file()); + assert!( + tmp_src_dir + .path() + .join("mutants.out") + .join(diff_path) + .is_file(), + "{diff_path:?} is not a file" + ); assert!(all_diffs.insert(diff_path)); } } From 8437f84226b2602c831c9b7cb2819a6545d450fc Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 25 Aug 2024 09:52:01 -0700 Subject: [PATCH 03/10] refactor: merge LogFile into ScenarioOutput Console and TailFile take a File, rather than reopening it. Store diff and log file names relative to the output directory. --- src/cargo.rs | 5 +-- src/console.rs | 10 ++---- src/lab.rs | 20 ++++++----- src/log_file.rs | 75 --------------------------------------- src/main.rs | 2 -- src/mutate.rs | 2 +- src/outcome.rs | 13 ++++--- src/output.rs | 91 ++++++++++++++++++++++++++++++++++++------------ src/process.rs | 16 ++++----- src/tail_file.rs | 11 +++--- tests/main.rs | 1 - 11 files changed, 106 insertions(+), 140 deletions(-) delete mode 100644 src/log_file.rs diff --git a/src/cargo.rs b/src/cargo.rs index b86ec01f..dae57e3e 100644 --- a/src/cargo.rs +++ b/src/cargo.rs @@ -9,6 +9,7 @@ use itertools::Itertools; use tracing::{debug, debug_span, warn}; use crate::outcome::PhaseResult; +use crate::output::ScenarioOutput; use crate::package::Package; use crate::process::{Process, ProcessStatus}; use crate::*; @@ -21,7 +22,7 @@ pub fn run_cargo( packages: Option<&[&Package]>, phase: Phase, timeout: Option, - log_file: &mut LogFile, + scenario_output: &mut ScenarioOutput, options: &Options, console: &Console, ) -> Result { @@ -45,7 +46,7 @@ pub fn run_cargo( build_dir.path(), timeout, jobserver, - log_file, + scenario_output, console, )?; check_interrupted()?; diff --git a/src/console.rs b/src/console.rs index 0226f892..d8ede0b1 100644 --- a/src/console.rs +++ b/src/console.rs @@ -11,7 +11,6 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use anyhow::Context; -use camino::Utf8Path; use console::{style, StyledObject}; use humantime::format_duration; use nutmeg::Destination; @@ -71,12 +70,7 @@ impl Console { } /// Update that a cargo task is starting. - pub fn scenario_started( - &self, - dir: &Path, - scenario: &Scenario, - log_file: &Utf8Path, - ) -> Result<()> { + pub fn scenario_started(&self, dir: &Path, scenario: &Scenario, log_file: File) -> Result<()> { let start = Instant::now(); let scenario_model = ScenarioModel::new(dir, scenario, start, log_file)?; self.view.update(|model| { @@ -502,7 +496,7 @@ impl ScenarioModel { dir: &Path, scenario: &Scenario, start: Instant, - log_file: &Utf8Path, + log_file: File, ) -> Result { let log_tail = TailFile::new(log_file).context("Failed to open log file")?; Ok(ScenarioModel { diff --git a/src/lab.rs b/src/lab.rs index 95444c92..f24bfad1 100644 --- a/src/lab.rs +++ b/src/lab.rs @@ -11,7 +11,6 @@ use std::thread; use std::time::Instant; use itertools::Itertools; -use output::ScenarioOutput; use tracing::{debug, debug_span, error, trace, warn}; use crate::cargo::run_cargo; @@ -210,13 +209,16 @@ fn test_scenario( options: &Options, console: &Console, ) -> Result { - let scenario_output = output_mutex + let mut scenario_output = output_mutex .lock() .expect("lock output_dir to start scenario") .start_scenario(scenario)?; - let ScenarioOutput { mut log_file, .. } = scenario_output; - log_file.message(&scenario.to_string()); - console.scenario_started(build_dir.path().as_ref(), scenario, log_file.path())?; + scenario_output.message(&scenario.to_string())?; + console.scenario_started( + build_dir.path().as_ref(), + scenario, + scenario_output.open_log_read()?, + )?; let phases: &[Phase] = if options.check_only { &[Phase::Check] @@ -228,14 +230,14 @@ fn test_scenario( .map(|mutant| { // TODO: This is slightly inefficient as it computes the mutated source twice, // once for the diff and once to write it out. - log_file.message(&format!("mutation diff:\n{}", mutant.diff())); + scenario_output.message(&format!("mutation diff:\n{}", mutant.diff()))?; mutant.apply(build_dir) }) .transpose()?; let dir: &Path = build_dir.path().as_ref(); - console.scenario_started(dir, scenario, log_file.path())?; + console.scenario_started(dir, scenario, scenario_output.open_log_read()?)?; - let mut outcome = ScenarioOutcome::new(&log_file, &scenario_output.diff_path, scenario.clone()); + let mut outcome = ScenarioOutcome::new(&scenario_output, scenario.clone()); for &phase in phases { console.scenario_phase_started(dir, phase); let timeout = match phase { @@ -248,7 +250,7 @@ fn test_scenario( Some(test_packages), phase, timeout, - &mut log_file, + &mut scenario_output, options, console, )?; diff --git a/src/log_file.rs b/src/log_file.rs deleted file mode 100644 index f078d36d..00000000 --- a/src/log_file.rs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2021-2023 Martin Pool - -//! Manage per-scenario log files, which contain the output from cargo -//! and test cases, mixed with commentary from cargo-mutants. - -use std::fs::{File, OpenOptions}; -use std::io::Write; - -use anyhow::Context; -use camino::{Utf8Path, Utf8PathBuf}; - -use crate::Result; - -/// Text inserted in log files to make important sections more visible. -pub const LOG_MARKER: &str = "***"; - -/// A log file for execution of a single scenario. -#[derive(Debug)] -pub struct LogFile { - path: Utf8PathBuf, - write_to: File, -} - -impl LogFile { - pub fn create_in(log_dir: &Utf8Path, basename: &str) -> Result { - let path = log_dir.join(format!("{basename}.log")); - let write_to = OpenOptions::new() - .create_new(true) - .read(true) - .append(true) - .open(&path) - .with_context(|| format!("create test log file {path:?}"))?; - Ok(LogFile { path, write_to }) - } - - /// Open the log file to append more content. - pub fn open_append(&self) -> Result { - OpenOptions::new() - .append(true) - .open(&self.path) - .with_context(|| format!("open {} for append", self.path)) - } - - /// Write a message, with a marker. Ignore errors. - pub fn message(&mut self, message: &str) { - write!(self.write_to, "\n{LOG_MARKER} {message}\n").expect("write message to log"); - } - - pub fn path(&self) -> &Utf8Path { - &self.path - } -} - -pub fn clean_filename(s: &str) -> String { - s.replace('/', "__") - .chars() - .map(|c| match c { - '\\' | ' ' | ':' | '<' | '>' | '?' | '*' | '|' | '"' => '_', - c => c, - }) - .collect::() -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn clean_filename_removes_special_characters() { - assert_eq!( - clean_filename("1/2\\3:4<5>6?7*8|9\"0"), - "1__2_3_4_5_6_7_8_9_0" - ); - } -} diff --git a/src/main.rs b/src/main.rs index f0843c93..ce5123af 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,6 @@ mod in_diff; mod interrupt; mod lab; mod list; -mod log_file; mod manifest; mod mutate; mod options; @@ -56,7 +55,6 @@ use crate::in_diff::diff_filter; use crate::interrupt::check_interrupted; use crate::lab::test_mutants; use crate::list::{list_files, list_mutants, FmtToIoWrite}; -use crate::log_file::LogFile; use crate::manifest::fix_manifest; use crate::mutate::{Genre, Mutant}; use crate::options::{Colors, Options, TestTool}; diff --git a/src/mutate.rs b/src/mutate.rs index 93a3bdd1..177d19fd 100644 --- a/src/mutate.rs +++ b/src/mutate.rs @@ -15,7 +15,7 @@ use tracing::error; use tracing::trace; use crate::build_dir::BuildDir; -use crate::log_file::clean_filename; +use crate::output::clean_filename; use crate::package::Package; use crate::source::SourceFile; use crate::span::Span; diff --git a/src/outcome.rs b/src/outcome.rs index 9d274a89..ecee7af6 100644 --- a/src/outcome.rs +++ b/src/outcome.rs @@ -3,11 +3,11 @@ //! The outcome of running a single mutation scenario, or a whole lab. use std::fmt; -use std::fs; use std::time::Duration; use std::time::Instant; use humantime::format_duration; +use output::ScenarioOutput; use serde::ser::SerializeStruct; use serde::Serialize; use serde::Serializer; @@ -143,6 +143,7 @@ impl LabOutcome { pub struct ScenarioOutcome { /// A file holding the text output from running this test. // TODO: Maybe this should be a log object? + output_dir: Utf8PathBuf, log_path: Utf8PathBuf, diff_path: Utf8PathBuf, /// What kind of scenario was being built? @@ -168,10 +169,11 @@ impl Serialize for ScenarioOutcome { } impl ScenarioOutcome { - pub fn new(log_file: &LogFile, diff_path: &Utf8Path, scenario: Scenario) -> ScenarioOutcome { + pub fn new(scenario_output: &ScenarioOutput, scenario: Scenario) -> ScenarioOutcome { ScenarioOutcome { - log_path: log_file.path().to_owned(), - diff_path: diff_path.to_owned(), + output_dir: scenario_output.output_dir.to_owned(), + log_path: scenario_output.log_path().to_owned(), + diff_path: scenario_output.diff_path.to_owned(), scenario, phase_results: Vec::new(), } @@ -182,7 +184,7 @@ impl ScenarioOutcome { } pub fn get_log_content(&self) -> Result { - fs::read_to_string(&self.log_path).context("read log file") + read_to_string(self.output_dir.join(&self.log_path)).context("read log file") } pub fn last_phase(&self) -> Phase { @@ -327,6 +329,7 @@ mod test { #[test] fn find_phase_result() { let outcome = ScenarioOutcome { + output_dir: "output".into(), log_path: "log".into(), diff_path: "mutant.diff".into(), scenario: Scenario::Baseline, diff --git a/src/output.rs b/src/output.rs index e047f1aa..c7e0b5b7 100644 --- a/src/output.rs +++ b/src/output.rs @@ -88,7 +88,7 @@ impl LockFile { #[derive(Debug)] pub struct OutputDir { path: Utf8PathBuf, - log_dir: Utf8PathBuf, + #[allow(unused)] // Lifetime controls the file lock lock_file: File, /// A file holding a list of missed mutants as text, one per line. @@ -100,9 +100,6 @@ pub struct OutputDir { unviable_list: File, /// The accumulated overall lab outcome. pub lab_outcome: LabOutcome, - /// Incrementing sequence numbers for each scenario, so that they can each have a unique - /// filename. - pub scenario_index: usize, /// Log filenames which have already been used, and the number of times that each /// basename has been used. used_log_names: HashMap, @@ -163,13 +160,11 @@ impl OutputDir { Ok(OutputDir { path: output_dir, lab_outcome: LabOutcome::new(), - log_dir, lock_file, missed_list, caught_list, timeout_list, unviable_list, - scenario_index: 0, used_log_names: HashMap::new(), }) } @@ -191,11 +186,6 @@ impl OutputDir { scenario_name } }; - // TODO: Maybe store pathse relative to the output directory; it would be more useful - // if the whole directory is later archived and moved. - let log_file = LogFile::create_in(&self.log_dir, &basename)?; - // TODO: Don't write a diff for the baseline? - let diff_path = Utf8PathBuf::from(format!("diff/{basename}.diff")); let diff = if let Scenario::Mutant(mutant) = scenario { // TODO: This calculates the mutated text again, and perhaps we could do it // only once in the caller. @@ -203,16 +193,11 @@ impl OutputDir { } else { String::new() }; - let full_diff_path = self.path().join(&diff_path); - write(&full_diff_path, diff.as_bytes()) - .with_context(|| format!("write diff file {full_diff_path:?}"))?; - Ok(ScenarioOutput { - log_file, - diff_path, - }) + ScenarioOutput::new(&self.path, &basename, &diff) } /// Return the path of the `mutants.out` directory. + #[allow(unused)] pub fn path(&self) -> &Utf8Path { &self.path } @@ -309,17 +294,70 @@ pub fn load_previously_caught(output_parent_dir: &Utf8Path) -> Result &Utf8Path { - self.log_file.path() + fn new(output_dir: &Utf8Path, basename: &str, diff: &str) -> Result { + let log_path = Utf8PathBuf::from(format!("log/{basename}.log")); + let log_file = File::options() + .append(true) + .create_new(true) + .read(true) + .open(output_dir.join(&log_path))?; + let diff_path = Utf8PathBuf::from(format!("diff/{basename}.diff")); + write(output_dir.join(&diff_path), diff.as_bytes()) + .with_context(|| format!("write {diff_path}"))?; + Ok(Self { + output_dir: output_dir.to_owned(), + log_path, + log_file, + diff_path, + }) + } + + pub fn log_path(&self) -> &Utf8Path { + &self.log_path + } + + /// Open a new handle reading from the start of the log file. + pub fn open_log_read(&self) -> Result { + let path = self.output_dir.join(&self.log_path); + OpenOptions::new() + .read(true) + .open(&path) + .with_context(|| format!("reopen {} for read", path)) + } + + /// Open a new handle that appends to the log file, so that it can be passed to a subprocess. + pub fn open_log_append(&self) -> Result { + let path = self.output_dir.join(&self.log_path); + OpenOptions::new() + .append(true) + .open(&path) + .with_context(|| format!("reopen {} for append", path)) + } + + /// Write a message, with a marker. + pub fn message(&mut self, message: &str) -> Result<()> { + write!(self.log_file, "\n*** {message}\n").context("write message to log") } } +pub fn clean_filename(s: &str) -> String { + s.replace('/', "__") + .chars() + .map(|c| match c { + '\\' | ' ' | ':' | '<' | '>' | '?' | '*' | '|' | '"' => '_', + c => c, + }) + .collect::() +} + #[cfg(test)] mod test { use std::fs::write; @@ -366,6 +404,14 @@ mod test { .collect_vec() } + #[test] + fn clean_filename_removes_special_characters() { + assert_eq!( + clean_filename("1/2\\3:4<5>6?7*8|9\"0"), + "1__2_3_4_5_6_7_8_9_0" + ); + } + #[test] fn create_output_dir() { let tmp = minimal_source_tree(); @@ -390,7 +436,6 @@ mod test { ] ); assert_eq!(output_dir.path(), workspace.dir.join("mutants.out")); - assert_eq!(output_dir.log_dir, workspace.dir.join("mutants.out/log")); assert!(output_dir.path().join("lock.json").is_file()); } diff --git a/src/process.rs b/src/process.rs index 27b6ceee..e62d4d26 100644 --- a/src/process.rs +++ b/src/process.rs @@ -21,7 +21,7 @@ use tracing::{debug, debug_span, error, span, trace, warn, Level}; use crate::console::Console; use crate::interrupt::check_interrupted; -use crate::log_file::LogFile; +use crate::output::ScenarioOutput; use crate::Result; /// How frequently to check if a subprocess finished. @@ -42,10 +42,10 @@ impl Process { cwd: &Utf8Path, timeout: Option, jobserver: &Option, - log_file: &mut LogFile, + scenario_output: &mut ScenarioOutput, console: &Console, ) -> Result { - let mut child = Process::start(argv, env, cwd, timeout, jobserver, log_file)?; + let mut child = Process::start(argv, env, cwd, timeout, jobserver, scenario_output)?; let process_status = loop { if let Some(exit_status) = child.poll()? { break exit_status; @@ -54,7 +54,7 @@ impl Process { sleep(WAIT_POLL_INTERVAL); } }; - log_file.message(&format!("result: {process_status:?}")); + scenario_output.message(&format!("result: {process_status:?}"))?; Ok(process_status) } @@ -65,11 +65,11 @@ impl Process { cwd: &Utf8Path, timeout: Option, jobserver: &Option, - log_file: &mut LogFile, + scenario_output: &mut ScenarioOutput, ) -> Result { let start = Instant::now(); let quoted_argv = cheap_shell_quote(argv); - log_file.message("ed_argv); + scenario_output.message("ed_argv)?; debug!(%quoted_argv, "start process"); let os_env = env.iter().map(|(k, v)| (OsStr::new(k), OsStr::new(v))); let mut child = Command::new(&argv[0]); @@ -77,8 +77,8 @@ impl Process { .args(&argv[1..]) .envs(os_env) .stdin(Stdio::null()) - .stdout(log_file.open_append()?) - .stderr(log_file.open_append()?) + .stdout(scenario_output.open_log_append()?) + .stderr(scenario_output.open_log_append()?) .current_dir(cwd); jobserver.as_ref().map(|js| js.configure(&mut child)); #[cfg(unix)] diff --git a/src/tail_file.rs b/src/tail_file.rs index 8ba3efb8..d459da4d 100644 --- a/src/tail_file.rs +++ b/src/tail_file.rs @@ -1,10 +1,9 @@ -// Copyright 2021-2023 Martin Pool +// Copyright 2021-2024 Martin Pool //! Tail a log file: watch for new writes and return the last line. use std::fs::File; use std::io::Read; -use std::path::Path; use anyhow::Context; @@ -18,9 +17,8 @@ pub struct TailFile { } impl TailFile { - /// Watch the given path. - pub fn new>(path: P) -> Result { - let file = File::open(path.as_ref()).context("Open log file")?; + /// Watch for newly appended data in a file. + pub fn new(file: File) -> Result { Ok(TailFile { file, last_line_seen: String::new(), @@ -66,7 +64,8 @@ mod test { fn last_line_of_file() { let mut tempfile = tempfile::NamedTempFile::new().unwrap(); let path: Utf8PathBuf = tempfile.path().to_owned().try_into().unwrap(); - let mut tailer = TailFile::new(path).unwrap(); + let reopened = File::open(&path).unwrap(); + let mut tailer = TailFile::new(reopened).unwrap(); assert_eq!( tailer.last_line().unwrap(), diff --git a/tests/main.rs b/tests/main.rs index 4f8b67e4..65137389 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -9,7 +9,6 @@ use std::path::Path; use std::thread::sleep; use std::time::Duration; -use camino::Utf8Path; use indoc::indoc; use itertools::Itertools; use predicate::str::{contains, is_match}; From 616bc19be1766dfef14955fe2736e37041afe469 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 4 Oct 2024 08:38:40 -0700 Subject: [PATCH 04/10] comments --- src/output.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/output.rs b/src/output.rs index c7e0b5b7..bf3532f5 100644 --- a/src/output.rs +++ b/src/output.rs @@ -124,7 +124,8 @@ impl OutputDir { if output_dir.exists() { LockFile::acquire_lock(output_dir.as_ref())?; // Now release the lock for a bit while we move the directory. This might be - // slightly racy. Maybe we should move the lock outside the directory. + // slightly racy. + // TODO: Move the lock outside the directory, . let rotated = in_dir.join(ROTATED_NAME); if rotated.exists() { @@ -292,7 +293,6 @@ pub fn load_previously_caught(output_parent_dir: &Utf8Path) -> Result Date: Fri, 4 Oct 2024 08:51:10 -0700 Subject: [PATCH 05/10] Check that mutant diff files look right --- tests/main.rs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/main.rs b/tests/main.rs index 65137389..13690952 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -92,16 +92,24 @@ fn tree_with_child_directories_is_well_tested() { let json: serde_json::Value = serde_json::from_str(&outcomes_json).unwrap(); let mut all_diffs = HashSet::new(); for outcome_json in json["outcomes"].as_array().unwrap() { + dbg!(&outcome_json); let diff_path = outcome_json["diff_path"].as_str().unwrap(); - assert!( - tmp_src_dir - .path() - .join("mutants.out") - .join(diff_path) - .is_file(), - "{diff_path:?} is not a file" - ); + let full_diff_path = tmp_src_dir.path().join("mutants.out").join(diff_path); + assert!(full_diff_path.is_file(), "{diff_path:?} is not a file"); assert!(all_diffs.insert(diff_path)); + let diff_content = read_to_string(&full_diff_path).expect("read diff file"); + if outcome_json["scenario"].as_str() == Some("Baseline") { + assert_eq!(diff_path, "diff/baseline.diff"); + assert!( + diff_content.is_empty(), + "baseline diff in {full_diff_path:?} should be empty" + ); + } else { + assert!( + diff_content.starts_with("--- src/"), + "diff content in {full_diff_path:?} doesn't look right:\n{diff_content}" + ); + } } } From 44f6b91b70a37ef499dd9d8fc05d6b4c7afbee2b Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sat, 5 Oct 2024 08:29:13 -0700 Subject: [PATCH 06/10] Fix win tests? Drop open log file to allow the output directory to be rotated. --- src/output.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/output.rs b/src/output.rs index bf3532f5..7b58d900 100644 --- a/src/output.rs +++ b/src/output.rs @@ -446,9 +446,10 @@ mod test { // Create an initial output dir with one log. let mut output_dir = OutputDir::new(temp_dir_path).unwrap(); - let _scenario_output = output_dir.start_scenario(&Scenario::Baseline).unwrap(); + let scenario_output = output_dir.start_scenario(&Scenario::Baseline).unwrap(); assert!(temp_dir_path.join("mutants.out/log/baseline.log").is_file()); drop(output_dir); // release the lock. + drop(scenario_output); // The second time we create it in the same directory, the old one is moved away. let mut output_dir = OutputDir::new(temp_dir_path).unwrap(); From f189c9bea16e239678d3f8f07c0a1c998424bb96 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sat, 5 Oct 2024 09:19:47 -0700 Subject: [PATCH 07/10] Generate mutant text and diff only once It's probably not material but it seems a bit wasteful to compute this repeatedly for every mutant. Just manually revert the mutant rather than using a RAII lifetime. Use Camino Utf8Path more broadly. --- NEWS.md | 2 ++ src/build_dir.rs | 9 ++++++++ src/console.rs | 33 ++++++++++++++++------------- src/copy_tree.rs | 5 ++++- src/lab.rs | 54 +++++++++++++++++++++++++----------------------- src/list.rs | 9 ++++---- src/mutate.rs | 54 ++++++++++++++---------------------------------- src/outcome.rs | 6 ++++-- src/output.rs | 36 ++++++++++++++++++-------------- tests/main.rs | 18 +++++++++------- 10 files changed, 116 insertions(+), 110 deletions(-) diff --git a/NEWS.md b/NEWS.md index e4122ef2..3871b2e5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,8 @@ - New: Mutate `proc_macro` targets and functions. +- New: Write diffs to dedicated files under `mutants.out/diff/`. The filename is included in the mutant json output. + ## 24.9.0 - Fixed: Avoid generating empty string elements in `ENCODED_RUSTFLAGS` when `--cap-lints` is set. In some situations these could cause a compiler error complaining about the empty argument. diff --git a/src/build_dir.rs b/src/build_dir.rs index d2132bf5..8be3e757 100644 --- a/src/build_dir.rs +++ b/src/build_dir.rs @@ -3,6 +3,7 @@ //! A directory containing mutated source to run cargo builds and tests. use std::fmt::{self, Debug}; +use std::fs::write; use tempfile::TempDir; use tracing::info; @@ -77,6 +78,14 @@ impl BuildDir { pub fn path(&self) -> &Utf8Path { self.path.as_path() } + + pub fn overwrite_file(&self, relative_path: &Utf8Path, code: &str) -> Result<()> { + let full_path = self.path.join(relative_path); + // for safety, don't follow symlinks + ensure!(full_path.is_file(), "{full_path:?} is not a file"); + write(&full_path, code.as_bytes()) + .with_context(|| format!("failed to write code to {full_path:?}")) + } } #[cfg(test)] diff --git a/src/console.rs b/src/console.rs index d8ede0b1..10fa10c7 100644 --- a/src/console.rs +++ b/src/console.rs @@ -6,11 +6,11 @@ use std::borrow::Cow; use std::fmt::Write; use std::fs::File; use std::io; -use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use anyhow::Context; +use camino::{Utf8Path, Utf8PathBuf}; use console::{style, StyledObject}; use humantime::format_duration; use nutmeg::Destination; @@ -70,7 +70,12 @@ impl Console { } /// Update that a cargo task is starting. - pub fn scenario_started(&self, dir: &Path, scenario: &Scenario, log_file: File) -> Result<()> { + pub fn scenario_started( + &self, + dir: &Utf8Path, + scenario: &Scenario, + log_file: File, + ) -> Result<()> { let start = Instant::now(); let scenario_model = ScenarioModel::new(dir, scenario, start, log_file)?; self.view.update(|model| { @@ -82,7 +87,7 @@ impl Console { /// Update that cargo finished. pub fn scenario_finished( &self, - dir: &Path, + dir: &Utf8Path, scenario: &Scenario, outcome: &ScenarioOutcome, options: &Options, @@ -143,13 +148,13 @@ impl Console { self.message(&s); } - pub fn start_copy(&self, dir: &Path) { + pub fn start_copy(&self, dir: &Utf8Path) { self.view.update(|model| { model.copy_models.push(CopyModel::new(dir.to_owned())); }); } - pub fn finish_copy(&self, dir: &Path) { + pub fn finish_copy(&self, dir: &Utf8Path) { self.view.update(|model| { let idx = model .copy_models @@ -160,7 +165,7 @@ impl Console { }); } - pub fn copy_progress(&self, dest: &Path, total_bytes: u64) { + pub fn copy_progress(&self, dest: &Utf8Path, total_bytes: u64) { self.view.update(|model| { model .copy_models @@ -191,13 +196,13 @@ impl Console { } /// A new phase of this scenario started. - pub fn scenario_phase_started(&self, dir: &Path, phase: Phase) { + pub fn scenario_phase_started(&self, dir: &Utf8Path, phase: Phase) { self.view.update(|model| { model.find_scenario_mut(dir).phase_started(phase); }) } - pub fn scenario_phase_finished(&self, dir: &Path, phase: Phase) { + pub fn scenario_phase_finished(&self, dir: &Utf8Path, phase: Phase) { self.view.update(|model| { model.find_scenario_mut(dir).phase_finished(phase); }) @@ -445,14 +450,14 @@ impl nutmeg::Model for LabModel { } impl LabModel { - fn find_scenario_mut(&mut self, dir: &Path) -> &mut ScenarioModel { + fn find_scenario_mut(&mut self, dir: &Utf8Path) -> &mut ScenarioModel { self.scenario_models .iter_mut() .find(|sm| sm.dir == *dir) .expect("scenario directory not found") } - fn remove_scenario(&mut self, dir: &Path) { + fn remove_scenario(&mut self, dir: &Utf8Path) { self.scenario_models.retain(|sm| sm.dir != *dir); } } @@ -482,7 +487,7 @@ impl nutmeg::Model for WalkModel { /// It draws the command and some description of what scenario is being tested. struct ScenarioModel { /// The directory where this is being built: unique across all models. - dir: PathBuf, + dir: Utf8PathBuf, name: Cow<'static, str>, phase_start: Instant, phase: Option, @@ -493,7 +498,7 @@ struct ScenarioModel { impl ScenarioModel { fn new( - dir: &Path, + dir: &Utf8Path, scenario: &Scenario, start: Instant, log_file: File, @@ -550,13 +555,13 @@ impl nutmeg::Model for ScenarioModel { /// A Nutmeg model for progress in copying a tree. struct CopyModel { - dest: PathBuf, + dest: Utf8PathBuf, bytes_copied: u64, start: Instant, } impl CopyModel { - fn new(dest: PathBuf) -> CopyModel { + fn new(dest: Utf8PathBuf) -> CopyModel { CopyModel { dest, start: Instant::now(), diff --git a/src/copy_tree.rs b/src/copy_tree.rs index e1e7c29c..ea759fd1 100644 --- a/src/copy_tree.rs +++ b/src/copy_tree.rs @@ -46,7 +46,10 @@ pub fn copy_tree( .suffix(".tmp") .tempdir() .context("create temp dir")?; - let dest = temp_dir.path(); + let dest = temp_dir + .path() + .try_into() + .context("Convert path to UTF-8")?; console.start_copy(dest); for entry in WalkBuilder::new(from_path) .standard_filters(gitignore) diff --git a/src/lab.rs b/src/lab.rs index f24bfad1..e7e6c2ab 100644 --- a/src/lab.rs +++ b/src/lab.rs @@ -5,7 +5,6 @@ use std::cmp::{max, min}; use std::panic::resume_unwind; -use std::path::Path; use std::sync::Mutex; use std::thread; use std::time::Instant; @@ -213,29 +212,20 @@ fn test_scenario( .lock() .expect("lock output_dir to start scenario") .start_scenario(scenario)?; - scenario_output.message(&scenario.to_string())?; - console.scenario_started( - build_dir.path().as_ref(), - scenario, - scenario_output.open_log_read()?, - )?; + let dir = build_dir.path(); + console.scenario_started(dir, scenario, scenario_output.open_log_read()?)?; let phases: &[Phase] = if options.check_only { &[Phase::Check] } else { &[Phase::Build, Phase::Test] }; - let applied = scenario - .mutant() - .map(|mutant| { - // TODO: This is slightly inefficient as it computes the mutated source twice, - // once for the diff and once to write it out. - scenario_output.message(&format!("mutation diff:\n{}", mutant.diff()))?; - mutant.apply(build_dir) - }) - .transpose()?; - let dir: &Path = build_dir.path().as_ref(); - console.scenario_started(dir, scenario, scenario_output.open_log_read()?)?; + if let Some(mutant) = scenario.mutant() { + let mutated_code = mutant.mutated_code(); + let diff = scenario.mutant().unwrap().diff(&mutated_code); + scenario_output.write_diff(&diff)?; + mutant.apply(build_dir, &mutated_code)?; + } let mut outcome = ScenarioOutcome::new(&scenario_output, scenario.clone()); for &phase in phases { @@ -244,7 +234,7 @@ fn test_scenario( Phase::Test => timeouts.test, Phase::Build | Phase::Check => timeouts.build, }; - let phase_result = run_cargo( + match run_cargo( build_dir, jobserver, Some(test_packages), @@ -253,15 +243,27 @@ fn test_scenario( &mut scenario_output, options, console, - )?; - let success = phase_result.is_success(); // so we can move it away - outcome.add_phase_result(phase_result); - console.scenario_phase_finished(dir, phase); - if !success { - break; + ) { + Ok(phase_result) => { + let success = phase_result.is_success(); // so we can move it away + outcome.add_phase_result(phase_result); + console.scenario_phase_finished(dir, phase); + if !success { + break; + } + } + Err(err) => { + // Some unexpected internal error that stops the program. + if let Some(mutant) = scenario.mutant() { + mutant.revert(build_dir)?; + return Err(err); + } + } } } - drop(applied); + if let Some(mutant) = scenario.mutant() { + mutant.revert(build_dir)?; + } output_mutex .lock() .expect("lock output dir to add outcome") diff --git a/src/list.rs b/src/list.rs index a3045040..bf1a6397 100644 --- a/src/list.rs +++ b/src/list.rs @@ -37,9 +37,10 @@ pub(crate) fn list_mutants( for mutant in mutants { let mut obj = serde_json::to_value(mutant)?; if options.emit_diffs { - obj.as_object_mut() - .unwrap() - .insert("diff".to_owned(), json!(mutant.diff())); + obj.as_object_mut().unwrap().insert( + "diff".to_owned(), + json!(mutant.diff(&mutant.mutated_code())), + ); } list.push(obj); } @@ -51,7 +52,7 @@ pub(crate) fn list_mutants( for mutant in mutants { writeln!(out, "{}", mutant.name(options.show_line_col, colors))?; if options.emit_diffs { - writeln!(out, "{}", mutant.diff())?; + writeln!(out, "{}", mutant.diff(&mutant.mutated_code()))?; } } } diff --git a/src/mutate.rs b/src/mutate.rs index 177d19fd..f5a69b50 100644 --- a/src/mutate.rs +++ b/src/mutate.rs @@ -1,17 +1,15 @@ -// Copyright 2021-2023 Martin Pool +// Copyright 2021-2024 Martin Pool //! Mutations to source files, and inference of interesting mutations to apply. use std::fmt; -use std::fs; use std::sync::Arc; -use anyhow::{ensure, Context, Result}; +use anyhow::Result; use console::{style, StyledObject}; use serde::ser::{SerializeStruct, Serializer}; use serde::Serialize; use similar::TextDiff; -use tracing::error; use tracing::trace; use crate::build_dir::BuildDir; @@ -172,11 +170,14 @@ impl Mutant { } /// Return a unified diff for the mutant. - pub fn diff(&self) -> String { + /// + /// The mutated text must be passed in because we should have already computed + /// it, and don't want to pointlessly recompute it here. + pub fn diff(&self, mutated_code: &str) -> String { let old_label = self.source_file.tree_relative_slashes(); // There shouldn't be any newlines, but just in case... let new_label = self.describe_change().replace('\n', " "); - TextDiff::from_lines(self.source_file.code(), &self.mutated_code()) + TextDiff::from_lines(self.source_file.code(), mutated_code) .unified_diff() .context_radius(8) .header(&old_label, &new_label) @@ -184,26 +185,17 @@ impl Mutant { } /// Apply this mutant to the relevant file within a BuildDir. - pub fn apply<'a>(&'a self, build_dir: &'a BuildDir) -> Result { + pub fn apply(&self, build_dir: &BuildDir, mutated_code: &str) -> Result<()> { trace!(?self, "Apply mutant"); - self.write_in_dir(build_dir, &self.mutated_code())?; - Ok(AppliedMutant { - mutant: self, - build_dir, - }) - } - - fn unapply(&self, build_dir: &BuildDir) -> Result<()> { - trace!(?self, "Unapply mutant"); - self.write_in_dir(build_dir, self.source_file.code()) + build_dir.overwrite_file(&self.source_file.tree_relative_path, mutated_code) } - fn write_in_dir(&self, build_dir: &BuildDir, code: &str) -> Result<()> { - let path = build_dir.path().join(&self.source_file.tree_relative_path); - // for safety, don't follow symlinks - ensure!(path.is_file(), "{path:?} is not a file"); - fs::write(&path, code.as_bytes()) - .with_context(|| format!("failed to write mutated code to {path:?}")) + pub fn revert(&self, build_dir: &BuildDir) -> Result<()> { + trace!(?self, "Revert mutant"); + build_dir.overwrite_file( + &self.source_file.tree_relative_path, + self.source_file.code(), + ) } /// Return a string describing this mutant that's suitable for building a log file name, @@ -250,22 +242,6 @@ impl Serialize for Mutant { } } -/// Manages the lifetime of a mutant being applied to a build directory; when -/// dropped, the mutant is unapplied. -#[must_use] -pub struct AppliedMutant<'a> { - mutant: &'a Mutant, - build_dir: &'a BuildDir, -} - -impl Drop for AppliedMutant<'_> { - fn drop(&mut self) { - if let Err(e) = self.mutant.unapply(self.build_dir) { - error!("Failed to unapply mutant: {}", e); - } - } -} - #[cfg(test)] mod test { use indoc::indoc; diff --git a/src/outcome.rs b/src/outcome.rs index ecee7af6..72413032 100644 --- a/src/outcome.rs +++ b/src/outcome.rs @@ -145,7 +145,9 @@ pub struct ScenarioOutcome { // TODO: Maybe this should be a log object? output_dir: Utf8PathBuf, log_path: Utf8PathBuf, - diff_path: Utf8PathBuf, + /// The path relative to `mutants.out` for a file showing the diff between the unmutated + /// and mutated source. Only present for mutant scenarios. + diff_path: Option, /// What kind of scenario was being built? pub scenario: Scenario, /// For each phase, the duration and the cargo result. @@ -331,7 +333,7 @@ mod test { let outcome = ScenarioOutcome { output_dir: "output".into(), log_path: "log".into(), - diff_path: "mutant.diff".into(), + diff_path: Some("mutant.diff".into()), scenario: Scenario::Baseline, phase_results: vec![ PhaseResult { diff --git a/src/output.rs b/src/output.rs index 7b58d900..e7d9deab 100644 --- a/src/output.rs +++ b/src/output.rs @@ -187,14 +187,7 @@ impl OutputDir { scenario_name } }; - let diff = if let Scenario::Mutant(mutant) = scenario { - // TODO: This calculates the mutated text again, and perhaps we could do it - // only once in the caller. - mutant.diff() - } else { - String::new() - }; - ScenarioOutput::new(&self.path, &basename, &diff) + ScenarioOutput::new(&self.path, scenario, &basename) } /// Return the path of the `mutants.out` directory. @@ -297,33 +290,44 @@ pub struct ScenarioOutput { pub output_dir: Utf8PathBuf, log_path: Utf8PathBuf, pub log_file: File, - /// File holding the diff of the mutated file. - pub diff_path: Utf8PathBuf, + /// File holding the diff of the mutated file, only if it's a mutation. + pub diff_path: Option, } impl ScenarioOutput { - fn new(output_dir: &Utf8Path, basename: &str, diff: &str) -> Result { + fn new(output_dir: &Utf8Path, scenario: &Scenario, basename: &str) -> Result { let log_path = Utf8PathBuf::from(format!("log/{basename}.log")); let log_file = File::options() .append(true) .create_new(true) .read(true) .open(output_dir.join(&log_path))?; - let diff_path = Utf8PathBuf::from(format!("diff/{basename}.diff")); - write(output_dir.join(&diff_path), diff.as_bytes()) - .with_context(|| format!("write {diff_path}"))?; - Ok(Self { + let diff_path = if scenario.is_mutant() { + Some(Utf8PathBuf::from(format!("diff/{basename}.diff"))) + } else { + None + }; + let mut scenario_output = Self { output_dir: output_dir.to_owned(), log_path, log_file, diff_path, - }) + }; + scenario_output.message(&scenario.to_string())?; + Ok(scenario_output) } pub fn log_path(&self) -> &Utf8Path { &self.log_path } + pub fn write_diff(&mut self, diff: &str) -> Result<()> { + self.message(&format!("mutation diff:\n{}", diff))?; + let diff_path = self.diff_path.as_ref().expect("should know the diff path"); + write(self.output_dir.join(diff_path), diff.as_bytes()) + .with_context(|| format!("write diff to {diff_path}")) + } + /// Open a new handle reading from the start of the log file. pub fn open_log_read(&self) -> Result { let path = self.output_dir.join(&self.log_path); diff --git a/tests/main.rs b/tests/main.rs index ca052dac..782216bb 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -93,18 +93,20 @@ fn tree_with_child_directories_is_well_tested() { let mut all_diffs = HashSet::new(); for outcome_json in json["outcomes"].as_array().unwrap() { dbg!(&outcome_json); - let diff_path = outcome_json["diff_path"].as_str().unwrap(); - let full_diff_path = tmp_src_dir.path().join("mutants.out").join(diff_path); - assert!(full_diff_path.is_file(), "{diff_path:?} is not a file"); - assert!(all_diffs.insert(diff_path)); - let diff_content = read_to_string(&full_diff_path).expect("read diff file"); if outcome_json["scenario"].as_str() == Some("Baseline") { - assert_eq!(diff_path, "diff/baseline.diff"); assert!( - diff_content.is_empty(), - "baseline diff in {full_diff_path:?} should be empty" + outcome_json + .get("diff_path") + .expect("has a diff_path") + .is_null(), + "diff_path should be null" ); } else { + let diff_path = outcome_json["diff_path"].as_str().unwrap(); + let full_diff_path = tmp_src_dir.path().join("mutants.out").join(diff_path); + assert!(full_diff_path.is_file(), "{diff_path:?} is not a file"); + assert!(all_diffs.insert(diff_path)); + let diff_content = read_to_string(&full_diff_path).expect("read diff file"); assert!( diff_content.starts_with("--- src/"), "diff content in {full_diff_path:?} doesn't look right:\n{diff_content}" From aa5feae6d65a106a736e7b35f6488c9698d304e4 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sat, 5 Oct 2024 09:23:47 -0700 Subject: [PATCH 08/10] Fix bad merge in README.md --- README.md | 1054 ----------------------------------------------------- 1 file changed, 1054 deletions(-) diff --git a/README.md b/README.md index 316d8c05..63ecbcb7 100644 --- a/README.md +++ b/README.md @@ -72,1063 +72,9 @@ This software is provided as-is with no warranty of any kind. See also: -<<<<<<< HEAD -`--no-times`: Don't print elapsed times. - -`--timeout`: Set a fixed timeout for each `cargo test` run, to catch mutations -that cause a hang. By default a timeout is automatically determined. - -`--cargo-arg`: Passes the option argument to `cargo check`, `build`, and `test`. -For example, `--cargo-arg --release`. - -### Filtering files - -Two options (each with short and long names) control which files are mutated: - -`-f GLOB`, `--file GLOB`: Mutate only functions in files matching -the glob. - -`-e GLOB`, `--exclude GLOB`: Exclude files that match the glob. - -These options may be repeated. - -If any `-f` options are given, only source files that match are -considered; otherwise all files are considered. This list is then further -reduced by exclusions. - -If the glob contains `/` (or on Windows, `\`), then it matches against the path from the root of the source -tree. For example, `src/*/*.rs` will exclude all files in subdirectories of `src`. - -If the glob does not contain a path separator, it matches against filenames -in any directory. - -`/` matches the path separator on both Unix and Windows. - -Note that the glob must contain `.rs` (or a matching wildcard) to match -source files with that suffix. For example, `-f network` will match -`src/network/mod.rs` but it will _not_ match `src/network.rs`. - -Files that are excluded are still parsed (and so must be syntactically -valid), and `mod` statements in them are followed to discover other -source files. So, for example, you can exclude `src/main.rs` but still -test mutants in other files referenced by `mod` statements in `main.rs`. - -The results of filters can be previewed with the `--list-files` and `--list` -options. - -Examples: - -* `cargo mutants -f visit.rs -f change.rs` -- test mutants only in files - called `visit.rs` or `change.rs` (in any directory). - -* `cargo mutants -e console.rs` -- test mutants in any file except `console.rs`. - -* `cargo mutants -f src/db/*.rs` -- test mutants in any file in this directory. - -### Filtering functions and mutants - -Two options filter mutants by the full name of the mutant, which includes the -function name, file name, and a description of the change. - -Mutant names are shown by `cargo mutants --list`, and the same command can be -used to preview the effect of filters. - -`-F REGEX`, `--re REGEX`: Only test mutants whose full name matches the given regex. - -`-E REGEX`, `--exclude-re REGEX`: Exclude mutants whose full name matches -the given regex. - -These options may be repeated. - -The regex matches a substring and can be anchored with `^` and `$`. - -The regex syntax is defined by the [`regex`](https://docs.rs/regex/latest/regex/) -crate. - -These filters are applied after filtering by filename, and `--re` is applied before -`--exclude-re`. - -Examples: - -* `-E 'impl Debug'` -- don't test `impl Debug` methods, because coverage of them - might be considered unimportant. - -* `-F 'impl Serialize' -F 'impl Deserialize'` -- test implementations of these - two traits. - -### Passing arguments to `cargo test` - -Command-line options following a `--` delimiter are passed through to -`cargo test`, which can be used for example to exclude doctests (which tend to -be slow to build and run): - -```sh -cargo mutants -- --all-targets -``` - -You can use a second double-dash to pass options through to the test targets: - -```sh -cargo mutants -- -- --test-threads 1 --nocapture -``` - -### Understanding the results - -If tests fail in a clean copy of the tree, there might be an (intermittent) -failure in the source directory, or there might be some problem that stops them -passing when run from a different location, such as a relative `path` in -`Cargo.toml`. Fix this first. - -Otherwise, cargo mutants generates every mutant it can. All mutants fall in to -one of these categories: - -* **caught** — A test failed with this mutant applied. This is a good sign about - test coverage. You can look in `mutants.out/log` to see which tests failed. - -* **missed** — No test failed with this mutation applied, which seems to - indicate a gap in test coverage. Or, it may be that the mutant is - undistinguishable from the correct code. You may wish to add a better test, or - mark that the function should be skipped. - -* **unviable** — The attempted mutation doesn't compile. This is inconclusive about test coverage and - no action is needed, but indicates an opportunity for cargo-mutants to either - generate better mutants, or at least not generate unviable mutants. - -* **timeout** — The mutation caused the test suite to run for a long time, until it was eventually killed. You might want to investigate the cause and potentially mark the function to be skipped. - -By default only missed mutants and timeouts are printed because they're the most actionable. Others can be shown with the `-v` and `-V` options. - -### Skipping functions - -To mark functions so they are not mutated: - -1. Add a Cargo dependency on the [mutants](https://crates.io/crates/mutants) - crate, version "0.0.3" or later. (This must be a regular `dependency` not a - `dev-dependency`, because the annotation will be on non-test code.) - -2. Mark functions with `#[mutants::skip]` or other attributes containing `mutants::skip` (e.g. `#[cfg_attr(test, mutants::skip)]`). - -See `testdata/tree/hang_avoided_by_attr/` for an example. - -The crate is tiny and the attribute has no effect on the compiled code. It only -flags the function for cargo-mutants. - -**Note:** Currently, `cargo-mutants` does not (yet) evaluate attributes like `cfg_attr`, it only looks for the sequence `mutants::skip` in the attribute. - -**Note:** Rust's "inner macro attributes" feature is currently unstable, so `#![mutants::skip]` can't be used at the top of a file in stable Rust. - -### Config file - -cargo-mutants looks for a `.cargo/mutants.toml` file in the root of the source -directory. If a config file exists, the values are appended to the corresponding -command-line arguments. (This may cause problems if you use `--` twice on the -command line to pass arguments to the inner test binary.) - -Configured exclusions may be particularly important when there are modules that -are inherently hard to test, and the project has made a decision to accept lower -test coverage for them. - -The following configuration options are supported: - -```toml -exclude_globs = ["src/main.rs", "src/cache/*.rs"] # same as -e -examine_globs = ["src/important/*.rs"] # same as -f, test *only* these files - -exclude_re = ["impl Debug"] # same as -E -examine_re = ["impl Serialize", "impl Deserialize"] # same as -F, test *only* matches - -additional_cargo_args = ["--all-features"] -additional_cargo_test_args = ["--jobs=1"] -``` - -### Exit codes - -* **0**: Success. No mutants were found that weren't caught by tests. - -* **1**: Usage error: bad command-line arguments etc. - -* **2**: Found some mutants that were not covered by tests. - -* **3**: Some tests timed out: possibly the mutatations caused an infinite loop, - or the timeout is too low. - -* **4**: The tests are already failing or hanging before any mutations are - applied, so no mutations were tested. - -### `mutants.out` - -A `mutants.out` directory is created in the source directory, or whichever directory you specify with `--output`. It contains: - -* A `logs/` directory, with one log file for each mutation plus the baseline - unmutated case. The log contains the diff of the mutation plus the output from - cargo. - -* A `diff/` directory, with one diff file for each mutation. - -* A `lock.json`, on which an [fs2 lock](https://docs.rs/fs2) is held while - cargo-mutants is running, to avoid two tasks trying to write to the same - directory at the same time. The lock contains the start time, cargo-mutants - version, username, and hostname. `lock.json` is left in `mutants.out` when the - run completes, but the lock on it is released. - -* `caught.txt`, `missed.txt`, `timeout.txt`, `unviable.txt`, each listing mutants with the corresponding outcome. - -* A `mutants.json` file describing all the generated mutants. - -* An `outcomes.json` file describing the results of all tests, including the path of the log and diff files. - -### Hangs and timeouts - -Some mutations to the tree can cause the test suite to hang. For example, in -this code, cargo-mutants might try changing `should_stop` to always return -`false`: - -```rust - while !should_stop() { - // something - } -``` - -`cargo mutants` automatically sets a timeout when running tests with mutations -applied, and reports mutations that hit a timeout. The automatic timeout is the greater of -20 seconds, or 5x the time to run tests with no mutations. - -The `CARGO_MUTANTS_MINIMUM_TEST_TIMEOUT` environment variable, measured in seconds, overrides the minimum time. - -You can also set an explicit timeout with the `--timeout` option. In this case -the timeout is also applied to tests run with no mutation. - -The timeout does not apply to `cargo check` or `cargo build`, only `cargo test`. - -When a test times out, you can mark it with `#[mutants::skip]` so that future -`cargo mutants` runs go faster. - -### Parallelism - -The `--jobs` or `-j` option allows to test multiple mutants in parallel, by spawning several Cargo processes. This can give 25-50% performance improvements, depending on the tree under test and the hardware resources available. - -It's common that for some periods of its execution, a single Cargo build or test job can't use all the available CPU cores. Running multiple jobs in parallel makes use of resources that would otherwise be idle. - -However, running many jobs simultaneously may also put high demands on the system's RAM (by running more compile/link/test tasks simultaneously), IO bandwidth, and cooling (by fully using all cores). - -The best setting will depend on many factors including the behavior of your program's test suite, the amount of memory on your system, and your system's behavior under high thermal load. - -The default is currently to run only one job at a time. It's reasonable to set this to any value up to the number of CPU cores. - -`-j 4` may be a good starting point, even if you have many more CPUs. Start there and watch memory and CPU usage, and tune towards a setting where all cores are always utilized without memory usage going too high, and without thermal issues. - -Because tests may be slower with high parallelism, you may see some spurious timeouts, and you may need to set `--timeout` manually to allow enough safety margin. - -### Performance - -Most of the runtime for cargo-mutants is spent in running the program test suite -and in running incremental builds: both are done once per viable mutant. - -So, anything you can do to make the `cargo build` and `cargo test` suite faster -will have a multiplicative effect on `cargo mutants` run time, and of course -will also make normal development more pleasant. - -There's lots of good advice on the web, including . - -Rust doctests are pretty slow, so if you're using them only as testable -documentation and not to assert correctness of the code, you can skip them with -`cargo mutants -- --all-targets`. - -On _some but not all_ projects, cargo-mutants can be faster if you use `-C --release`, which will make the build slower but may make the tests faster. Typically this will help on projects with very long CPU-intensive test suites. Cargo-mutants now shows the breakdown of build versus test time which may help you work out if this will help. (On projects like this you might also choose just to turn up optimization for all debug builds in [`.cargo/config.toml`](https://doc.rust-lang.org/cargo/reference/config.html). - -By default cargo-mutants copies the `target/` directory from the source tree. Rust target directories can accumulate excessive volumes of old build products. - -cargo-mutants causes the Rust toolchain (and, often, the program under test) to read and write _many_ temporary files. Setting the temporary directory onto a ramdisk can improve performance significantly. This is particularly important with parallel builds, which might otherwise hit disk bandwidth limits. For example on Linux: - -```shell -sudo mkdir /ram -sudo mount -t tmpfs /ram /ram # or put this in fstab, or just change /tmp -env TMPDIR=/ram cargo mutants -``` - -### Using the Mold linker - -Using the [Mold linker](https://github.com/rui314/mold) on Unix can give a 20% performance improvement, depending on the tree. -Because cargo-mutants does many -incremental builds, link time is important, especially if the test suite is relatively fast. - -Because of limitations in the way cargo-mutants runs Cargo, the standard way of configuring Mold for Rust in `~/.cargo/config.toml` won't work. - -Instead, set the `RUSTFLAGS` environment variable to `-Clink-arg=-fuse-ld=mold`. - -### Workspace and package support - -cargo-mutants now supports testing Cargo workspaces that contain multiple packages. - -All source files in all packages in the workspace are tested. For each mutant, only the containing packages tests are run. - -### Hard-to-test cases - -Some functions don't cause a test suite failure if emptied, but also cannot be -removed. For example, functions to do with managing caches or that have other -performance side effects. - -Ideally, these should be tested, but doing so in a way that's not flaky can be -difficult. cargo-mutants can help in a few ways: - -* It helps to at least highlight to the developer that the function is not - covered by tests, and so should perhaps be treated with extra care, or tested - manually. -* A `#[mutants::skip]` annotation can be added to suppress warnings and explain - the decision. -* Sometimes these effects can be tested by making the side-effect observable - with, for example, a counter of the number of memory allocations or cache - misses/hits. - -### Continuous integration - -Here is an example of a GitHub Actions workflow that runs mutation tests and uploads the results as an artifact. This will fail if it finds any uncaught mutants. - -```yml -name: cargo-mutants - -on: [pull_request, push] - -jobs: - cargo-mutants: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - name: Install cargo-mutants - run: cargo install --locked cargo-mutants - - name: Run mutant tests - run: cargo mutants -- --all-features - - name: Archive results - uses: actions/upload-artifact@v3 - if: failure() - with: - name: mutation-report - path: mutants.out -``` - -## How to help - -Experience reports in [GitHub Discussions](https://github.com/sourcefrog/cargo-mutants/discussions) or issues are very welcome: - -* Did it find a bug or important coverage gap? -* Did it fail to build and test your tree? (Some cases that aren't supported yet - are already listed in this doc or the bug tracker.) - -It's especially helpful if you can either point to an open source tree that will -reproduce the problem (or success) or at least describe how to reproduce it. - -If you are interested in contributing a patch, please read [CONTRIBUTING.md](CONTRIBUTING.md). - -## Goals - -**The goal of cargo-mutants is to be _easy_ to run on any Rust source tree, and -to tell you something _interesting_ about areas where bugs might be lurking or -the tests might be insufficient.** - -Being _easy_ to use means: - -* cargo-mutants requires no changes to the source tree or other setup: just - install and run. So, if it does not find anything interesting to say about a - well-tested tree, it didn't cost you much. (This worked out really well: - `cargo install cargo-mutants && cargo mutants` will do it.) - -* There is no chance that running cargo-mutants will change the released - behavior of your program (other than by helping you to fix bugs!), because you - don't need to change the source to use it. - -* cargo-mutants should be reasonably fast even on large Rust trees. The overall - run time is, roughly, the product of the number of viable mutations multiplied - by the time to run the test suite for each mutation. Typically, one `cargo - mutants` run will give you all the information it can find about missing test - coverage in the tree, and you don't need to run it again as you iterate on - tests, so it's relatively OK if it takes a while. (There is currently very - little overhead beyond the cost to do an incremental build and run the tests - for each mutant, but that can still be significant for large trees. There's - room to improve by testing multiple mutants in parallel.) - -* cargo-mutants should run correctly on any Rust source trees that are built and - tested by Cargo, that will build and run their tests in a copy of the tree, - and that have hermetic tests. (It's not all the way there yet; in particular - it assumes the source is in `src/`.) - -* cargo-mutants shouldn't crash or hang, even if it generates mutants that cause - the software under test to crash or hang. (This is generally met today: - cargo-mutants runs tests with an automatically set and configurable timeout.) - -* The results should be reproducible, assuming the build and test suite is - deterministic. (This should be true today; please file a bug if it's not. Mutants are run in random order unless `--no-shuffle` is specified, but this should not affect the results.) - -* cargo-mutants should avoid generating unviable mutants that don't compile, - because that wastes time. However, when it's uncertain whether the mutant will - build, it's worth trying things that _might_ find interesting results even if - they might fail to build. (It does currently generate _some_ unviable mutants, but typically not too many, and they don't have a large effect on runtime in most trees.) - -* Realistically, cargo-mutants may generate some mutants that aren't caught by - tests but also aren't interesting, or aren't feasible to test. In those cases - it should be easy to permanently dismiss them (e.g. by adding a - `#[mutants::skip]` attribute or a config file.) (The attribute exists but - there is no config file yet.) - -Showing _interesting results_ mean: - -* cargo-mutants should tell you about places where the code could be wrong and - the test suite wouldn't catch it. If it doesn't find any interesting results - on typical trees, there's no point. Aspirationally, it will even find useful - results in code with high line coverage, when there is code that is reached by - a test, but no test depends on its behavior. - -* In superbly-tested projects cargo-mutants may find nothing to say, but hey, at - least it was easy to run, and hopefully the assurance that the tests really do - seem to be good is useful data. - -* _Most_, ideally all, findings should indicate something that really should be - tested more, or that may already be buggy, or that's at least worth looking at. - -* It should be easy to understand what the output is telling you about a - potential bug that wouldn't be caught. (This seems true today.) It might take - some thought to work out _why_ the existing tests don't cover it, or how to - check it, but at least you know where to begin. - -* As much as possible cargo-mutants should avoid generating trivial mutants, - where the mutated code is effectively equivalent to the original code, and so - it's not interesting that the test suite doesn't catch the change. (Not much - has been done here yet.) - -* For trees that are thoroughly tested, you can use `cargo mutants` in CI to - check that they remain so. - -## How it works - -The basic approach is: - -* Make a copy of the source tree into a scratch directory, excluding - version-control directories like `.git` and the `/target` directory. The same directory is reused across all the mutations to benefit from incremental builds. - - * After copying the tree, cargo-mutants scans the top-level `Cargo.toml` and any - `.cargo/config.toml` for relative dependencies. If there are any, the paths are - rewritten to be absolute, so that they still work when cargo is run in the - scratch directory. - - * Before applying any mutations, check that `cargo test` succeeds in the - scratch directory: perhaps a test is already broken, or perhaps the tree - doesn't build when copied because it relies on relative paths to find - dependencies, etc. - -* Build a list of mutations: - * Run `cargo metadata` to find directories containing Rust source files. - * Walk all source files and parse each one looking for functions. - * Skip functions that should not be mutated for any of several reasons: - because they're tests, because they have a `#[mutants::skip]` attribute, - etc. - * For each function, depending on its return type, generate every mutation - pattern that produces a result of that type. - -* For each mutation: - * Apply the mutation to the scratch tree by patching the affected file. - * Run `cargo test` in the tree, saving output to a log file. - * If the build fails or the tests fail, that's good: the mutation was somehow - caught. - * If the build and tests succeed, that might mean test coverage was - inadequate, or it might mean we accidentally generated a no-op mutation. - * Revert the mutation to return the tree to its clean state. - -The file is parsed using the [`syn`](https://docs.rs/syn) crate, but mutations -are applied textually, rather than to the token stream, so that unmutated code -retains its prior formatting, comments, line numbers, etc. This makes it -possible to show a text diff of the mutation and should make it easier to -understand any error messages from the build of the mutated code. - -For more details, see [DESIGN.md](DESIGN.md). - -## Related work - -cargo-mutants was inspired by reading about the -[Descartes mutation-testing tool for Java](https://github.com/STAMP-project/pitest-descartes/) -described in -[Increment magazine's testing issue](https://increment.com/reliability/testing-beyond-coverage/). - -It's an interesting insight that mutation at the level of a whole function is a -practical sweet-spot to discover missing tests, while still making it feasible -to exhaustively generate every mutant, at least for moderate-sized trees. - -See also: [more information on how cargo-mutants compares to other techniques and tools](https://github.com/sourcefrog/cargo-mutants/wiki/Compared). - -## Supported Rust versions - -Building cargo-mutants requires a reasonably recent stable (or nightly or beta) Rust toolchain. - -Currently it is [tested with Rust 1.63](https://github.com/sourcefrog/cargo-mutants/actions/workflows/msrv.yml). - -After installing cargo-mutants, you should be able to use it to run tests under -any toolchain, even toolchains that are far too old to build cargo-mutants, using the standard `+` option to `cargo`: - -```sh -cargo +1.48 mutants -``` - -### Limitations, caveats, known bugs, and future enhancements - -**CAUTION**: This tool builds and runs code with machine-generated -modifications. If the code under test, or the test suite, has side effects such -as writing or deleting files, running it with mutations may be dangerous. Think -first about what side effects the test suite could possibly have, and/or run it -in a restricted or disposable environment. - -cargo-mutants behavior, output formats, command-line syntax, json output -formats, etc, may change from one release to the next. - -cargo-mutants sees the AST of the tree but doesn't fully "understand" the types. -Possibly it could learn to get type information from the compiler (or -rust-analyzer?), which would help it generate more interesting viable mutants, -and fewer unviable mutants. - -cargo-mutants reads `CARGO_ENCODED_RUSTFLAGS` and `RUSTFLAGS` environment variables, and sets `CARGO_ENCODED_RUSTFLAGS`. It does not read `.cargo/config.toml` files, and so any rust flags set there will be ignored. - -## Integrations and related work - -### vim-cargomutants - -[`vim-cargomutants`](https://github.com/yining/vim-cargomutants) provides commands -view cargo-mutants results, see the diff of mutations, and to launch cargo-mutants -from within vim. - -## Code of Conduct - -Interaction with or participation in this project is governed by the [Rust Code -of Conduct](https://www.rust-lang.org/policies/code-of-conduct). -||||||| 188530b -`--no-times`: Don't print elapsed times. - -`--timeout`: Set a fixed timeout for each `cargo test` run, to catch mutations -that cause a hang. By default a timeout is automatically determined. - -`--cargo-arg`: Passes the option argument to `cargo check`, `build`, and `test`. -For example, `--cargo-arg --release`. - -### Filtering files - -Two options (each with short and long names) control which files are mutated: - -`-f GLOB`, `--file GLOB`: Mutate only functions in files matching -the glob. - -`-e GLOB`, `--exclude GLOB`: Exclude files that match the glob. - -These options may be repeated. - -If any `-f` options are given, only source files that match are -considered; otherwise all files are considered. This list is then further -reduced by exclusions. - -If the glob contains `/` (or on Windows, `\`), then it matches against the path from the root of the source -tree. For example, `src/*/*.rs` will exclude all files in subdirectories of `src`. - -If the glob does not contain a path separator, it matches against filenames -in any directory. - -`/` matches the path separator on both Unix and Windows. - -Note that the glob must contain `.rs` (or a matching wildcard) to match -source files with that suffix. For example, `-f network` will match -`src/network/mod.rs` but it will _not_ match `src/network.rs`. - -Files that are excluded are still parsed (and so must be syntactically -valid), and `mod` statements in them are followed to discover other -source files. So, for example, you can exclude `src/main.rs` but still -test mutants in other files referenced by `mod` statements in `main.rs`. - -The results of filters can be previewed with the `--list-files` and `--list` -options. - -Examples: - -* `cargo mutants -f visit.rs -f change.rs` -- test mutants only in files - called `visit.rs` or `change.rs` (in any directory). - -* `cargo mutants -e console.rs` -- test mutants in any file except `console.rs`. - -* `cargo mutants -f src/db/*.rs` -- test mutants in any file in this directory. - -### Filtering functions and mutants - -Two options filter mutants by the full name of the mutant, which includes the -function name, file name, and a description of the change. - -Mutant names are shown by `cargo mutants --list`, and the same command can be -used to preview the effect of filters. - -`-F REGEX`, `--re REGEX`: Only test mutants whose full name matches the given regex. - -`-E REGEX`, `--exclude-re REGEX`: Exclude mutants whose full name matches -the given regex. - -These options may be repeated. - -The regex matches a substring and can be anchored with `^` and `$`. - -The regex syntax is defined by the [`regex`](https://docs.rs/regex/latest/regex/) -crate. - -These filters are applied after filtering by filename, and `--re` is applied before -`--exclude-re`. - -Examples: - -* `-E 'impl Debug'` -- don't test `impl Debug` methods, because coverage of them - might be considered unimportant. - -* `-F 'impl Serialize' -F 'impl Deserialize'` -- test implementations of these - two traits. - -### Passing arguments to `cargo test` - -Command-line options following a `--` delimiter are passed through to -`cargo test`, which can be used for example to exclude doctests (which tend to -be slow to build and run): - -```sh -cargo mutants -- --all-targets -``` - -You can use a second double-dash to pass options through to the test targets: - -```sh -cargo mutants -- -- --test-threads 1 --nocapture -``` - -### Understanding the results - -If tests fail in a clean copy of the tree, there might be an (intermittent) -failure in the source directory, or there might be some problem that stops them -passing when run from a different location, such as a relative `path` in -`Cargo.toml`. Fix this first. - -Otherwise, cargo mutants generates every mutant it can. All mutants fall in to -one of these categories: - -* **caught** — A test failed with this mutant applied. This is a good sign about - test coverage. You can look in `mutants.out/log` to see which tests failed. - -* **missed** — No test failed with this mutation applied, which seems to - indicate a gap in test coverage. Or, it may be that the mutant is - undistinguishable from the correct code. You may wish to add a better test, or - mark that the function should be skipped. - -* **unviable** — The attempted mutation doesn't compile. This is inconclusive about test coverage and - no action is needed, but indicates an opportunity for cargo-mutants to either - generate better mutants, or at least not generate unviable mutants. - -* **timeout** — The mutation caused the test suite to run for a long time, until it was eventually killed. You might want to investigate the cause and potentially mark the function to be skipped. - -By default only missed mutants and timeouts are printed because they're the most actionable. Others can be shown with the `-v` and `-V` options. - -### Skipping functions - -To mark functions so they are not mutated: - -1. Add a Cargo dependency on the [mutants](https://crates.io/crates/mutants) - crate, version "0.0.3" or later. (This must be a regular `dependency` not a - `dev-dependency`, because the annotation will be on non-test code.) - -2. Mark functions with `#[mutants::skip]` or other attributes containing `mutants::skip` (e.g. `#[cfg_attr(test, mutants::skip)]`). - -See `testdata/tree/hang_avoided_by_attr/` for an example. - -The crate is tiny and the attribute has no effect on the compiled code. It only -flags the function for cargo-mutants. - -**Note:** Currently, `cargo-mutants` does not (yet) evaluate attributes like `cfg_attr`, it only looks for the sequence `mutants::skip` in the attribute. - -**Note:** Rust's "inner macro attributes" feature is currently unstable, so `#![mutants::skip]` can't be used at the top of a file in stable Rust. - -### Config file - -cargo-mutants looks for a `.cargo/mutants.toml` file in the root of the source -directory. If a config file exists, the values are appended to the corresponding -command-line arguments. (This may cause problems if you use `--` twice on the -command line to pass arguments to the inner test binary.) - -Configured exclusions may be particularly important when there are modules that -are inherently hard to test, and the project has made a decision to accept lower -test coverage for them. - -The following configuration options are supported: - -```toml -exclude_globs = ["src/main.rs", "src/cache/*.rs"] # same as -e -examine_globs = ["src/important/*.rs"] # same as -f, test *only* these files - -exclude_re = ["impl Debug"] # same as -E -examine_re = ["impl Serialize", "impl Deserialize"] # same as -F, test *only* matches - -additional_cargo_args = ["--all-features"] -additional_cargo_test_args = ["--jobs=1"] -``` - -### Exit codes - -* **0**: Success. No mutants were found that weren't caught by tests. - -* **1**: Usage error: bad command-line arguments etc. - -* **2**: Found some mutants that were not covered by tests. - -* **3**: Some tests timed out: possibly the mutatations caused an infinite loop, - or the timeout is too low. - -* **4**: The tests are already failing or hanging before any mutations are - applied, so no mutations were tested. - -### `mutants.out` - -A `mutants.out` directory is created in the source directory, or whichever directory you specify with `--output`. It contains: - -* A `logs/` directory, with one log file for each mutation plus the baseline - unmutated case. The log contains the diff of the mutation plus the output from - cargo. - -* A `lock.json`, on which an [fs2 lock](https://docs.rs/fs2) is held while - cargo-mutants is running, to avoid two tasks trying to write to the same - directory at the same time. The lock contains the start time, cargo-mutants - version, username, and hostname. `lock.json` is left in `mutants.out` when the - run completes, but the lock on it is released. - -* `caught.txt`, `missed.txt`, `timeout.txt`, `unviable.txt`, each listing mutants with the corresponding outcome. - -* A `mutants.json` file describing all the generated mutants. - -* An `outcomes.json` file describing the results of all tests. - -### Hangs and timeouts - -Some mutations to the tree can cause the test suite to hang. For example, in -this code, cargo-mutants might try changing `should_stop` to always return -`false`: - -```rust - while !should_stop() { - // something - } -``` - -`cargo mutants` automatically sets a timeout when running tests with mutations -applied, and reports mutations that hit a timeout. The automatic timeout is the greater of -20 seconds, or 5x the time to run tests with no mutations. - -The `CARGO_MUTANTS_MINIMUM_TEST_TIMEOUT` environment variable, measured in seconds, overrides the minimum time. - -You can also set an explicit timeout with the `--timeout` option. In this case -the timeout is also applied to tests run with no mutation. - -The timeout does not apply to `cargo check` or `cargo build`, only `cargo test`. - -When a test times out, you can mark it with `#[mutants::skip]` so that future -`cargo mutants` runs go faster. - -### Parallelism - -The `--jobs` or `-j` option allows to test multiple mutants in parallel, by spawning several Cargo processes. This can give 25-50% performance improvements, depending on the tree under test and the hardware resources available. - -It's common that for some periods of its execution, a single Cargo build or test job can't use all the available CPU cores. Running multiple jobs in parallel makes use of resources that would otherwise be idle. - -However, running many jobs simultaneously may also put high demands on the system's RAM (by running more compile/link/test tasks simultaneously), IO bandwidth, and cooling (by fully using all cores). - -The best setting will depend on many factors including the behavior of your program's test suite, the amount of memory on your system, and your system's behavior under high thermal load. - -The default is currently to run only one job at a time. It's reasonable to set this to any value up to the number of CPU cores. - -`-j 4` may be a good starting point, even if you have many more CPUs. Start there and watch memory and CPU usage, and tune towards a setting where all cores are always utilized without memory usage going too high, and without thermal issues. - -Because tests may be slower with high parallelism, you may see some spurious timeouts, and you may need to set `--timeout` manually to allow enough safety margin. - -### Performance - -Most of the runtime for cargo-mutants is spent in running the program test suite -and in running incremental builds: both are done once per viable mutant. - -So, anything you can do to make the `cargo build` and `cargo test` suite faster -will have a multiplicative effect on `cargo mutants` run time, and of course -will also make normal development more pleasant. - -There's lots of good advice on the web, including . - -Rust doctests are pretty slow, so if you're using them only as testable -documentation and not to assert correctness of the code, you can skip them with -`cargo mutants -- --all-targets`. - -On _some but not all_ projects, cargo-mutants can be faster if you use `-C --release`, which will make the build slower but may make the tests faster. Typically this will help on projects with very long CPU-intensive test suites. Cargo-mutants now shows the breakdown of build versus test time which may help you work out if this will help. (On projects like this you might also choose just to turn up optimization for all debug builds in [`.cargo/config.toml`](https://doc.rust-lang.org/cargo/reference/config.html). - -By default cargo-mutants copies the `target/` directory from the source tree. Rust target directories can accumulate excessive volumes of old build products. - -cargo-mutants causes the Rust toolchain (and, often, the program under test) to read and write _many_ temporary files. Setting the temporary directory onto a ramdisk can improve performance significantly. This is particularly important with parallel builds, which might otherwise hit disk bandwidth limits. For example on Linux: - -```shell -sudo mkdir /ram -sudo mount -t tmpfs /ram /ram # or put this in fstab, or just change /tmp -env TMPDIR=/ram cargo mutants -``` - -### Using the Mold linker - -Using the [Mold linker](https://github.com/rui314/mold) on Unix can give a 20% performance improvement, depending on the tree. -Because cargo-mutants does many -incremental builds, link time is important, especially if the test suite is relatively fast. - -Because of limitations in the way cargo-mutants runs Cargo, the standard way of configuring Mold for Rust in `~/.cargo/config.toml` won't work. - -Instead, set the `RUSTFLAGS` environment variable to `-Clink-arg=-fuse-ld=mold`. - -### Workspace and package support - -cargo-mutants now supports testing Cargo workspaces that contain multiple packages. - -All source files in all packages in the workspace are tested. For each mutant, only the containing packages tests are run. - -### Hard-to-test cases - -Some functions don't cause a test suite failure if emptied, but also cannot be -removed. For example, functions to do with managing caches or that have other -performance side effects. - -Ideally, these should be tested, but doing so in a way that's not flaky can be -difficult. cargo-mutants can help in a few ways: - -* It helps to at least highlight to the developer that the function is not - covered by tests, and so should perhaps be treated with extra care, or tested - manually. -* A `#[mutants::skip]` annotation can be added to suppress warnings and explain - the decision. -* Sometimes these effects can be tested by making the side-effect observable - with, for example, a counter of the number of memory allocations or cache - misses/hits. - -### Continuous integration - -Here is an example of a GitHub Actions workflow that runs mutation tests and uploads the results as an artifact. This will fail if it finds any uncaught mutants. - -```yml -name: cargo-mutants - -on: [pull_request, push] - -jobs: - cargo-mutants: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - name: Install cargo-mutants - run: cargo install --locked cargo-mutants - - name: Run mutant tests - run: cargo mutants -- --all-features - - name: Archive results - uses: actions/upload-artifact@v3 - if: failure() - with: - name: mutation-report - path: mutants.out -``` - -## How to help - -Experience reports in [GitHub Discussions](https://github.com/sourcefrog/cargo-mutants/discussions) or issues are very welcome: - -* Did it find a bug or important coverage gap? -* Did it fail to build and test your tree? (Some cases that aren't supported yet - are already listed in this doc or the bug tracker.) - -It's especially helpful if you can either point to an open source tree that will -reproduce the problem (or success) or at least describe how to reproduce it. - -If you are interested in contributing a patch, please read [CONTRIBUTING.md](CONTRIBUTING.md). - -## Goals - -**The goal of cargo-mutants is to be _easy_ to run on any Rust source tree, and -to tell you something _interesting_ about areas where bugs might be lurking or -the tests might be insufficient.** - -Being _easy_ to use means: - -* cargo-mutants requires no changes to the source tree or other setup: just - install and run. So, if it does not find anything interesting to say about a - well-tested tree, it didn't cost you much. (This worked out really well: - `cargo install cargo-mutants && cargo mutants` will do it.) - -* There is no chance that running cargo-mutants will change the released - behavior of your program (other than by helping you to fix bugs!), because you - don't need to change the source to use it. - -* cargo-mutants should be reasonably fast even on large Rust trees. The overall - run time is, roughly, the product of the number of viable mutations multiplied - by the time to run the test suite for each mutation. Typically, one `cargo - mutants` run will give you all the information it can find about missing test - coverage in the tree, and you don't need to run it again as you iterate on - tests, so it's relatively OK if it takes a while. (There is currently very - little overhead beyond the cost to do an incremental build and run the tests - for each mutant, but that can still be significant for large trees. There's - room to improve by testing multiple mutants in parallel.) - -* cargo-mutants should run correctly on any Rust source trees that are built and - tested by Cargo, that will build and run their tests in a copy of the tree, - and that have hermetic tests. (It's not all the way there yet; in particular - it assumes the source is in `src/`.) - -* cargo-mutants shouldn't crash or hang, even if it generates mutants that cause - the software under test to crash or hang. (This is generally met today: - cargo-mutants runs tests with an automatically set and configurable timeout.) - -* The results should be reproducible, assuming the build and test suite is - deterministic. (This should be true today; please file a bug if it's not. Mutants are run in random order unless `--no-shuffle` is specified, but this should not affect the results.) - -* cargo-mutants should avoid generating unviable mutants that don't compile, - because that wastes time. However, when it's uncertain whether the mutant will - build, it's worth trying things that _might_ find interesting results even if - they might fail to build. (It does currently generate _some_ unviable mutants, but typically not too many, and they don't have a large effect on runtime in most trees.) - -* Realistically, cargo-mutants may generate some mutants that aren't caught by - tests but also aren't interesting, or aren't feasible to test. In those cases - it should be easy to permanently dismiss them (e.g. by adding a - `#[mutants::skip]` attribute or a config file.) (The attribute exists but - there is no config file yet.) - -Showing _interesting results_ mean: - -* cargo-mutants should tell you about places where the code could be wrong and - the test suite wouldn't catch it. If it doesn't find any interesting results - on typical trees, there's no point. Aspirationally, it will even find useful - results in code with high line coverage, when there is code that is reached by - a test, but no test depends on its behavior. - -* In superbly-tested projects cargo-mutants may find nothing to say, but hey, at - least it was easy to run, and hopefully the assurance that the tests really do - seem to be good is useful data. - -* _Most_, ideally all, findings should indicate something that really should be - tested more, or that may already be buggy, or that's at least worth looking at. - -* It should be easy to understand what the output is telling you about a - potential bug that wouldn't be caught. (This seems true today.) It might take - some thought to work out _why_ the existing tests don't cover it, or how to - check it, but at least you know where to begin. - -* As much as possible cargo-mutants should avoid generating trivial mutants, - where the mutated code is effectively equivalent to the original code, and so - it's not interesting that the test suite doesn't catch the change. (Not much - has been done here yet.) - -* For trees that are thoroughly tested, you can use `cargo mutants` in CI to - check that they remain so. - -## How it works - -The basic approach is: - -* Make a copy of the source tree into a scratch directory, excluding - version-control directories like `.git` and the `/target` directory. The same directory is reused across all the mutations to benefit from incremental builds. - - * After copying the tree, cargo-mutants scans the top-level `Cargo.toml` and any - `.cargo/config.toml` for relative dependencies. If there are any, the paths are - rewritten to be absolute, so that they still work when cargo is run in the - scratch directory. - - * Before applying any mutations, check that `cargo test` succeeds in the - scratch directory: perhaps a test is already broken, or perhaps the tree - doesn't build when copied because it relies on relative paths to find - dependencies, etc. - -* Build a list of mutations: - * Run `cargo metadata` to find directories containing Rust source files. - * Walk all source files and parse each one looking for functions. - * Skip functions that should not be mutated for any of several reasons: - because they're tests, because they have a `#[mutants::skip]` attribute, - etc. - * For each function, depending on its return type, generate every mutation - pattern that produces a result of that type. - -* For each mutation: - * Apply the mutation to the scratch tree by patching the affected file. - * Run `cargo test` in the tree, saving output to a log file. - * If the build fails or the tests fail, that's good: the mutation was somehow - caught. - * If the build and tests succeed, that might mean test coverage was - inadequate, or it might mean we accidentally generated a no-op mutation. - * Revert the mutation to return the tree to its clean state. - -The file is parsed using the [`syn`](https://docs.rs/syn) crate, but mutations -are applied textually, rather than to the token stream, so that unmutated code -retains its prior formatting, comments, line numbers, etc. This makes it -possible to show a text diff of the mutation and should make it easier to -understand any error messages from the build of the mutated code. - -For more details, see [DESIGN.md](DESIGN.md). - -## Related work - -cargo-mutants was inspired by reading about the -[Descartes mutation-testing tool for Java](https://github.com/STAMP-project/pitest-descartes/) -described in -[Increment magazine's testing issue](https://increment.com/reliability/testing-beyond-coverage/). - -It's an interesting insight that mutation at the level of a whole function is a -practical sweet-spot to discover missing tests, while still making it feasible -to exhaustively generate every mutant, at least for moderate-sized trees. - -See also: [more information on how cargo-mutants compares to other techniques and tools](https://github.com/sourcefrog/cargo-mutants/wiki/Compared). - -## Supported Rust versions - -Building cargo-mutants requires a reasonably recent stable (or nightly or beta) Rust toolchain. - -Currently it is [tested with Rust 1.63](https://github.com/sourcefrog/cargo-mutants/actions/workflows/msrv.yml). - -After installing cargo-mutants, you should be able to use it to run tests under -any toolchain, even toolchains that are far too old to build cargo-mutants, using the standard `+` option to `cargo`: - -```sh -cargo +1.48 mutants -``` - -### Limitations, caveats, known bugs, and future enhancements - -**CAUTION**: This tool builds and runs code with machine-generated -modifications. If the code under test, or the test suite, has side effects such -as writing or deleting files, running it with mutations may be dangerous. Think -first about what side effects the test suite could possibly have, and/or run it -in a restricted or disposable environment. - -cargo-mutants behavior, output formats, command-line syntax, json output -formats, etc, may change from one release to the next. - -cargo-mutants sees the AST of the tree but doesn't fully "understand" the types. -Possibly it could learn to get type information from the compiler (or -rust-analyzer?), which would help it generate more interesting viable mutants, -and fewer unviable mutants. - -cargo-mutants reads `CARGO_ENCODED_RUSTFLAGS` and `RUSTFLAGS` environment variables, and sets `CARGO_ENCODED_RUSTFLAGS`. It does not read `.cargo/config.toml` files, and so any rust flags set there will be ignored. - -## Integrations and related work - -### vim-cargomutants - -[`vim-cargomutants`](https://github.com/yining/vim-cargomutants) provides commands -view cargo-mutants results, see the diff of mutations, and to launch cargo-mutants -from within vim. - -## Code of Conduct - -Interaction with or participation in this project is governed by the [Rust Code -of Conduct](https://www.rust-lang.org/policies/code-of-conduct). -======= * [cargo-mutants manual](https://mutants.rs/) * [How cargo-mutants compares to other techniques and tools](https://github.com/sourcefrog/cargo-mutants/wiki/Compared). * [Design notes](DESIGN.md) * [Contributing](CONTRIBUTING.md) * [Release notes](NEWS.md) * [Discussions](https://github.com/sourcefrog/cargo-mutants/discussions) ->>>>>>> origin/main From 43ea72da1c6f807c66118772ac61de2d12ff9d28 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sat, 5 Oct 2024 09:31:50 -0700 Subject: [PATCH 09/10] comment --- src/cargo.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cargo.rs b/src/cargo.rs index dae57e3e..3ea434ab 100644 --- a/src/cargo.rs +++ b/src/cargo.rs @@ -162,6 +162,9 @@ fn cargo_argv( /// Return adjusted CARGO_ENCODED_RUSTFLAGS, including any changes to cap-lints. /// +/// It seems we have to set this in the environment because Cargo doesn't expose +/// a way to pass it in as an option from all commands? +/// /// This does not currently read config files; it's too complicated. /// /// See From 2d977d1d7c5bf528f4b450bd36713e6d5c78ef29 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sat, 5 Oct 2024 09:57:04 -0700 Subject: [PATCH 10/10] Mention diff/ in book --- book/src/mutants-out.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/book/src/mutants-out.md b/book/src/mutants-out.md index 32450ffe..20182c09 100644 --- a/book/src/mutants-out.md +++ b/book/src/mutants-out.md @@ -19,6 +19,9 @@ The output directory contains: * An `outcomes.json` file describing the results of all tests, and summary counts of each outcome. +* A `diff/` directory, containing a diff file for each mutation, relative to the unmutated baseline. + `mutants.json` includes for each mutant the name of the diff file. + * A `logs/` directory, with one log file for each mutation plus the baseline unmutated case. The log contains the diff of the mutation plus the output from cargo. `outcomes.json` includes for each mutant the name of the log file. @@ -31,4 +34,4 @@ The contents of the directory and the format of these files is subject to change These files are incrementally updated while cargo-mutants runs, so other programs can read them to follow progress. -There is generally no reason to include this directory in version control, so it is recommended that you add `/mutants.out*` to your `.gitignore` file. This will exclude both `mutants.out` and `mutants.out.old`. +There is generally no reason to include this directory in version control, so it is recommended that you add `/mutants.out*` to your `.gitignore` file or equivalent. This will exclude both `mutants.out` and `mutants.out.old`.