diff --git a/Cargo.lock b/Cargo.lock index 5b324721ed8d1..f995834b6ac34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4128,22 +4128,27 @@ dependencies = [ "inferno", "itertools 0.14.0", "mockall", + "num-bigint", "opener", "parking_lot", "paste", "path-slash", "proptest", "quick-junit", + "rand 0.9.2", "rayon", "regex", "reqwest", "revm", + "rstest", "semver 1.0.27", "serde", "serde_json", "similar", "similar-asserts", "solar-compiler", + "solar-interface", + "solar-parse", "soldeer-commands", "strum 0.27.2", "svm-rs", @@ -5105,6 +5110,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.31" @@ -8026,6 +8037,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.23" @@ -8359,6 +8376,36 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rstest" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", + "rustc_version 0.4.1", +] + +[[package]] +name = "rstest_macros" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version 0.4.1", + "syn 2.0.106", + "unicode-ident", +] + [[package]] name = "rtoolbox" version = "0.0.3" diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 5950db5bde974..6342669ae1673 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -334,6 +334,8 @@ pub struct Config { pub coverage_pattern_inverse: Option, /// Path where last test run failures are recorded. pub test_failures_file: PathBuf, + /// Pathe where mutation tests are cached, to resume running them + pub mutation_dir: PathBuf, /// Max concurrent threads to use. pub threads: Option, /// Whether to show test execution progress. @@ -1209,6 +1211,9 @@ impl Config { // Remove last test run failures file. let _ = fs::remove_file(&self.test_failures_file); + // Remove mutation test cache directory + let _ = fs::remove_dir_all(project.root().join(&self.mutation_dir)); + // Remove fuzz and invariant cache directories. let remove_test_dir = |test_dir: &Option| { if let Some(test_dir) = test_dir { @@ -2494,6 +2499,7 @@ impl Default for Config { path_pattern_inverse: None, coverage_pattern_inverse: None, test_failures_file: "cache/test-failures".into(), + mutation_dir: "cache/mutation".into(), threads: None, show_progress: false, fuzz: FuzzConfig::new("cache/fuzz".into()), diff --git a/crates/forge/Cargo.toml b/crates/forge/Cargo.toml index 2b8143b334cee..071435d6e76e9 100644 --- a/crates/forge/Cargo.toml +++ b/crates/forge/Cargo.toml @@ -62,8 +62,6 @@ alloy-serde.workspace = true alloy-signer.workspace = true alloy-transport.workspace = true -revm.workspace = true - clap = { version = "4", features = ["derive", "env", "unicode", "wrap_help"] } clap_complete.workspace = true dunce.workspace = true @@ -71,12 +69,16 @@ indicatif.workspace = true inferno = { version = "0.12", default-features = false } itertools.workspace = true parking_lot.workspace = true +rand.workspace = true regex = { workspace = true, default-features = false } +reqwest = { workspace = true, features = ["json"] } +revm.workspace = true semver.workspace = true serde_json.workspace = true similar = { version = "2", features = ["inline"] } solar.workspace = true strum = { workspace = true, features = ["derive"] } +tempfile.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["time"] } toml_edit.workspace = true @@ -85,6 +87,7 @@ watchexec-events = "6.0" watchexec-signals = "5.0" clearscreen = "4.0" evm-disassembler.workspace = true +num-bigint = "0.4" path-slash.workspace = true # doc server @@ -95,6 +98,8 @@ opener = "0.8" # soldeer soldeer-commands.workspace = true quick-junit = "0.5.1" +solar-parse = "0.1.8" +solar-interface = "0.1.8" [dev-dependencies] alloy-hardforks.workspace = true @@ -108,6 +113,8 @@ reqwest = { workspace = true, features = ["json"] } mockall = "0.13" globset = "0.4" paste = "1.0" + +rstest = "0.25.0" similar-asserts.workspace = true svm.workspace = true tempfile.workspace = true diff --git a/crates/forge/src/cmd/test/mod.rs b/crates/forge/src/cmd/test/mod.rs index b7593a339d6dc..4cc4098919e82 100644 --- a/crates/forge/src/cmd/test/mod.rs +++ b/crates/forge/src/cmd/test/mod.rs @@ -4,6 +4,7 @@ use crate::{ decode::decode_console_logs, gas_report::GasReport, multi_runner::matches_artifact, + mutation::{MutationHandler, MutationReporter, MutationsSummary}, result::{SuiteResult, TestOutcome, TestStatus}, traces::{ CallTraceDecoderBuilder, InternalTraceMode, TraceKind, @@ -57,6 +58,7 @@ mod filter; mod summary; use crate::{result::TestKind, traces::render_trace_arena_inner}; pub use filter::FilterArgs; +use foundry_cli::utils::FoundryPathExt; use quick_junit::{NonSuccessKind, Report, TestCase, TestCaseStatus, TestSuite}; use summary::{TestSummaryReport, format_invariant_metrics_table}; @@ -199,6 +201,19 @@ pub struct TestArgs { #[command(flatten)] pub watch: WatchArgs, + + /// Enable mutation testing. + /// If passed with file paths, only those files will be tested. + #[arg(long, num_args(0..), value_name = "PATH")] + pub mutate: Option>, + + /// Specify which files to mutate with glob pattern matching. + #[arg(long, value_name = "PATTERN", requires = "mutate")] + pub mutate_path: Option, + + /// Only run tests in contracts matching the specified regex pattern. + #[arg(long, value_name = "REGEX", requires = "mutate")] + pub mutate_contract: Option, } impl TestArgs { @@ -257,6 +272,14 @@ impl TestArgs { // Merge all configs. let (mut config, evm_opts) = self.load_config_and_evm_opts()?; + let should_mutate = self.mutate.is_some(); + + // Force dyn test linking for mutation testing + if should_mutate { + config.dynamic_test_linking = true; + config.cache = true; + } + // Install missing dependencies. if install::install_missing_dependencies(&mut config) && config.auto_detect_remappings { // need to re-configure here to also catch additional remappings @@ -338,10 +361,11 @@ impl TestArgs { .networks(evm_opts.networks) .fail_fast(self.fail_fast) .set_coverage(coverage) - .build::(output, env, evm_opts)?; + .build::(output, env.clone(), evm_opts.clone())?; let libraries = runner.libraries.clone(); - let mut outcome = self.run_tests_inner(runner, config, verbosity, filter, output).await?; + let mut outcome = + self.run_tests_inner(runner.clone(), config.clone(), verbosity, filter, output).await?; if should_draw { let (suite_name, test_name, mut test_result) = @@ -412,6 +436,240 @@ impl TestArgs { } } + // All test have been run once before reaching this point + if self.mutate.is_some() { + // check outcome here, stop if any test failed + // @todo rather set non-allowed failed tests in config and ensure_ok() here? + // @todo other checks: no fork (or just exclude based on clap arg?) + if outcome.failed() > 0 { + eyre::bail!("Cannot run mutation testing with failed tests"); + } + + let mutate_paths = if let Some(pattern) = &self.mutate_path { + // If --mutate-path is provided, use it to filter paths + source_files_iter(&config.src, MultiCompilerLanguage::FILE_EXTENSIONS) + .filter(|entry| { + // @todo filter out interfaces here? + // we do it in lexing for now + entry.is_sol() && !entry.is_sol_test() && pattern.is_match(entry) + }) + .collect() + } else if let Some(contract_pattern) = &self.mutate_contract { + // If --mutate-contract is provided, use it to filter contracts + source_files_iter(&config.src, MultiCompilerLanguage::FILE_EXTENSIONS) + .filter(|entry| { + entry.is_sol() + && !entry.is_sol_test() + && output + .artifact_ids() + .find(|(id, _)| id.source == *entry) + .is_some_and(|(id, _)| contract_pattern.is_match(&id.name)) + }) + .collect() + } else if self.mutate.as_ref().unwrap().is_empty() { + // If --mutate is passed without arguments, use all Solidity files + source_files_iter(&config.src, MultiCompilerLanguage::FILE_EXTENSIONS) + .filter(|entry| entry.is_sol() && !entry.is_sol_test()) + .collect() + } else { + // If --mutate is passed with arguments, use those paths + self.mutate.as_ref().unwrap().clone() + }; + + sh_println!("Running mutation tests...").unwrap(); + let mut mutation_summary = MutationsSummary::new(); + + for path in mutate_paths { + sh_println!("Running mutation tests for {}", path.display()).unwrap(); + + // Check if this file has already been tested and if the build id is the + // same - if so, just add the mutants to the summary + let mut handler = MutationHandler::new(path.clone(), config.clone()); + + handler.read_source_contract()?; + + let build_id = output + .artifact_ids() + .find_map(|(id, _)| if id.source == path { Some(id.build_id) } else { None }) + .unwrap_or_default(); + + // If we have cached results for these mutations and build id, use them and skip + // running tests + if let Some(prior) = handler.retrieve_cached_mutant_results(&build_id) { + for (mutant, status) in prior { + match status { + crate::mutation::mutant::MutationResult::Dead => { + handler.add_dead_mutant(mutant) + } + crate::mutation::mutant::MutationResult::Alive => { + handler.add_survived_mutant(mutant) + } + crate::mutation::mutant::MutationResult::Invalid => { + handler.add_invalid_mutant(mutant) + } + crate::mutation::mutant::MutationResult::Skipped => { + handler.add_skipped_mutant(mutant) + } + } + } + // Add this handler's results to the overall summary + mutation_summary.merge(handler.get_report()); + continue; + } + + // Load survived spans for adaptive mutation testing + handler.retrieve_survived_spans(&build_id); + + // Try cached mutants first + let mut mutants = if let Some(ms) = handler.retrieve_cached_mutants(&build_id) { + ms + } else { + // No cache match: generate fresh mutants (will use survived spans filter) + handler.generate_ast().await; + handler.mutations.clone() + }; + + // Sort mutations by span: group by start position, then test wider spans first + // This maximizes effectiveness of adaptive mutation testing: + // - If a wide span survives, all nested child spans can be skipped + // - Sort by (lo ascending, hi descending) to group spans and test parents first + mutants.sort_by(|a, b| { + let lo_cmp = a.span.lo().0.cmp(&b.span.lo().0); + if lo_cmp != std::cmp::Ordering::Equal { + lo_cmp + } else { + // Same start: wider span (larger hi) comes first + b.span.hi().0.cmp(&a.span.hi().0) + } + }); + + // Debug: Analyze mutation distribution by span + let mut span_counts = std::collections::HashMap::new(); + for m in &mutants { + *span_counts.entry((m.span.lo().0, m.span.hi().0)).or_insert(0) += 1; + } + let spans_with_multiple = + span_counts.iter().filter(|(_, count)| **count > 1).count(); + let total_mutations_at_multi_spans: usize = span_counts + .iter() + .filter(|(_, count)| **count > 1) + .map(|(_, count)| *count) + .sum(); + eprintln!("\n[Adaptive Debug] Mutation distribution:"); + eprintln!(" Total mutations: {}", mutants.len()); + eprintln!(" Unique spans: {}", span_counts.len()); + eprintln!( + " Spans with >1 mutation: {spans_with_multiple} (these are candidates for skipping)" + ); + eprintln!(" Mutations at multi-mutation spans: {total_mutations_at_multi_spans}"); + eprintln!( + " Max mutations at any span: {}", + span_counts.values().max().unwrap_or(&0) + ); + + // Accumulate per-mutant results for persistence + let mut results_vec: Vec<( + crate::mutation::mutant::Mutant, + crate::mutation::mutant::MutationResult, + )> = Vec::with_capacity(mutants.len()); + + for (i, mutant) in mutants.iter().enumerate() { + // Adaptive mutation: Skip if this span already has a surviving mutation + if handler.should_skip_span(mutant.span) { + sh_println!("Skipping mutant {} of {} (adaptive: span already has surviving mutation)", i + 1, mutants.len()).unwrap(); + handler.add_skipped_mutant(mutant.clone()); + results_vec.push(( + mutant.clone(), + crate::mutation::mutant::MutationResult::Skipped, + )); + continue; + } + + sh_println!("Testing mutant {} of {}", i + 1, mutants.len()).unwrap(); + + handler.generate_mutated_solidity(mutant); + let new_filter = self.filter(&config).unwrap(); + let compiler = ProjectCompiler::new() + .dynamic_test_linking(config.dynamic_test_linking) + .quiet(true); + + let compile_output = compiler.compile(&config.project().unwrap()); + + if compile_output.is_err() { + handler.add_invalid_mutant(mutant.clone()); + results_vec.push(( + mutant.clone(), + crate::mutation::mutant::MutationResult::Invalid, + )); + } else { + let mut runner = MultiContractRunnerBuilder::new(config.clone()) + .set_debug(false) + .initial_balance(evm_opts.initial_balance) + .evm_spec(config.evm_spec_id()) + .sender(evm_opts.sender) + .with_fork(evm_opts.clone().get_fork(&config, env.clone())) + .build::( + &compile_output.unwrap(), + env.clone(), + evm_opts.clone(), + )?; + + let results: BTreeMap = + runner.test_collect(&new_filter)?; + + let outcome = TestOutcome::new(Some(runner), results, self.allow_failure); + if outcome.failures().count() > 0 { + handler.add_dead_mutant(mutant.clone()); + results_vec.push(( + mutant.clone(), + crate::mutation::mutant::MutationResult::Dead, + )); + } else { + // Mutation survived! Mark this span to skip similar mutations + let span_key = (mutant.span.lo().0, mutant.span.hi().0); + let remaining_at_span = mutants + .iter() + .skip(i + 1) + .filter(|m| { + (m.span.lo().0, m.span.hi().0) == span_key + || handler.should_skip_span(m.span) + }) + .count(); + eprintln!( + "[Adaptive] Mutation {} SURVIVED at span {:?} - will skip {} remaining mutations", + i + 1, + span_key, + remaining_at_span + ); + handler.mark_span_survived(mutant.span); + handler.add_survived_mutant(mutant.clone()); + results_vec.push(( + mutant.clone(), + crate::mutation::mutant::MutationResult::Alive, + )); + } + } + } + + handler.restore_original_source(); + + // If we generated fresh mutants, persist them for this build id + if !handler.mutations.is_empty() && !build_id.is_empty() { + let _ = handler.persist_cached_mutants(&build_id, &handler.mutations); + let _ = handler.persist_cached_results(&build_id, &results_vec); + // Persist survived spans for adaptive mutation testing + let _ = handler.persist_survived_spans(&build_id); + } + + // Add this handler's results to the overall summary + mutation_summary.merge(handler.get_report()); + } + + MutationReporter::new().report(&mutation_summary); + + outcome = TestOutcome::empty(Some(runner.clone()), true); + } + Ok(outcome) } diff --git a/crates/forge/src/lib.rs b/crates/forge/src/lib.rs index 23879df9156f6..dfb2d32c1f8f5 100644 --- a/crates/forge/src/lib.rs +++ b/crates/forge/src/lib.rs @@ -20,6 +20,8 @@ pub mod gas_report; pub mod multi_runner; pub use multi_runner::{MultiContractRunner, MultiContractRunnerBuilder}; +pub mod mutation; + mod runner; pub use runner::ContractRunner; diff --git a/crates/forge/src/mutation/mod.rs b/crates/forge/src/mutation/mod.rs new file mode 100644 index 0000000000000..116200db39f40 --- /dev/null +++ b/crates/forge/src/mutation/mod.rs @@ -0,0 +1,423 @@ +pub mod mutant; +mod mutators; +mod reporter; +mod visitor; + +// Generate mutants then run tests (reuse the whole unit test flow for now, including compilation to +// select mutants) Use Solar: +use solar_parse::{ + Parser, + ast::interface::{Session, source_map::FileName}, +}; +use std::sync::Arc; + +use crate::mutation::{ + mutant::{Mutant, MutationResult}, + visitor::MutantVisitor, +}; + +pub use crate::mutation::reporter::MutationReporter; + +use crate::result::TestOutcome; +use solar_parse::ast::{Span, visit::Visit}; +use std::{collections::HashSet, path::PathBuf}; + +pub struct MutationsSummary { + dead: Vec, + survived: Vec, + invalid: Vec, + skipped: Vec, +} + +impl Default for MutationsSummary { + fn default() -> Self { + Self::new() + } +} + +impl MutationsSummary { + pub fn new() -> Self { + Self { dead: vec![], survived: vec![], invalid: vec![], skipped: vec![] } + } + + pub fn update_valid_mutant(&mut self, outcome: &TestOutcome, mutant: Mutant) { + if outcome.failures().count() > 0 { + self.dead.push(mutant); + } else { + self.survived.push(mutant); + } + } + + pub fn update_invalid_mutant(&mut self, mutant: Mutant) { + self.invalid.push(mutant); + } + + pub fn add_dead_mutant(&mut self, mutant: Mutant) { + self.dead.push(mutant); + } + + pub fn add_survived_mutant(&mut self, mutant: Mutant) { + self.survived.push(mutant); + } + + pub fn add_skipped_mutant(&mut self, mutant: Mutant) { + self.skipped.push(mutant); + } + + pub fn total_mutants(&self) -> usize { + self.dead.len() + self.survived.len() + self.invalid.len() + self.skipped.len() + } + + pub fn total_dead(&self) -> usize { + self.dead.len() + } + + pub fn total_survived(&self) -> usize { + self.survived.len() + } + + pub fn total_invalid(&self) -> usize { + self.invalid.len() + } + + pub fn total_skipped(&self) -> usize { + self.skipped.len() + } + + pub fn dead(&self) -> String { + self.dead.iter().map(|m| m.to_string()).collect::>().join("\n") + } + + pub fn survived(&self) -> String { + self.survived.iter().map(|m| m.to_string()).collect::>().join("\n") + } + + pub fn invalid(&self) -> String { + self.invalid.iter().map(|m| m.to_string()).collect::>().join("\n") + } + + pub fn skipped(&self) -> String { + self.skipped.iter().map(|m| m.to_string()).collect::>().join("\n") + } + + pub fn get_dead(&self) -> &Vec { + &self.dead + } + + pub fn get_survived(&self) -> &Vec { + &self.survived + } + + pub fn get_invalid(&self) -> &Vec { + &self.invalid + } + + pub fn get_skipped(&self) -> &Vec { + &self.skipped + } + + /// Merge another MutationsSummary into this one + pub fn merge(&mut self, other: &Self) { + self.dead.extend(other.dead.clone()); + self.survived.extend(other.survived.clone()); + self.invalid.extend(other.invalid.clone()); + self.skipped.extend(other.skipped.clone()); + } + + /// Calculate mutation score (percentage of dead mutants out of valid mutants) + /// Higher scores indicate better test coverage + pub fn mutation_score(&self) -> f64 { + let valid_mutants = self.dead.len() + self.survived.len(); + if valid_mutants == 0 { 0.0 } else { self.dead.len() as f64 / valid_mutants as f64 * 100.0 } + } +} + +/// Tracks spans where mutations have survived (weren't killed by tests). +/// Used for adaptive mutation testing to skip redundant mutations. +#[derive(Debug, Clone, Default)] +pub struct SurvivedSpans { + spans: HashSet<(u32, u32)>, // (lo, hi) byte positions +} + +impl SurvivedSpans { + pub fn new() -> Self { + Self { spans: HashSet::new() } + } + + /// Mark a span as having a surviving mutation + pub fn mark_survived(&mut self, span: Span) { + self.spans.insert((span.lo().0, span.hi().0)); + } + + /// Check if this span or any parent span has a surviving mutation + pub fn should_skip(&self, span: Span) -> bool { + let (lo, hi) = (span.lo().0, span.hi().0); + + // Check if this exact span has survived + if self.spans.contains(&(lo, hi)) { + return true; + } + + // Check if any parent span (that contains this span) has survived + for &(parent_lo, parent_hi) in &self.spans { + if parent_lo <= lo && hi <= parent_hi && (parent_lo != lo || parent_hi != hi) { + return true; + } + } + + false + } + + /// Serialize to a list of (lo, hi) pairs for caching + fn to_vec(&self) -> Vec<(u32, u32)> { + self.spans.iter().copied().collect() + } + + /// Deserialize from a list of (lo, hi) pairs + fn from_vec(pairs: Vec<(u32, u32)>) -> Self { + Self { spans: pairs.into_iter().collect() } + } +} + +pub struct MutationHandler { + contract_to_mutate: PathBuf, + src: Arc, + pub mutations: Vec, + config: Arc, + report: MutationsSummary, + survived_spans: SurvivedSpans, +} + +impl MutationHandler { + pub fn new(contract_to_mutate: PathBuf, config: Arc) -> Self { + Self { + contract_to_mutate, + src: Arc::default(), + mutations: vec![], + config, + report: MutationsSummary::new(), + survived_spans: SurvivedSpans::new(), + } + } + + pub fn read_source_contract(&mut self) -> Result<(), std::io::Error> { + let content = std::fs::read_to_string(&self.contract_to_mutate)?; + self.src = Arc::new(content); + Ok(()) + } + + /// Add a dead mutant to the report + pub fn add_dead_mutant(&mut self, mutant: Mutant) { + self.report.add_dead_mutant(mutant); + } + + /// Add a survived mutant to the report + pub fn add_survived_mutant(&mut self, mutant: Mutant) { + self.report.add_survived_mutant(mutant); + } + + /// Add an invalid mutant to the report + pub fn add_invalid_mutant(&mut self, mutant: Mutant) { + self.report.update_invalid_mutant(mutant); + } + + pub fn add_skipped_mutant(&mut self, mutant: Mutant) { + self.report.add_skipped_mutant(mutant); + } + + /// Get a reference to the current report + pub fn get_report(&self) -> &MutationsSummary { + &self.report + } + + /// Get a mutable reference to the current report + pub fn get_report_mut(&mut self) -> &mut MutationsSummary { + &mut self.report + } + + // Note: we now get the build hash directly from the recent compile output (see test flow) + + /// Persists cached mutants using build hash for cache invalidation. + /// Writes to `cache/mutation/_.mutants`. + pub fn persist_cached_mutants(&self, hash: &str, mutants: &[Mutant]) -> std::io::Result<()> { + let cache_dir = self.config.root.join(&self.config.mutation_dir); + std::fs::create_dir_all(&cache_dir)?; + + let filename = + self.contract_to_mutate.file_stem().and_then(|s| s.to_str()).unwrap_or("unknown"); + let cache_file = cache_dir.join(format!("{hash}_{filename}.mutants")); + let json = serde_json::to_string_pretty(mutants).map_err(std::io::Error::other)?; + std::fs::write(cache_file, json)?; + + Ok(()) + } + + /// Persists results for mutants using build hash for cache invalidation. + /// Writes to `cache/mutation/_.results`. + pub fn persist_cached_results( + &self, + hash: &str, + results: &[(Mutant, crate::mutation::mutant::MutationResult)], + ) -> std::io::Result<()> { + let cache_dir = self.config.root.join(&self.config.mutation_dir); + std::fs::create_dir_all(&cache_dir)?; + + let filename = + self.contract_to_mutate.file_stem().and_then(|s| s.to_str()).unwrap_or("unknown"); + let cache_file = cache_dir.join(format!("{hash}_{filename}.results")); + let json = serde_json::to_string_pretty(results).map_err(std::io::Error::other)?; + std::fs::write(cache_file, json)?; + + Ok(()) + } + + /// Read a source string, and for each contract found, gets its ast and visit it to list + /// all mutations to conduct + pub async fn generate_ast(&mut self) { + let path = &self.contract_to_mutate; + let target_content = Arc::clone(&self.src); + let sess = Session::builder().with_silent_emitter(None).build(); + + // Clone survived_spans for use in the closure + let survived_spans_clone = self.survived_spans.clone(); + + let _ = sess.enter(|| -> solar_parse::interface::Result<()> { + let arena = solar_parse::ast::Arena::new(); + let mut parser = + Parser::from_lazy_source_code(&sess, &arena, FileName::from(path.clone()), || { + Ok((*target_content).to_string()) + })?; + + let ast = parser.parse_file().map_err(|e| e.emit())?; + + // Create visitor with adaptive span filter + let mut mutant_visitor = MutantVisitor::default(path.clone()) + .with_span_filter(move |span| survived_spans_clone.should_skip(span)); + let _ = mutant_visitor.visit_source_unit(&ast); + self.mutations.extend(mutant_visitor.mutation_to_conduct); + // Log skipped mutations for debugging + if mutant_visitor.skipped_count > 0 { + eprintln!( + "Adaptive mutation: Skipped {} mutation points (already have surviving mutations)", + mutant_visitor.skipped_count + ); + } + Ok(()) + }); + } + + /// Based on a given mutation, emit the corresponding mutated solidity code and write it to disk + pub fn generate_mutated_solidity(&self, mutation: &Mutant) { + let span = mutation.span; + let replacement = mutation.mutation.to_string(); + + let src_content = Arc::clone(&self.src); + + let start_pos = span.lo().0 as usize; + let end_pos = span.hi().0 as usize; + + let before = &src_content[..start_pos]; + let after = &src_content[end_pos..]; + + let mut new_content = String::with_capacity(before.len() + replacement.len() + after.len()); + new_content.push_str(before); + new_content.push_str(&replacement); + new_content.push_str(after); + + std::fs::write(&self.contract_to_mutate, new_content).unwrap_or_else(|_| { + panic!("Failed to write to target file {:?}", &self.contract_to_mutate) + }); + } + + // @todo src to mutate should be in a tmp dir for safety (and modify config accordingly) + /// Restore the original source contract to the target file (end of mutation tests) + pub fn restore_original_source(&self) { + std::fs::write(&self.contract_to_mutate, &*self.src).unwrap_or_else(|_| { + panic!("Failed to write to target file {:?}", &self.contract_to_mutate) + }); + } + + /// Retrieves cached mutants using build hash. + /// Reads from `cache/mutation/_.mutants`. + pub fn retrieve_cached_mutants(&self, hash: &str) -> Option> { + let cache_dir = self.config.root.join(&self.config.mutation_dir); + let filename = + self.contract_to_mutate.file_stem().and_then(|s| s.to_str()).unwrap_or("unknown"); + let cache_file = cache_dir.join(format!("{hash}_{filename}.mutants")); + + if !cache_file.exists() { + return None; + } + + let data = std::fs::read_to_string(cache_file).ok()?; + serde_json::from_str(&data).ok() + } + + /// Retrieves cached results using build hash. + /// Reads from `cache/mutation/_.results`. + pub fn retrieve_cached_mutant_results( + &self, + hash: &str, + ) -> Option> { + let cache_dir = self.config.root.join(&self.config.mutation_dir); + let filename = + self.contract_to_mutate.file_stem().and_then(|s| s.to_str()).unwrap_or("unknown"); + let cache_file = cache_dir.join(format!("{hash}_{filename}.results")); + + if !cache_file.exists() { + return None; + } + + let data = std::fs::read_to_string(cache_file).ok()?; + serde_json::from_str(&data).ok() + } + + /// Mark a span as having a surviving mutation + pub fn mark_span_survived(&mut self, span: Span) { + self.survived_spans.mark_survived(span); + } + + /// Check if a span should be skipped (has survived mutation or is child of survived span) + pub fn should_skip_span(&self, span: Span) -> bool { + self.survived_spans.should_skip(span) + } + + /// Persist survived spans to cache for adaptive mutation testing. + /// Writes to `cache/mutation/_.survived`. + pub fn persist_survived_spans(&self, hash: &str) -> std::io::Result<()> { + let cache_dir = self.config.root.join(&self.config.mutation_dir); + std::fs::create_dir_all(&cache_dir)?; + + let filename = + self.contract_to_mutate.file_stem().and_then(|s| s.to_str()).unwrap_or("unknown"); + let cache_file = cache_dir.join(format!("{hash}_{filename}.survived")); + + let spans = self.survived_spans.to_vec(); + let json = serde_json::to_string_pretty(&spans).map_err(std::io::Error::other)?; + std::fs::write(cache_file, json)?; + + Ok(()) + } + + /// Retrieve survived spans from cache. + /// Reads from `cache/mutation/_.survived`. + pub fn retrieve_survived_spans(&mut self, hash: &str) -> bool { + let cache_dir = self.config.root.join(&self.config.mutation_dir); + let filename = + self.contract_to_mutate.file_stem().and_then(|s| s.to_str()).unwrap_or("unknown"); + let cache_file = cache_dir.join(format!("{hash}_{filename}.survived")); + + if !cache_file.exists() { + return false; + } + + if let Ok(data) = std::fs::read_to_string(cache_file) + && let Ok(pairs) = serde_json::from_str::>(&data) + { + self.survived_spans = SurvivedSpans::from_vec(pairs); + return true; + } + + false + } +} diff --git a/crates/forge/src/mutation/mutant.rs b/crates/forge/src/mutation/mutant.rs new file mode 100644 index 0000000000000..2d195b8dac61f --- /dev/null +++ b/crates/forge/src/mutation/mutant.rs @@ -0,0 +1,290 @@ +// Generate mutants then run tests (reuse the whole unit test flow for now, including compilation to +// select mutants) Use Solar: +use super::visitor::AssignVarTypes; +use serde::{Deserialize, Serialize}; +use solar_parse::ast::{BinOpKind, LitKind, Span, StrKind, UnOpKind}; +use std::{fmt::Display, path::PathBuf}; + +/// Wraps an unary operator mutated, to easily store pre/post-fix op swaps +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UnaryOpMutated { + /// String containing the whole new expression (operator and its target) + /// eg `a++` + new_expression: String, + + /// The underlying operator used by this mutant + #[serde(serialize_with = "serialize_unop_kind", deserialize_with = "deserialize_unop_kind")] + pub resulting_op_kind: UnOpKind, +} + +// Custom serialization for UnOpKind +fn serialize_unop_kind(value: &UnOpKind, serializer: S) -> Result +where + S: serde::Serializer, +{ + let s = format!("{value:?}"); + serializer.serialize_str(&s) +} + +fn deserialize_unop_kind<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + match s.as_str() { + "PreInc" => Ok(UnOpKind::PreInc), + "PostInc" => Ok(UnOpKind::PostInc), + "PreDec" => Ok(UnOpKind::PreDec), + "PostDec" => Ok(UnOpKind::PostDec), + "Not" => Ok(UnOpKind::Not), + "BitNot" => Ok(UnOpKind::BitNot), + "Neg" => Ok(UnOpKind::Neg), + other => Err(serde::de::Error::custom(format!("Unknown UnOpKind: {other}"))), + } +} + +impl UnaryOpMutated { + pub fn new(new_expression: String, resulting_op_kind: UnOpKind) -> Self { + Self { new_expression, resulting_op_kind } + } +} + +impl Display for UnaryOpMutated { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.new_expression) + } +} + +// Custom serialization for BinOpKind +fn serialize_binop(value: &BinOpKind, serializer: S) -> Result +where + S: serde::Serializer, +{ + let s = format!("{value:?}"); + serializer.serialize_str(&s) +} + +fn deserialize_binop<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + match s.as_str() { + "Add" => Ok(BinOpKind::Add), + "Sub" => Ok(BinOpKind::Sub), + "Mul" => Ok(BinOpKind::Mul), + "Div" => Ok(BinOpKind::Div), + "And" => Ok(BinOpKind::And), + "Or" => Ok(BinOpKind::Or), + "Eq" => Ok(BinOpKind::Eq), + "Ne" => Ok(BinOpKind::Ne), + "Lt" => Ok(BinOpKind::Lt), + "Le" => Ok(BinOpKind::Le), + "Gt" => Ok(BinOpKind::Gt), + "Ge" => Ok(BinOpKind::Ge), + "BitAnd" => Ok(BinOpKind::BitAnd), + "BitOr" => Ok(BinOpKind::BitOr), + "BitXor" => Ok(BinOpKind::BitXor), + "Shl" => Ok(BinOpKind::Shl), + "Shr" => Ok(BinOpKind::Shr), + "Sar" => Ok(BinOpKind::Sar), + other => Err(serde::de::Error::custom(format!("Unknown BinOpKind: {other}"))), + } +} + +// @todo add a mutation from universalmutator: line swap (swap two lines of code, as it +// could theoretically uncover untested reentrancies +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum OwnedStrKind { + Str, + Unicode, + Hex, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum OwnedLiteral { + Str { kind: OwnedStrKind, text: String }, + Number(alloy_primitives::U256), + Rational(String), + Address(String), + Bool(bool), + Err(String), +} + +impl From<&LitKind<'_>> for OwnedLiteral { + fn from(lit_kind: &LitKind<'_>) -> Self { + match lit_kind { + LitKind::Bool(b) => Self::Bool(*b), + LitKind::Number(n) => Self::Number(*n), + LitKind::Rational(r) => Self::Rational(r.to_string()), + LitKind::Address(addr) => Self::Address(addr.to_string()), + LitKind::Str(sk, bytesym, _extras) => { + let text = String::from_utf8_lossy(bytesym.as_byte_str()).into_owned(); + let kind = match sk { + StrKind::Str => OwnedStrKind::Str, + StrKind::Unicode => OwnedStrKind::Unicode, + StrKind::Hex => OwnedStrKind::Hex, + }; + Self::Str { kind, text } + } + LitKind::Err(_) => Self::Err("parse_error".to_string()), + } + } +} + +impl Display for OwnedLiteral { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Bool(val) => write!(f, "{val}"), + Self::Number(val) => write!(f, "{val}"), + Self::Rational(s) => write!(f, "{s}"), + Self::Address(s) => write!(f, "{s}"), + Self::Str { kind, text } => match kind { + OwnedStrKind::Str => write!(f, "\"{text}\""), + OwnedStrKind::Unicode => write!(f, "unicode\"{text}\""), + OwnedStrKind::Hex => write!(f, "hex\"{text}\""), + }, + Self::Err(s) => write!(f, "{s}"), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum MutationType { + // @todo Solar doesn't differentiate numeric type in LitKind (only on declaration?) -> for + // now, planket and let solc filter out the invalid mutants -> we might/should add a + // hashtable of the var to their underlying type (signed or not) so we avoid *a lot* of + // invalid mutants + /// For an initializer x, of type + /// bool: replace x with !x + /// uint: replace x with 0 + /// int: replace x with 0; replace x with -x (temp: this is mutated for uint as well) + /// + /// For a binary op y: apply BinaryOp(y) + Assignment(AssignVarTypes), + + /// For a binary op y in BinOpKind ("+", "-", ">=", etc) + /// replace y with each non-y in op + #[serde(serialize_with = "serialize_binop", deserialize_with = "deserialize_binop")] + BinaryOp(BinOpKind), + + /// For a delete expr x `delete foo`, replace x with `assert(true)` + DeleteExpression, + + /// replace "delegatecall" with "call" + ElimDelegate, + + /// Gambit doesn't implement nor define it? + FunctionCall, + + // /// For a if(x) condition x: + // /// replace x with true; replace x with false + // This mutation is not used anymore, as we mutate the condition as an expression, + // which will creates true/false mutant as well as more complex conditions (eg if(foo++ > + // --bar) ) IfStatementMutation, + /// For a require(x) condition: + /// replace x with true; replace x with false + // Same as for IfStatementMutation, the expression inside the require is mutated as an + // expression to handle increment etc + Require, + + // @todo review if needed -> this might creates *a lot* of combinations for super-polyadic fn + // tho only swapping same type (to avoid obvious compilation failure), but should + // take into account implicit casting too... + /// For 2 args of the same type x,y in a function args: + /// swap(x, y) + SwapArgumentsFunction, + + // @todo same remark as above, might end up in a space too big to explore + filtering out + // based on type + /// For an expr taking 2 expression x, y (x+y, x-y, x = x + ...): + /// swap(x, y) + SwapArgumentsOperator, + + /// For an unary operator x in UnOpKind (eg "++", "--", "~", "!"): + /// replace x with all other operator in op + /// Pre or post- are different UnOp + UnaryOperator(UnaryOpMutated), +} + +impl Display for MutationType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Assignment(kind) => match kind { + AssignVarTypes::Literal(lit) => write!(f, "{lit}"), + AssignVarTypes::Identifier(ident) => write!(f, "{ident}"), + }, + Self::BinaryOp(kind) => write!(f, "{}", kind.to_str()), + Self::DeleteExpression => write!(f, "assert(true)"), + Self::ElimDelegate => write!(f, "call"), + Self::UnaryOperator(mutated) => write!(f, "{mutated}"), + + Self::FunctionCall + | Self::Require + | Self::SwapArgumentsFunction + | Self::SwapArgumentsOperator => write!(f, ""), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum MutationResult { + Dead, + Alive, + Invalid, + Skipped, +} + +/// A given mutation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Mutant { + /// The path to the project root where this mutant (tries to) live + pub path: PathBuf, + #[serde(serialize_with = "serialize_span", deserialize_with = "deserialize_span")] + pub span: Span, + pub mutation: MutationType, +} + +// Custom serialization for Span (since solar_parse::ast::Span doesn't implement Serialize) +fn serialize_span(span: &Span, serializer: S) -> Result +where + S: serde::Serializer, +{ + use serde::Serialize; + #[derive(Serialize)] + struct SpanHelper { + lo: u32, + hi: u32, + } + SpanHelper { lo: span.lo().0, hi: span.hi().0 }.serialize(serializer) +} + +fn deserialize_span<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + use serde::Deserialize; + #[derive(Deserialize)] + struct SpanHelper { + lo: u32, + hi: u32, + } + let helper = SpanHelper::deserialize(deserializer)?; + Ok(Span::new( + solar_parse::interface::BytePos(helper.lo), + solar_parse::interface::BytePos(helper.hi), + )) +} + +impl Display for Mutant { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}:{}-{}:{}", + self.path.display(), + self.span.lo().0, + self.span.hi().0, + self.mutation + ) + } +} diff --git a/crates/forge/src/mutation/mutators/assignement_mutator.rs b/crates/forge/src/mutation/mutators/assignement_mutator.rs new file mode 100644 index 0000000000000..7669dc7dc4356 --- /dev/null +++ b/crates/forge/src/mutation/mutators/assignement_mutator.rs @@ -0,0 +1,119 @@ +use crate::mutation::{ + mutant::{Mutant, MutationType, OwnedLiteral}, + mutators::{MutationContext, Mutator}, + visitor::AssignVarTypes, +}; + +use alloy_primitives::U256; +use eyre::Result; +use solar_parse::ast::{ExprKind, Span}; + +pub struct AssignmentMutator; + +impl Mutator for AssignmentMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let (assign_var_type, replacement_span) = match extract_rhs_info(context) { + Some(info) => info, + None => return Ok(vec![]), // is_applicable should filter this + }; + + match assign_var_type { + AssignVarTypes::Literal(lit) => match lit { + OwnedLiteral::Bool(val) => Ok(vec![Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal( + OwnedLiteral::Bool(!val), + )), + path: context.path.clone(), + }]), + OwnedLiteral::Number(val) => Ok(vec![ + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal( + OwnedLiteral::Number(U256::ZERO), + )), + path: context.path.clone(), + }, + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal( + OwnedLiteral::Number(-val), + )), + path: context.path.clone(), + }, + ]), + // todo: should we bail instead of returning an empty vec? + OwnedLiteral::Str { .. } => Ok(vec![]), + OwnedLiteral::Rational(_) => Ok(vec![]), + OwnedLiteral::Address(_) => Ok(vec![]), + OwnedLiteral::Err(_) => Ok(vec![]), + _ => { + eyre::bail!("AssignmentMutator: unhandled literal kind on RHS: {:?}", lit) + } + }, + AssignVarTypes::Identifier(ident) => Ok(vec![ + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal( + OwnedLiteral::Number(U256::ZERO), + )), + path: context.path.clone(), + }, + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Identifier(format!( + "-{ident}" + ))), + path: context.path.clone(), + }, + ]), + } + } + + /// Match is the expr is an assign with a var definiton having a literal or identifier as + /// initializer + fn is_applicable(&self, context: &MutationContext<'_>) -> bool { + if let Some(expr) = context.expr { + if let ExprKind::Assign(_lhs, _op_opt, rhs_actual_expr) = &expr.kind { + matches!(rhs_actual_expr.kind, ExprKind::Lit(..) | ExprKind::Ident(..)) + } else { + false // Not an assign + } + } else if let Some(var_definition) = context.var_definition { + if let Some(init) = &var_definition.initializer { + matches!(&init.kind, ExprKind::Lit(..) | ExprKind::Ident(..)) + } else { + false // No initializer + } + } else { + false // Not an expression or var_definition + } + } +} + +fn extract_rhs_info<'ast>(context: &MutationContext<'ast>) -> Option<(AssignVarTypes, Span)> { + let relevant_expr_for_rhs = if let Some(var_definition) = context.var_definition { + var_definition.initializer.as_ref()? + } else if let Some(expr) = context.expr { + match &expr.kind { + ExprKind::Assign(_lhs, _op_opt, rhs_actual_expr) => &**rhs_actual_expr, + // If the context.expr is already what we want to get the type from + // (e.g. a simple Lit or Ident being passed directly, though is_applicable filters this) + ExprKind::Lit(..) | ExprKind::Ident(..) => expr, + _ => return None, + } + } else { + return None; // No var_definition or expr in context (shouldn't happen?) + }; + + match &relevant_expr_for_rhs.kind { + ExprKind::Lit(kind, _) => { + let owned = OwnedLiteral::from(&kind.kind); + Some((AssignVarTypes::Literal(owned), relevant_expr_for_rhs.span)) + } + ExprKind::Ident(val) => { + Some((AssignVarTypes::Identifier(val.to_string()), relevant_expr_for_rhs.span)) + } + _ => None, + } +} diff --git a/crates/forge/src/mutation/mutators/binary_op_mutator.rs b/crates/forge/src/mutation/mutators/binary_op_mutator.rs new file mode 100644 index 0000000000000..2b1ba7cc53d0c --- /dev/null +++ b/crates/forge/src/mutation/mutators/binary_op_mutator.rs @@ -0,0 +1,81 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType}; +use eyre::{OptionExt, Result}; +use solar_parse::ast::{BinOp, BinOpKind, ExprKind}; + +pub struct BinaryOpMutator; + +// @todo Add the other way to get there + +impl Mutator for BinaryOpMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let bin_op = get_bin_op(context)?; + let op = bin_op.kind; + + let operations_bools = vec![ + // Bool + BinOpKind::Lt, + BinOpKind::Le, + BinOpKind::Gt, + BinOpKind::Ge, + BinOpKind::Eq, + BinOpKind::Ne, + BinOpKind::Or, + BinOpKind::And, + ]; // this cover the "if" mutations, as every other mutant is tested, at least once + // @todo to optimize -> replace whole stmt (need new visitor override for visit_stmt tho) + // with true/false and skip operations_bools here (mayve some "level"/depth of + // mutation as param?) + + let operations_num_bitwise = vec![ + // Arithm + BinOpKind::Shr, + BinOpKind::Shl, + BinOpKind::Sar, + BinOpKind::BitAnd, + BinOpKind::BitOr, + BinOpKind::BitXor, + BinOpKind::Add, + BinOpKind::Sub, + BinOpKind::Pow, + BinOpKind::Mul, + BinOpKind::Div, + BinOpKind::Rem, + ]; + + let operations = + if operations_bools.contains(&op) { operations_bools } else { operations_num_bitwise }; + + Ok(operations + .into_iter() + .filter(|&kind| kind != op) + .map(|kind| Mutant { + span: context.span, + mutation: MutationType::BinaryOp(kind), + path: context.path.clone(), + }) + .collect()) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if ctxt.expr.is_none() { + return false; + } + + match ctxt.expr.unwrap().kind { + ExprKind::Binary(_, _, _) => true, + ExprKind::Assign(_, bin_op, _) => bin_op.is_some(), + _ => false, + } + } +} + +fn get_bin_op(ctxt: &MutationContext<'_>) -> Result { + let expr = ctxt.expr.ok_or_eyre("BinaryOpMutator: unexpected expression")?; + + match expr.kind { + ExprKind::Assign(_, Some(bin_op), _) => Ok(bin_op), + ExprKind::Binary(_, op, _) => Ok(op), + _ => eyre::bail!("BinaryOpMutator: unexpected expression kind"), + } +} diff --git a/crates/forge/src/mutation/mutators/delete_expression_mutator.rs b/crates/forge/src/mutation/mutators/delete_expression_mutator.rs new file mode 100644 index 0000000000000..e536bbbb3a184 --- /dev/null +++ b/crates/forge/src/mutation/mutators/delete_expression_mutator.rs @@ -0,0 +1,21 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType}; +use solar_parse::ast::ExprKind; + +use eyre::Result; + +pub struct DeleteExpressionMutator; + +impl Mutator for DeleteExpressionMutator { + fn generate_mutants(&self, ctxt: &MutationContext<'_>) -> Result> { + Ok(vec![Mutant { + span: ctxt.span, + mutation: MutationType::DeleteExpression, + path: ctxt.path.clone(), + }]) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if let Some(expr) = ctxt.expr { matches!(expr.kind, ExprKind::Delete(_)) } else { false } + } +} diff --git a/crates/forge/src/mutation/mutators/elim_delegate_mutator.rs b/crates/forge/src/mutation/mutators/elim_delegate_mutator.rs new file mode 100644 index 0000000000000..2b6c6b337852b --- /dev/null +++ b/crates/forge/src/mutation/mutators/elim_delegate_mutator.rs @@ -0,0 +1,38 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType}; + +use eyre::Result; +use solar_parse::ast::ExprKind; +use std::fmt::Display; + +pub struct ElimDelegateMutator; + +impl Mutator for ElimDelegateMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + Ok(vec![Mutant { + span: context.span, + mutation: MutationType::ElimDelegate, + path: context.path.clone(), + }]) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + ctxt.expr + .as_ref() + .and_then(|expr| match &expr.kind { + ExprKind::Call(callee, _) => Some(callee), + _ => None, + }) + .and_then(|callee| match &callee.kind { + ExprKind::Member(_, ident) => Some(ident), + _ => None, + }) + .is_some_and(|ident| ident.to_string() == "delegatecall") + } +} + +impl Display for ElimDelegateMutator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "") + } +} diff --git a/crates/forge/src/mutation/mutators/mod.rs b/crates/forge/src/mutation/mutators/mod.rs new file mode 100644 index 0000000000000..daedb674236c4 --- /dev/null +++ b/crates/forge/src/mutation/mutators/mod.rs @@ -0,0 +1,84 @@ +pub mod assignement_mutator; +pub mod binary_op_mutator; +pub mod delete_expression_mutator; +pub mod elim_delegate_mutator; +pub mod unary_op_mutator; + +pub mod mutator_registry; + +use eyre::Result; +use solar_parse::ast::{Expr, Span, VariableDefinition}; +use std::path::PathBuf; + +use crate::mutation::Mutant; + +pub trait Mutator: Send + Sync { + /// Generate all mutant corresponding to a given context + fn generate_mutants(&self, ctxt: &MutationContext<'_>) -> Result>; + /// True if a mutator can be applied to an expression/node + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool; +} + +#[derive(Debug)] +pub struct MutationContext<'a> { + pub path: PathBuf, + pub span: Span, + /// The expression to mutate + pub expr: Option<&'a Expr<'a>>, + + pub var_definition: Option<&'a VariableDefinition<'a>>, +} + +impl<'a> MutationContext<'a> { + pub fn builder() -> MutationContextBuilder<'a> { + MutationContextBuilder::new() + } +} + +pub struct MutationContextBuilder<'a> { + path: Option, + span: Option, + expr: Option<&'a Expr<'a>>, + var_definition: Option<&'a VariableDefinition<'a>>, +} + +impl<'a> MutationContextBuilder<'a> { + // Create a new empty builder + pub fn new() -> Self { + MutationContextBuilder { path: None, span: None, expr: None, var_definition: None } + } + + // Required + pub fn with_path(mut self, path: PathBuf) -> Self { + self.path = Some(path); + self + } + + // Required + pub fn with_span(mut self, span: Span) -> Self { + self.span = Some(span); + self + } + + // Optional + pub fn with_expr(mut self, expr: &'a Expr<'a>) -> Self { + self.expr = Some(expr); + self + } + + // Optional + pub fn with_var_definition(mut self, var_definition: &'a VariableDefinition<'a>) -> Self { + self.var_definition = Some(var_definition); + self + } + + pub fn build(self) -> Result, &'static str> { + let span = self.span.ok_or("Span is required for MutationContext")?; + let path = self.path.ok_or("Path is required for MutationContext")?; + + Ok(MutationContext { path, span, expr: self.expr, var_definition: self.var_definition }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/forge/src/mutation/mutators/mutator_registry.rs b/crates/forge/src/mutation/mutators/mutator_registry.rs new file mode 100644 index 0000000000000..c24321458dcf6 --- /dev/null +++ b/crates/forge/src/mutation/mutators/mutator_registry.rs @@ -0,0 +1,40 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::Mutant; + +use super::{ + assignement_mutator, binary_op_mutator, delete_expression_mutator, elim_delegate_mutator, + unary_op_mutator, +}; + +/// Registry of all available mutators (ie implementing the Mutator trait) +pub struct MutatorRegistry { + mutators: Vec>, +} + +impl MutatorRegistry { + pub fn default() -> Self { + let mut registry = Self { mutators: Vec::new() }; + + registry.mutators.push(Box::new(assignement_mutator::AssignmentMutator)); + registry.mutators.push(Box::new(binary_op_mutator::BinaryOpMutator)); + registry.mutators.push(Box::new(delete_expression_mutator::DeleteExpressionMutator)); + registry.mutators.push(Box::new(elim_delegate_mutator::ElimDelegateMutator)); + registry.mutators.push(Box::new(unary_op_mutator::UnaryOperatorMutator)); + + registry + } + + pub fn new_with_mutators(mutators: Vec>) -> Self { + Self { mutators } + } + + /// Find all applicable mutators for a given context and return the corresponding mutations + pub fn generate_mutations(&self, context: &MutationContext<'_>) -> Vec { + self.mutators + .iter() + .filter(|mutator| mutator.is_applicable(context)) + .filter_map(|mutator| mutator.generate_mutants(context).ok()) + .flatten() + .collect() + } +} diff --git a/crates/forge/src/mutation/mutators/tests/assignement_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/assignement_mutator_test.rs new file mode 100644 index 0000000000000..ff11e843d2009 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/assignement_mutator_test.rs @@ -0,0 +1,24 @@ +use crate::mutation::mutators::{ + assignement_mutator::AssignmentMutator, + tests::helper::{MutatorTestCase, MutatorTester}, +}; + +use rstest::*; + +impl MutatorTester for AssignmentMutator {} + +#[rstest] +#[case::assign_lit("x = y", Some(vec!["x = 0", "x = -y"]))] +#[case::assign_number("x = 123", Some(vec!["x = 0", "x = -123"]))] +#[case::assign_bool("x = true", Some(vec!["x = false"]))] +#[case::assign_bool("x = false", Some(vec!["x = true"]))] +#[case::assign_declare("uint256 x = 123", Some(vec!["uint256 x = 0", "uint256 x = -123"]))] +#[case::non_assign("a = b + c", None)] +fn test_mutator_assignment( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: AssignmentMutator = AssignmentMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + AssignmentMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/tests/binary_op_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/binary_op_mutator_test.rs new file mode 100644 index 0000000000000..84a874d017043 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/binary_op_mutator_test.rs @@ -0,0 +1,31 @@ +use crate::mutation::mutators::{ + binary_op_mutator::BinaryOpMutator, + tests::helper::{MutatorTestCase, MutatorTester}, +}; + +use rstest::*; + +impl MutatorTester for BinaryOpMutator {} + +#[rstest] +#[case::add("x + y", Some(vec!["x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::sub("x - y", Some(vec!["x + y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::mul("x * y", Some(vec!["x + y", "x - y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::div("x / y", Some(vec!["x + y", "x - y", "x * y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::modulus("x % y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::pow("x ** y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::bit_shift_left("x << y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::bit_shift_right("x >> y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::bit_shift_right_unsigned("x >>> y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x & y", "x | y", "x ^ y"]))] +#[case::bit_and("x & y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x | y", "x ^ y"]))] +#[case::bit_or("x | y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x ^ y"]))] +#[case::bit_xor("x ^ y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y"]))] +#[case::non_binary("a = true", None)] +fn test_mutator_bitwise( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: BinaryOpMutator = BinaryOpMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + BinaryOpMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/tests/delete_expression_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/delete_expression_mutator_test.rs new file mode 100644 index 0000000000000..ba94e5295837d --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/delete_expression_mutator_test.rs @@ -0,0 +1,20 @@ +use crate::mutation::mutators::{ + delete_expression_mutator::DeleteExpressionMutator, + tests::helper::{MutatorTestCase, MutatorTester}, +}; + +use rstest::*; + +impl MutatorTester for DeleteExpressionMutator {} + +#[rstest] +#[case::delete_expr("delete x", Some(vec!["x"]))] +#[case::non_delete("a = b + c", None)] +fn test_mutator_delete_expr( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: DeleteExpressionMutator = DeleteExpressionMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + DeleteExpressionMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/tests/elim_delegate_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/elim_delegate_mutator_test.rs new file mode 100644 index 0000000000000..05828beb24822 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/elim_delegate_mutator_test.rs @@ -0,0 +1,20 @@ +use crate::mutation::mutators::{ + elim_delegate_mutator::ElimDelegateMutator, + tests::helper::{MutatorTestCase, MutatorTester}, +}; + +use rstest::*; + +impl MutatorTester for ElimDelegateMutator {} + +#[rstest] +#[case::delegate_expr("address(this).delegatecall{value: 1 ether}(0)", Some(vec!["address(this).call{value: 1 ether}(0)"]))] +#[case::non_delegate("address(this).call{value: 1 ether}(0)", None)] +fn test_mutator_delegate_expr( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: ElimDelegateMutator = ElimDelegateMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + ElimDelegateMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/tests/helper.rs b/crates/forge/src/mutation/mutators/tests/helper.rs new file mode 100644 index 0000000000000..a5468a9e7b96b --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/helper.rs @@ -0,0 +1,59 @@ +use crate::mutation::{Session, mutators::Mutator, visitor::MutantVisitor}; +use solar_parse::{ + Parser, + ast::{Arena, interface::source_map::FileName, visit::Visit}, +}; + +use std::path::PathBuf; +pub struct MutatorTestCase<'a> { + /// @dev needs to be in a function, to avoid parsing error from solar + /// eg `let input = "function f() { x = 1; }"` to test x = 1 + pub input: &'a str, + /// All the mutations expected for this input, using this mutator + pub expected_mutations: Option>, +} + +pub trait MutatorTester { + fn test_mutator(mutator: M, test_case: MutatorTestCase<'_>) { + let sess = Session::builder().with_silent_emitter(None).build(); + + // let mut mutations: Vec = Vec::new(); + let mut mutant_visitor = MutantVisitor::new_with_mutators( + PathBuf::from(test_case.input), + vec![Box::new(mutator)], + ); + + let _ = sess.enter(|| -> solar_parse::interface::Result<()> { + let arena = Arena::new(); + + let mut parser = Parser::from_lazy_source_code( + &sess, + &arena, + FileName::Real(PathBuf::from(test_case.input)), + || Ok(test_case.input.to_string()), + )?; + + let ast = parser.parse_file().map_err(|e| e.emit())?; + + let _ = mutant_visitor.visit_source_unit(&ast); + + let mutations = mutant_visitor.mutation_to_conduct; + + // @todo test mutants content... + if let Some(expected) = test_case.expected_mutations { + assert_eq!(mutations.len(), expected.len()); + + for mutation in mutations { + assert!(expected.contains(&mutation.mutation.to_string().as_str())); + } + } else { + assert_eq!(mutations.len(), 0); + } + + Ok(()) + }); + } +} + +// Implement for unit test module +impl MutatorTester for () {} diff --git a/crates/forge/src/mutation/mutators/tests/mod.rs b/crates/forge/src/mutation/mutators/tests/mod.rs new file mode 100644 index 0000000000000..403ee2be95bd9 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/mod.rs @@ -0,0 +1,10 @@ +mod assignement_mutator_test; +mod binary_op_mutator_test; + +mod delete_expression_mutator_test; + +mod elim_delegate_mutator_test; + +mod helper; + +mod unary_op_mutator_test; diff --git a/crates/forge/src/mutation/mutators/tests/unary_op_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/unary_op_mutator_test.rs new file mode 100644 index 0000000000000..4c7ece82de2da --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/unary_op_mutator_test.rs @@ -0,0 +1,26 @@ +use crate::mutation::mutators::{ + tests::helper::{MutatorTestCase, MutatorTester}, + unary_op_mutator::UnaryOperatorMutator, +}; + +use rstest::*; + +impl MutatorTester for UnaryOperatorMutator {} + +#[rstest] +#[case::pre_inc("++x", Some(vec!["--x", "~x", "-x", "x++", "x--"]))] +#[case::pre_dec("--x", Some(vec!["++x", "~x", "-x", "x++", "x--"]))] +#[case::neg("-x", Some(vec!["++x", "--x", "~x", "x++", "x--"]))] +#[case::bit_not("~x", Some(vec!["++x", "--x", "-x", "x++", "x--"]))] +#[case::post_inc("x++",Some(vec!["++x", "--x", "~x", "-x", "x--"]))] +#[case::post_dec("x--",Some(vec!["++x", "--x", "~x", "-x", "x++"]))] +#[case::bool("!x", Some(vec!["x"]))] +#[case::non_unary("a = b + c", None)] +fn test_unary_op_mutator_arithmetic( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: UnaryOperatorMutator = UnaryOperatorMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + UnaryOperatorMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/unary_op_mutator.rs b/crates/forge/src/mutation/mutators/unary_op_mutator.rs new file mode 100644 index 0000000000000..7232cb2c8397a --- /dev/null +++ b/crates/forge/src/mutation/mutators/unary_op_mutator.rs @@ -0,0 +1,106 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType, UnaryOpMutated}; +use eyre::Result; +use solar_parse::ast::{ExprKind, LitKind, UnOpKind}; + +pub struct UnaryOperatorMutator; + +impl Mutator for UnaryOperatorMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let operations = vec![ + UnOpKind::PreInc, // number + UnOpKind::PreDec, // n + UnOpKind::Neg, // n @todo filter this one only for int + UnOpKind::BitNot, // n + ]; + + let post_fixed_operations = vec![UnOpKind::PostInc, UnOpKind::PostDec]; + + let expr = context.expr.unwrap(); + + let target_kind; + let op; + + match &expr.kind { + ExprKind::Unary(un_op, target) => { + target_kind = &target.kind; + op = un_op.kind; + } + _ => unreachable!(), + }; + + let target_content = match target_kind { + ExprKind::Lit(lit, _) => match &lit.kind { + LitKind::Bool(val) => val.to_string(), + LitKind::Number(val) => val.to_string(), + _ => String::new(), + }, + ExprKind::Ident(inner) => inner.to_string(), + ExprKind::Member(expr, ident) => { + match expr.kind { + ExprKind::Ident(inner) => { + format!("{}{}", ident.as_str(), inner.to_string()) + } // @todo not supporting something like a.b[0]++ + _ => String::new(), + } + } + _ => String::new(), + }; + + // Bool has only the Not operator as possible target -> we try removing it + if op == UnOpKind::Not { + return Ok(vec![Mutant { + span: expr.span, + mutation: MutationType::UnaryOperator(UnaryOpMutated::new( + target_content, + UnOpKind::Not, + )), + path: context.path.clone(), + }]); + } + + let mut mutations: Vec; + + mutations = operations + .into_iter() + .filter(|&kind| kind != op) + .map(|kind| { + let new_expression = format!("{}{}", kind.to_str(), target_content); + + let mutated = UnaryOpMutated::new(new_expression, kind); + + Mutant { + span: expr.span, + mutation: MutationType::UnaryOperator(mutated), + path: context.path.clone(), + } + }) + .collect(); + + mutations.extend(post_fixed_operations.into_iter().filter(|&kind| kind != op).map( + |kind| { + let new_expression = format!("{}{}", target_content, kind.to_str()); + + let mutated = UnaryOpMutated::new(new_expression, kind); + + Mutant { + span: expr.span, + mutation: MutationType::UnaryOperator(mutated), + path: context.path.clone(), + } + }, + )); + + Ok(mutations) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if let Some(expr) = ctxt.expr + && let ExprKind::Unary(_, _) = &expr.kind + { + return true; + } + + false + } +} diff --git a/crates/forge/src/mutation/reporter.rs b/crates/forge/src/mutation/reporter.rs new file mode 100644 index 0000000000000..6fd1c6a4d11e8 --- /dev/null +++ b/crates/forge/src/mutation/reporter.rs @@ -0,0 +1,71 @@ +use crate::mutation::MutationsSummary; +use comfy_table::{Cell, Color, Row, Table, modifiers::UTF8_ROUND_CORNERS}; +pub struct MutationReporter { + table: Table, +} + +impl Default for MutationReporter { + fn default() -> Self { + Self::new() + } +} + +impl MutationReporter { + pub fn new() -> Self { + let mut table = Table::new(); + + table.apply_modifier(UTF8_ROUND_CORNERS); + + table.set_header(vec![ + Cell::new("Status"), + Cell::new("# Mutants"), + Cell::new("% of Total"), + ]); + + Self { table } + } + + pub fn report(&mut self, summary: &MutationsSummary) { + let mut row = Row::new(); + row.add_cell(Cell::new("Survived").fg(Color::Red)) + .add_cell(Cell::new(summary.total_survived().to_string())) + .add_cell(Cell::new(format!( + "{:.2}%", + summary.total_survived() as f64 / summary.total_mutants() as f64 * 100. + ))); + self.table.add_row(row); + + row = Row::new(); + row.add_cell(Cell::new("Dead").fg(Color::Green)) + .add_cell(Cell::new(summary.total_dead().to_string())) + .add_cell(Cell::new(format!( + "{:.2}%", + summary.total_dead() as f64 / summary.total_mutants() as f64 * 100. + ))); + self.table.add_row(row); + + row = Row::new(); + row.add_cell(Cell::new("Invalid").fg(Color::Green)) + .add_cell(Cell::new(summary.total_invalid().to_string())) + .add_cell(Cell::new(format!( + "{:.2}%", + summary.total_invalid() as f64 / summary.total_mutants() as f64 * 100. + ))); + self.table.add_row(row); + + row = Row::new(); + row.add_cell(Cell::new("Skipped").fg(Color::Yellow)) + .add_cell(Cell::new(summary.total_skipped().to_string())) + .add_cell(Cell::new(format!( + "{:.2}%", + summary.total_skipped() as f64 / summary.total_mutants() as f64 * 100. + ))); + self.table.add_row(row); + + let _ = sh_println!("Total number of mutants generated: {}", summary.total_mutants()); + let _ = sh_println!("Mutation score: {:.2}%", summary.mutation_score()); + let _ = sh_println!("\n{}\n", self.table); + let _ = sh_println!("Dead mutants: {}\n", summary.dead()); + let _ = sh_println!("Survived mutants: {}\n", summary.survived()); + } +} diff --git a/crates/forge/src/mutation/visitor.rs b/crates/forge/src/mutation/visitor.rs new file mode 100644 index 0000000000000..1554fce1b1634 --- /dev/null +++ b/crates/forge/src/mutation/visitor.rs @@ -0,0 +1,104 @@ +use crate::mutation::{mutant::OwnedLiteral, mutators::Mutator}; +use solar_parse::ast::{Expr, Span, VariableDefinition, visit::Visit}; +use std::{ops::ControlFlow, path::PathBuf}; + +use crate::mutation::{ + mutant::Mutant, + mutators::{MutationContext, mutator_registry::MutatorRegistry}, +}; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum AssignVarTypes { + Literal(OwnedLiteral), + Identifier(String), /* not using Ident as the symbol is slow to convert as to_str() <-- + * maybe will have to switch back if validating more aggressively */ +} + +/// A visitor which collect all expression to mutate as well as the mutation types +pub struct MutantVisitor { + pub mutation_to_conduct: Vec, + pub mutator_registry: MutatorRegistry, + pub path: PathBuf, + pub span_filter: Option bool>>, + pub skipped_count: usize, +} + +impl MutantVisitor { + /// Use all mutator from registry::default + pub fn default(path: PathBuf) -> Self { + Self { + mutation_to_conduct: Vec::new(), + mutator_registry: MutatorRegistry::default(), + path, + span_filter: None, + skipped_count: 0, + } + } + + /// Use only a set of mutators + pub fn new_with_mutators(path: PathBuf, mutators: Vec>) -> Self { + Self { + mutation_to_conduct: Vec::new(), + mutator_registry: MutatorRegistry::new_with_mutators(mutators), + path, + span_filter: None, + skipped_count: 0, + } + } + + /// Set a filter function to skip certain spans (for adaptive mutation testing) + pub fn with_span_filter(mut self, filter: F) -> Self + where + F: Fn(Span) -> bool + 'static, + { + self.span_filter = Some(Box::new(filter)); + self + } +} + +impl<'ast> Visit<'ast> for MutantVisitor { + type BreakValue = (); + + fn visit_variable_definition( + &mut self, + var: &'ast VariableDefinition<'ast>, + ) -> ControlFlow { + // Check if we should skip this span (adaptive mutation testing) + if let Some(ref filter) = self.span_filter + && filter(var.span) + { + self.skipped_count += 1; + return self.walk_variable_definition(var); + } + + let context = MutationContext::builder() + .with_path(self.path.clone()) + .with_span(var.span) + .with_var_definition(var) + .build() + .unwrap(); + + self.mutation_to_conduct.extend(self.mutator_registry.generate_mutations(&context)); + self.walk_variable_definition(var) + } + + fn visit_expr(&mut self, expr: &'ast Expr<'ast>) -> ControlFlow { + // Check if we should skip this span (adaptive mutation testing) + if let Some(ref filter) = self.span_filter + && filter(expr.span) + { + self.skipped_count += 1; + return self.walk_expr(expr); + } + + let context = MutationContext::builder() + .with_path(self.path.clone()) + .with_span(expr.span) + .with_expr(expr) + .build() + .unwrap(); + + self.mutation_to_conduct.extend(self.mutator_registry.generate_mutations(&context)); + self.walk_expr(expr) + } +} diff --git a/crates/forge/tests/cli/config.rs b/crates/forge/tests/cli/config.rs index 1f30e7d31d4b7..94905e295c9a0 100644 --- a/crates/forge/tests/cli/config.rs +++ b/crates/forge/tests/cli/config.rs @@ -255,6 +255,7 @@ forgetest!(can_extract_config_values, |prj, cmd| { path_pattern_inverse: None, coverage_pattern_inverse: None, test_failures_file: "test-cache/test-failures".into(), + mutation_dir: "test-cache/mutation".into(), threads: None, show_progress: false, fuzz: FuzzConfig { diff --git a/crates/forge/tests/it/main.rs b/crates/forge/tests/it/main.rs index c8890af3d4614..45e30e91583bb 100644 --- a/crates/forge/tests/it/main.rs +++ b/crates/forge/tests/it/main.rs @@ -8,6 +8,7 @@ mod fs; mod fuzz; mod inline; mod invariant; +mod mutation; mod repros; mod spec; mod table; diff --git a/crates/forge/tests/it/mutation.rs b/crates/forge/tests/it/mutation.rs new file mode 100644 index 0000000000000..16066f30a821a --- /dev/null +++ b/crates/forge/tests/it/mutation.rs @@ -0,0 +1,119 @@ +// use forge::mutation::MutationHandler; +// use forge_script::ScriptArgs; +// use foundry_common::shell::{ColorChoice, OutputFormat, OutputMode, Shell}; +// use std::sync::Arc; + +// #[tokio::test(flavor = "multi_thread")] +// async fn test_mutation_test_lifecycle() { +// let contract = r#" +// // SPDX-License-Identifier: UNLICENSED +// pragma solidity ^0.8.13; + +// contract Counter { +// uint256 public number; + +// function increment() public { +// number++; +// // This should result in 5 mutants: ++number, --number, -number, ~number, +// number-- // -number should be invalid +// // ++number should be alive +// // the rest should be dead +// } +// }"#; + +// let test = r#" +// // SPDX-License-Identifier: UNLICENSED +// pragma solidity ^0.8.13; + +// // Avoid having to manage a libs folder +// import {Counter} from "../src/Counter.sol"; + +// contract CounterTest { +// Counter public counter; + +// function setUp() public { +// counter = new Counter(); +// } + +// function test_Increment() public { +// uint256 _countBefore = counter.number(); + +// counter.increment(); + +// assert(counter.number() == _countBefore + 1); +// } +// }"#; + +// let temp_dir = tempfile::tempdir().unwrap(); + +// let src_dir = temp_dir.path().join("src"); +// std::fs::create_dir_all(&src_dir).expect("Failed to create src directory"); + +// let test_dir = temp_dir.path().join("test"); +// std::fs::create_dir_all(&test_dir).expect("Failed to create test directory"); + +// let cache_dir = temp_dir.path().join("cache"); +// std::fs::create_dir_all(&cache_dir).expect("Failed to create test directory"); + +// let out_dir = temp_dir.path().join("out"); +// std::fs::create_dir_all(&out_dir).expect("Failed to create test directory"); + +// std::fs::write(&src_dir.join("Counter.sol"), contract) +// .unwrap_or_else(|_| panic!("Failed to write to target file {:?}", &src_dir)); + +// std::fs::write(&test_dir.join("CounterTest.t.sol"), test) +// .unwrap_or_else(|_| panic!("Failed to write to target file {:?}", &src_dir)); + +// let mut config = foundry_config::Config::default(); +// config.cache_path = cache_dir; +// config.out = out_dir; +// config.src = src_dir.clone(); +// config.test = test_dir.clone(); + +// let mut mutation_handler = MutationHandler::new(src_dir.join("Counter.sol"), +// Arc::new(config)); + +// mutation_handler.read_source_contract(); +// mutation_handler.generate_ast().await; +// mutation_handler.create_mutation_folders(); +// let mutants = mutation_handler.generate_and_compile().await; + +// // Test if we compile and collect the valid/invalid mutants +// assert_eq!(mutants.iter().filter(|(_, output)| output.is_none()).count(), 1); +// assert_eq!(mutants.iter().filter(|(_, output)| output.is_some()).count(), 4); + +// // @todo run the tests +// let mut invalids = 0; +// let mut alive = 0; +// let mut dead = 0; + +// // Create a new shell to suppress any script output +// let shell = Shell::new_with(OutputFormat::Json, OutputMode::Quiet, ColorChoice::Never, 0); +// shell.set(); + +// // Run the tests as scripts, for convenience +// for mutant in mutants { +// if mutant.1.is_some() { +// let result = ScriptArgs { +// path: mutant.0.path.join("test/CounterTest.t.sol").to_string_lossy().to_string(), +// sig: "test_Increment".to_string(), +// args: vec![], +// ..Default::default() +// } +// .run_script() +// .await; + +// if result.is_err() { +// dead += 1; +// } else { +// alive += 1; +// } +// } else { +// invalids += 1; +// } +// } + +// assert_eq!(invalids, 1); +// assert_eq!(alive, 1); +// assert_eq!(dead, 3); +// }