Run tests on native (#3730)

Author: ksew1

.github/workflows/ci.yml

@@ -35,7 +35,8 @@ jobs:
         with:
           tool: [email protected]
       - name: Build and archive tests
-        run: cargo nextest archive --release -p forge --features skip_plugin_checks --archive-file 'nextest-archive-${{ runner.os }}.tar.zst'
+        # TODO: run native only on scheduled workflow
+        run: cargo nextest archive --release -p forge --features skip_plugin_checks,run-native --archive-file 'nextest-archive-${{ runner.os }}.tar.zst'
       - name: Upload archive to workflow
         uses: actions/upload-artifact@v4
         with:
@@ -175,7 +176,8 @@ jobs:
       - uses: software-mansion/setup-scarb@v1
       - uses: software-mansion/setup-universal-sierra-compiler@v1
       - name: Run Cheatnet tests
-        run: cargo test --release -p cheatnet
+        # TODO: run native only on scheduled workflow
+        run: cargo test --release -p cheatnet --features run-native
 
   test-data-transformer:
     name: Test Data Transformer
@@ -195,7 +197,8 @@ jobs:
       - uses: software-mansion/setup-scarb@v1
       - uses: software-mansion/setup-universal-sierra-compiler@58146c4184fa6ec5e8aaf02309ab85e35f782ed0 # v1.0.0
       - name: Run Forge Debugging tests
-        run: cargo test --release -p forge --features debugging --test main e2e::debugging
+        # TODO: run native only on scheduled workflow
+        run: cargo test --release -p forge --features debugging,run-native --test main e2e::debugging
 
   test-forge-scarb-plugin:
     name: Test Forge Scarb Plugin
@@ -261,7 +264,8 @@ jobs:
       - uses: ./.github/actions/setup-rust-llvm
       - uses: software-mansion/setup-scarb@v1
       - uses: software-mansion/setup-universal-sierra-compiler@v1
-      - run: cargo test --release -p scarb-api
+        # TODO: run native only on scheduled workflow
+      - run: cargo test --release -p scarb-api --features run-native
 
   scarbfmt:
     runs-on: ubuntu-latest

crates/cheatnet/Cargo.toml

@@ -5,6 +5,7 @@ edition.workspace = true
 
 [features]
 testing = []
+run-native = []
 
 [dependencies]
 anyhow.workspace = true

crates/cheatnet/tests/common/mod.rs

@@ -90,7 +90,10 @@ pub fn get_contracts() -> ContractsData {
         &target_dir,
         package,
         &ui,
-        CompilationOpts::default(),
+        CompilationOpts {
+            use_test_target_contracts: false,
+            run_native: cfg!(feature = "run-native"),
+        },
     )
     .unwrap();
     ContractsData::try_from(contracts).unwrap()

crates/forge/Cargo.toml

@@ -13,6 +13,7 @@ debugging = []
 assert_non_exact_gas = ["test_utils/assert_non_exact_gas"]
 skip_plugin_checks = []
 interact-with-state = []
+run-native = ["test_utils/run-native"]
 
 [dependencies]
 anyhow.workspace = true

crates/forge/src/lib.rs

@@ -151,7 +151,7 @@ pub struct TestArgs {
     #[command(flatten)]
     trace_args: TraceArgs,
 
-    /// Run contracts on `cairo-native` instead of the default `cairo-vm`.
+    /// Run contracts on `cairo-native` instead of the default `cairo-vm`. This will set `tracked-resource` to `sierra-gas`.
     ///
     /// Note: Only contracts execution through native is supported, test code itself will still run on `cairo-vm`.
     #[arg(long)]
@@ -184,7 +184,7 @@ pub struct TestArgs {
     include_ignored: bool,
 
     /// Display more detailed info about used resources
-    #[arg(long, conflicts_with = "run_native")]
+    #[arg(long)]
     detailed_resources: bool,
 
     /// Control when colored output is used
@@ -231,6 +231,20 @@ pub struct TestArgs {
     scarb_args: ScarbArgs,
 }
 
+impl TestArgs {
+    /// Adjust dependent arguments based on related flags.
+    ///
+    /// This function mutates the `TestArgs` instance to enforce logical coherence
+    /// between fields.
+    pub fn normalize(&mut self) {
+        // Force using `SierraGas` as tracked resource when running with `cairo-native`,
+        // as otherwise it would run on vm.
+        if self.run_native {
+            self.tracked_resource = ForgeTrackedResource::SierraGas;
+        }
+    }
+}
+
 #[derive(Parser, Debug)]
 pub struct ScarbArgs {
     #[command(flatten)]
@@ -306,7 +320,8 @@ pub fn main_execution(ui: Arc<UI>) -> Result<ExitStatus> {
 
             Ok(ExitStatus::Success)
         }
-        ForgeSubcommand::Test { args } => {
+        ForgeSubcommand::Test { mut args } => {
+            args.normalize();
             check_requirements(false, args.tracked_resource, &ui)?;
             let cores = if let Ok(available_cores) = available_parallelism() {
                 available_cores.get()

crates/forge/test_utils/Cargo.toml

@@ -7,6 +7,7 @@ edition.workspace = true
 
 [features]
 assert_non_exact_gas = []
+run-native = []
 
 [dependencies]
 walkdir.workspace = true

crates/forge/test_utils/src/runner.rs

@@ -109,7 +109,10 @@ impl Contract {
             &artifacts_dir,
             package,
             ui,
-            CompilationOpts::default(),
+            CompilationOpts {
+                use_test_target_contracts: false,
+                run_native: cfg!(feature = "run-native"),
+            },
         )
         .unwrap()
         .remove(&self.name)

crates/forge/tests/e2e/clean.rs

@@ -21,6 +21,7 @@ struct CleanComponentsState {
 }
 
 #[test]
+#[cfg_attr(feature = "run-native", ignore = "Native doesn't support coverage yet")]
 fn test_clean_coverage() {
     let temp_dir = setup_package("coverage_project");
 
@@ -54,6 +55,7 @@ fn test_clean_coverage() {
 }
 
 #[test]
+#[cfg_attr(feature = "run-native", ignore = "Native doesn't support profiler yet")]
 fn test_clean_profile() {
     let temp_dir = setup_package("coverage_project");
 
@@ -118,6 +120,10 @@ fn test_clean_cache() {
 }
 
 #[test]
+#[cfg_attr(
+    feature = "run-native",
+    ignore = "Native doesn't support trace, coverage and profiler yet"
+)]
 fn test_clean_all() {
     let temp_dir = setup_package("coverage_project");
 
@@ -152,7 +158,7 @@ fn test_clean_all_and_component() {
     let clean_components_state = CleanComponentsState {
         coverage: false,
         cache: true,
-        trace: true,
+        trace: false,
         profile: false,
     };
     generate_clean_components(clean_components_state, &temp_dir);

crates/forge/tests/e2e/common/mod.rs

@@ -1,9 +1,11 @@
+#[cfg(not(feature = "run-native"))]
 use cairo_annotations::trace_data::{
     CallTraceNode as ProfilerCallTraceNode, CallTraceV1 as ProfilerCallTrace,
 };
 
 pub mod runner;
 
+#[cfg(not(feature = "run-native"))]
 pub fn get_trace_from_trace_node(trace_node: &ProfilerCallTraceNode) -> &ProfilerCallTrace {
     if let ProfilerCallTraceNode::EntryPointCall(trace) = trace_node {
         trace

crates/forge/tests/e2e/common/runner.rs

@@ -31,7 +31,29 @@ pub fn snforge_test_bin_path() -> PathBuf {
     cargo_bin!("snforge").to_path_buf()
 }
 
+/// Returns a command that runs `snforge test` in the given temporary directory.
+/// If the `run-native` feature is enabled, it adds the `--run-native` flag.
 pub(crate) fn test_runner<T: AsRef<Path>>(temp_dir: T) -> SnapboxCommand {
+    if cfg!(feature = "run-native") {
+        test_runner_native(temp_dir)
+    } else {
+        test_runner_vm(temp_dir)
+    }
+}
+
+/// Returns a command that runs `snforge test --run-native` in the given temporary directory.
+///
+/// This is useful for testing behavior that occurs only when the `--run-native` flag is passed.
+/// If the behavior is not specific to native execution, use `test_runner` instead.
+pub(crate) fn test_runner_native<T: AsRef<Path>>(temp_dir: T) -> SnapboxCommand {
+    runner(temp_dir).arg("test").arg("--run-native")
+}
+
+/// Returns a command that runs `snforge test` in the given temporary directory.
+///
+/// This is useful for testing behavior that occurs only in the VM execution.
+/// If the behavior is not specific to VM execution, use `test_runner` instead.
+pub(crate) fn test_runner_vm<T: AsRef<Path>>(temp_dir: T) -> SnapboxCommand {
     runner(temp_dir).arg("test")
 }