apollo_compile_to_casm,apollo_compile_to_native,scripts: move compiler install to runtime

.github/actions/install_compiler_binaries/action.yml

@@ -0,0 +1,29 @@
+name: Install compiler binaries
+description: Install starknet-sierra-compile and starknet-native-compile (cached).
+
+inputs:
+  build_mode:
+    description: "debug or release"
+    required: false
+    default: "debug"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Cache compiler binaries
+      id: cache
+      uses: actions/cache@v4
+      with:
+        path: target/${{ inputs.build_mode }}/shared_executables
+        key: compiler-binaries-${{ runner.os }}-${{ inputs.build_mode }}-${{ hashFiles('crates/apollo_infra_utils/src/cairo_compiler_version.rs', 'crates/apollo_compile_to_native/src/constants.rs') }}
+
+    - name: Install compiler binaries
+      if: steps.cache.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        BUILD_FLAGS=""
+        if [ "${{ inputs.build_mode }}" = "release" ]; then
+          BUILD_FLAGS="--release"
+        fi
+        cargo run ${BUILD_FLAGS} -p apollo_compile_to_casm --bin install_starknet_sierra_compile
+        cargo run ${BUILD_FLAGS} -p apollo_compile_to_native --bin install_starknet_native_compile

.github/workflows/blockifier_ci.yml

@@ -80,6 +80,8 @@ jobs:
       - uses: ./.github/actions/bootstrap
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
+      # Pre-install compiler binaries so tests don't each try to install them.
+      - uses: ./.github/actions/install_compiler_binaries
       # cairo_native is not activated by any workspace crate; test the build.
       - run: cargo build -p blockifier --features cairo_native
       - run: cargo test -p blockifier --features cairo_native
@@ -116,6 +118,10 @@ jobs:
           cache-on-failure: true
           cache-workspace-crates: true
 
+      # Pre-install compiler binaries; ignore failure if base branch lacks install binaries.
+      - name: "Install compiler binaries (base branch)"
+        run: scripts/install_compiler_binaries.sh --release || true
+
       # Benchmark the base branch code.
       - run: cargo run -p bench_tools -- run --package blockifier --out /tmp/base_results
 
@@ -124,6 +130,11 @@ jobs:
         with:
           clean: false
 
+      # Pre-install compiler binaries so benchmarks can invoke them.
+      - uses: ./.github/actions/install_compiler_binaries
+        with:
+          build_mode: release
+
       # Benchmark the current branch and compare to the previous run.
       - run: |
           cargo run -p bench_tools -- run-and-compare \

.github/workflows/blockifier_reexecution_ci.yml

@@ -46,6 +46,10 @@ jobs:
       - uses: "google-github-actions/setup-gcloud@v2"
       - run: echo "REEXECUTION_INPUT_FILES_PREFIX=$(cat ./crates/blockifier_reexecution/resources/offline_reexecution_files_prefix)" >> $GITHUB_ENV
       - run: gcloud storage cp -r gs://reexecution_artifacts/$REEXECUTION_INPUT_FILES_PREFIX/resources/* ./crates/blockifier_reexecution/resources/
+      # Pre-install compiler binaries so tests don't each try to install them.
+      - uses: ./.github/actions/install_compiler_binaries
+        with:
+          build_mode: release
       # Run blockifier re-execution tests.
       - run: cargo test --release -p blockifier_reexecution -- --include-ignored
       # Compile the rpc-tests, without running them.

.github/workflows/main.yml

@@ -110,6 +110,9 @@ jobs:
         run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
       - run: pip install -r scripts/requirements.txt
 
+      # Pre-install compiler binaries so tests don't each try to install them.
+      - uses: ./.github/actions/install_compiler_binaries
+
       - name: "Run tests"
         run: |
           scripts/run_tests.py --command test --changes_only --include_dependencies --commit_id ${{ github.event.pull_request.base.sha }}
@@ -140,6 +143,9 @@ jobs:
         run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
       - run: pip install -r scripts/requirements.txt
 
+      # Pre-install compiler binaries so integration tests don't each try to install them.
+      - uses: ./.github/actions/install_compiler_binaries
+
       - name: "Run integration tests"
         run: |
           scripts/run_tests.py --command integration --changes_only --include_dependencies --commit_id ${{ github.event.pull_request.base.sha }}

.github/workflows/main_nightly.yml

@@ -55,6 +55,9 @@ jobs:
         run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
       - run: pip install -r scripts/requirements.txt
 
+      # Pre-install compiler binaries so tests don't each try to install them.
+      - uses: ./.github/actions/install_compiler_binaries
+
       - name: "Run codecov"
         run: cargo llvm-cov --codecov --output-path codecov.json
         env:
@@ -103,6 +106,9 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
+      # Pre-install compiler binaries so tests don't each try to install them.
+      - uses: ./.github/actions/install_compiler_binaries
+
       # Run feature combo test.
       - name: "Run feature combo on all crates."
         run: scripts/run_feature_combos_test.py
@@ -141,6 +147,10 @@ jobs:
           LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin
         run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
       - run: pip install -r scripts/requirements.txt
+
+      # Pre-install compiler binaries so tests don't each try to install them.
+      - uses: ./.github/actions/install_compiler_binaries
+
       - name: "Run integration tests pull request"
         run: |
           scripts/run_tests.py --command integration --is_nightly

crates/apollo_compilation_utils/Cargo.toml

@@ -28,7 +28,3 @@ thiserror.workspace = true
 apollo_infra_utils.workspace = true
 assert_matches.workspace = true
 rstest.workspace = true
-
-[build-dependencies]
-apollo_infra_utils.workspace = true
-tempfile.workspace = true

crates/apollo_compilation_utils/src/build_utils.rs

@@ -1,3 +1,5 @@
+use std::io::ErrorKind;
+use std::path::Path;
 use std::process::Command;
 
 use tempfile::TempDir;
@@ -8,9 +10,8 @@ pub fn install_compiler_binary(
     binary_name: &str,
     required_version: &str,
     cargo_install_args: &[&str],
-    out_dir: &std::path::Path,
 ) {
-    let binary_path = binary_path(out_dir, binary_name);
+    let binary_path = binary_path(binary_name);
     match Command::new(&binary_path).args(["--version"]).output() {
         Ok(binary_version) => {
             let binary_version = String::from_utf8(binary_version.stdout)
@@ -49,18 +50,23 @@ pub fn install_compiler_binary(
     }
 
     // Move the '{binary_name}' executable to a shared location.
-    std::fs::create_dir_all(shared_folder_dir(out_dir))
+    std::fs::create_dir_all(shared_folder_dir())
         .expect("Failed to create shared executables folder");
-    let move_command_status = Command::new("mv")
-        .args([post_install_file_path.as_os_str(), binary_path.as_os_str()])
-        .status()
-        .expect("Failed to perform mv command.");
-
-    if !move_command_status.success() {
-        panic!("Failed to move the {binary_name} binary to the shared folder.");
-    }
-
-    std::fs::remove_dir_all(temp_cargo_path).expect("Failed to remove the cargo directory.");
+    move_binary(&post_install_file_path, &binary_path);
 
     println!("Successfully set executable file: {:?}", binary_path.display());
 }
+
+fn move_binary(source: &Path, target: &Path) {
+    match std::fs::rename(source, target) {
+        Ok(()) => {}
+        Err(error) if error.kind() == ErrorKind::CrossesDevices => {
+            std::fs::copy(source, target).expect("Failed to copy the binary to the shared folder.");
+            std::fs::remove_file(source)
+                .expect("Failed to remove the temporary binary after copy.");
+        }
+        Err(error) => {
+            panic!("Failed to move binary to shared folder: {error}");
+        }
+    }
+}

crates/apollo_compilation_utils/src/paths.rs

@@ -1,19 +1,51 @@
-// Note: This module includes path resolution functions that are needed during build and run times.
-// It must not contain functionality that is available in only in one of these modes. Specifically,
-// it must avoid relying on env variables such as 'CARGO_*' or 'OUT_DIR'.
-
-fn target_dir(out_dir: &std::path::Path) -> std::path::PathBuf {
-    out_dir
-        .ancestors()
-        .nth(3)
-        .expect("Failed to navigate up three levels from OUT_DIR")
-        .to_path_buf()
+use std::ffi::OsStr;
+use std::path::{Path, PathBuf};
+
+fn profile_dir_from_executable(executable_path: &Path) -> PathBuf {
+    let executable_dir =
+        executable_path.parent().expect("Current executable path has no parent directory");
+    if executable_dir.file_name() == Some(OsStr::new("deps")) {
+        executable_dir
+            .parent()
+            .expect("Failed to navigate from deps to profile directory")
+            .to_path_buf()
+    } else {
+        executable_dir.to_path_buf()
+    }
+}
+
+fn profile_dir() -> PathBuf {
+    let executable_path = std::env::current_exe().expect("Failed to resolve current executable.");
+    profile_dir_from_executable(&executable_path)
+}
+
+pub fn shared_folder_dir() -> PathBuf {
+    profile_dir().join("shared_executables")
 }
 
-pub fn shared_folder_dir(out_dir: &std::path::Path) -> std::path::PathBuf {
-    target_dir(out_dir).join("shared_executables")
+pub fn binary_path(binary_name: &str) -> PathBuf {
+    shared_folder_dir().join(binary_name)
 }
 
-pub fn binary_path(out_dir: &std::path::Path, binary_name: &str) -> std::path::PathBuf {
-    shared_folder_dir(out_dir).join(binary_name)
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+
+    use super::profile_dir_from_executable;
+
+    #[test]
+    fn resolve_profile_dir_for_test_binary() {
+        let executable_path = PathBuf::from("/tmp/target/debug/deps/test_binary");
+        let profile_dir = profile_dir_from_executable(&executable_path);
+
+        assert_eq!(profile_dir, PathBuf::from("/tmp/target/debug"));
+    }
+
+    #[test]
+    fn resolve_profile_dir_for_regular_binary() {
+        let executable_path = PathBuf::from("/tmp/target/release/install_binary");
+        let profile_dir = profile_dir_from_executable(&executable_path);
+
+        assert_eq!(profile_dir, PathBuf::from("/tmp/target/release"));
+    }
 }

crates/apollo_compile_to_casm/Cargo.toml

@@ -16,6 +16,7 @@ workspace = true
 apollo_compilation_utils.workspace = true
 apollo_compile_to_casm_types.workspace = true
 apollo_infra.workspace = true
+apollo_infra_utils.workspace = true
 apollo_metrics.workspace = true
 apollo_proc_macros.workspace = true
 apollo_sierra_compilation_config.workspace = true
@@ -29,11 +30,6 @@ tracing.workspace = true
 
 [dev-dependencies]
 apollo_compilation_utils = { workspace = true, features = ["testing"] }
-apollo_infra_utils.workspace = true
 assert_matches.workspace = true
 mempool_test_utils.workspace = true
 regex.workspace = true
-
-[build-dependencies]
-apollo_compilation_utils.workspace = true
-apollo_infra_utils.workspace = true

crates/apollo_compile_to_casm/src/bin/install_starknet_sierra_compile.rs

@@ -0,0 +1,11 @@
+use apollo_compilation_utils::build_utils::install_compiler_binary;
+use apollo_compile_to_casm::constants::CAIRO_LANG_BINARY_NAME;
+use apollo_infra_utils::cairo_compiler_version::CAIRO1_COMPILER_VERSION;
+
+fn main() {
+    install_compiler_binary(
+        CAIRO_LANG_BINARY_NAME,
+        CAIRO1_COMPILER_VERSION,
+        &[CAIRO_LANG_BINARY_NAME, "--version", CAIRO1_COMPILER_VERSION],
+    );
+}

crates/apollo_compile_to_casm/src/compiler.rs

@@ -19,7 +19,7 @@ pub struct SierraToCasmCompiler {
 
 impl SierraToCasmCompiler {
     pub fn new(config: SierraCompilationConfig) -> Self {
-        let path_to_binary = binary_path(&out_dir(), CAIRO_LANG_BINARY_NAME);
+        let path_to_binary = binary_path(CAIRO_LANG_BINARY_NAME);
         info!("Using Sierra compiler binary at: {:?}", path_to_binary);
         Self { config, path_to_binary }
     }
@@ -47,8 +47,3 @@ impl SierraToCasmCompiler {
         Ok(serde_json::from_slice::<CasmContractClass>(&stdout)?)
     }
 }
-
-// Returns the OUT_DIR. This function is only operable at run time.
-fn out_dir() -> PathBuf {
-    env!("RUNTIME_ACCESSIBLE_OUT_DIR").into()
-}

crates/apollo_compile_to_casm/src/constants.rs

@@ -2,4 +2,4 @@
 // not contain functionality that is available in only in one of these modes. Specifically, it
 // must avoid relying on env variables such as 'CARGO_*' or 'OUT_DIR'.
 
-pub(crate) const CAIRO_LANG_BINARY_NAME: &str = "starknet-sierra-compile";
+pub const CAIRO_LANG_BINARY_NAME: &str = "starknet-sierra-compile";

crates/apollo_compile_to_native/Cargo.toml

@@ -23,7 +23,3 @@ assert_matches.workspace = true
 mempool_test_utils.workspace = true
 rstest.workspace = true
 toml_test_utils.workspace = true
-
-[build-dependencies]
-apollo_compilation_utils.workspace = true
-apollo_infra_utils.workspace = true