feat(apollo_starknet_os_program): define and implement static Program

Author: dorimedini-starkware

.github/workflows/committer_and_os_cli_push.yml

@@ -65,6 +65,19 @@ jobs:
           COMMIT_SHA: ${{ github.event.after }}
         run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV
 
+      # Setup pypy and link to the location expected by .cargo/config.toml.
+      # Python + requirements are needed to compile the OS.
+      - uses: actions/setup-python@v5
+        id: setup-pypy
+        with:
+          python-version: "pypy3.9"
+          cache: 'pip'
+      - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9
+      - env:
+          LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin
+        run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+      - run: pip install -r scripts/requirements.txt
+
       - name: Build CLI binary
         run: ./build_native_in_docker.sh rustup toolchain install && cargo build -p starknet_committer_and_os_cli -r --bin starknet_committer_and_os_cli --target-dir CLI_TARGET
 

.github/workflows/committer_ci.yml

@@ -40,6 +40,19 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
+      # Setup pypy and link to the location expected by .cargo/config.toml.
+      # Python + requirements are needed to compile the OS.
+      - uses: actions/setup-python@v5
+        id: setup-pypy
+        with:
+          python-version: "pypy3.9"
+          cache: 'pip'
+      - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9
+      - env:
+          LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin
+        run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+      - run: pip install -r scripts/requirements.txt
+
       - id: auth
         uses: "google-github-actions/auth@v2"
         with:
@@ -61,6 +74,19 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
+      # Setup pypy and link to the location expected by .cargo/config.toml.
+      # Python + requirements are needed to compile the OS.
+      - uses: actions/setup-python@v5
+        id: setup-pypy
+        with:
+          python-version: "pypy3.9"
+          cache: 'pip'
+      - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9
+      - env:
+          LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin
+        run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+      - run: pip install -r scripts/requirements.txt
+
       # Download the old benchmark inputs.
       - id: auth
         uses: "google-github-actions/auth@v2"

Cargo.lock

@@ -37,26 +37,38 @@ pub enum Cairo0CompilerVersionError {
         "{0}. Are you in the venv? If not, run the following commands:\n{}",
         *ENTER_VENV_INSTRUCTIONS
     )]
-    NotFound(String),
+    CompilerNotFound(String),
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum Cairo0CompilerError {
+    #[error(transparent)]
+    Cairo0CompilerVersion(#[from] Cairo0CompilerVersionError),
+    #[error("Cairo root path not found at {0:?}.")]
+    CairoRootNotFound(PathBuf),
+    #[error("Failed to compile the program. Error: {0}.")]
+    CompileError(String),
+    #[error("Invalid path unicode: {0:?}.")]
+    InvalidPath(PathBuf),
+    #[error(transparent)]
+    Io(#[from] std::io::Error),
+    #[error("No file found at path {0:?}.")]
+    SourceFileNotFound(PathBuf),
 }
 
 pub fn cairo0_compilers_correct_version() -> Result<(), Cairo0CompilerVersionError> {
     for compiler in [CAIRO0_COMPILE, STARKNET_COMPILE_DEPRECATED] {
         let version = match Command::new(compiler).arg("--version").output() {
             Ok(output) => String::from_utf8_lossy(&output.stdout).to_string(),
             Err(error) => {
-                return Err(Cairo0CompilerVersionError::NotFound(format!(
+                return Err(Cairo0CompilerVersionError::CompilerNotFound(format!(
                     "Failed to get {compiler} version: {error}."
                 )));
             }
         };
-        if version
-            .trim()
-            .replace("==", " ")
-            .split(" ")
-            .nth(1)
-            .ok_or(Cairo0CompilerVersionError::NotFound("No compiler version found.".to_string()))?
-            != EXPECTED_CAIRO0_VERSION
+        if version.trim().replace("==", " ").split(" ").nth(1).ok_or(
+            Cairo0CompilerVersionError::CompilerNotFound("No compiler version found.".to_string()),
+        )? != EXPECTED_CAIRO0_VERSION
         {
             return Err(Cairo0CompilerVersionError::IncorrectVersion {
                 compiler: compiler.to_string(),
@@ -69,6 +81,39 @@ pub fn cairo0_compilers_correct_version() -> Result<(), Cairo0CompilerVersionErr
     Ok(())
 }
 
+/// Compile a Cairo0 program.
+pub fn compile_cairo0_program(
+    path_to_main: PathBuf,
+    cairo_root_path: PathBuf,
+) -> Result<Vec<u8>, Cairo0CompilerError> {
+    cairo0_compilers_correct_version()?;
+    if !path_to_main.exists() {
+        return Err(Cairo0CompilerError::SourceFileNotFound(path_to_main));
+    }
+    if !cairo_root_path.exists() {
+        return Err(Cairo0CompilerError::CairoRootNotFound(cairo_root_path));
+    }
+    let mut compile_command = Command::new(CAIRO0_COMPILE);
+    compile_command.args([
+        path_to_main.to_str().ok_or(Cairo0CompilerError::InvalidPath(path_to_main.clone()))?,
+        "--debug_info_with_source",
+        "--cairo_path",
+        cairo_root_path
+            .to_str()
+            .ok_or(Cairo0CompilerError::InvalidPath(cairo_root_path.clone()))?,
+    ]);
+    let compile_output = compile_command.output()?;
+
+    // Verify output.
+    if !compile_output.status.success() {
+        return Err(Cairo0CompilerError::CompileError(
+            String::from_utf8_lossy(&compile_output.stderr).trim().to_string(),
+        ));
+    }
+
+    Ok(compile_output.stdout)
+}
+
 /// Verifies that the required Cairo0 compiler is available; panics if unavailable.
 /// For use in tests only. If cairo0 compiler verification is required in business logic, use
 /// `crate::cairo0_compiler::cairo0_compilers_correct_version` instead.
@@ -78,7 +123,9 @@ pub fn verify_cairo0_compiler_deps() {
         Ok(_) => {
             return;
         }
-        Err(Cairo0CompilerVersionError::NotFound(_)) => "no installed cairo-lang found".to_string(),
+        Err(Cairo0CompilerVersionError::CompilerNotFound(_)) => {
+            "no installed cairo-lang found".to_string()
+        }
         Err(Cairo0CompilerVersionError::IncorrectVersion { existing, .. }) => {
             format!("installed version: {existing}")
         }

crates/apollo_infra_utils/src/cairo0_compiler.rs

@@ -14,6 +14,7 @@ dump_source_files = []
 workspace = true
 
 [dependencies]
+cairo-vm.workspace = true
 serde_json.workspace = true
 
 [build-dependencies]

crates/apollo_starknet_os_program/Cargo.toml

@@ -0,0 +1,24 @@
+use std::path::PathBuf;
+
+use apollo_infra_utils::cairo0_compiler::{compile_cairo0_program, Cairo0CompilerError};
+use apollo_infra_utils::compile_time_cargo_manifest_dir;
+
+/// Compile the StarkNet OS program.
+pub fn compile_starknet_os() -> Vec<u8> {
+    let cairo_root_path = PathBuf::from(compile_time_cargo_manifest_dir!()).join("src/cairo");
+    let os_main_path = cairo_root_path.join("starkware/starknet/core/os/os.cairo");
+    match compile_cairo0_program(os_main_path, cairo_root_path) {
+        Ok(bytes) => bytes,
+        Err(Cairo0CompilerError::Cairo0CompilerVersion(error)) => {
+            panic!(
+                "Failed to verify correct cairo0 package installation. Please make sure you do \
+                 not have a conflicting installation in your {}/bin directory.\nOriginal error: \
+                 {error:?}.",
+                std::env::var("CARGO_HOME").unwrap_or("${CARGO_HOME}".to_string())
+            )
+        }
+        Err(other_error) => {
+            panic!("Failed to compile the StarkNet OS program. Error:\n{other_error}.")
+        }
+    }
+}

crates/apollo_starknet_os_program/build/compile_program.rs

@@ -28,7 +28,7 @@ fn get_cairo_file_map_recursive(entry: DirEntry) -> HashMap<String, String> {
 
 /// Find all files with a .cairo extension in the `src` directory, insert them into a map and dump
 /// the map as JSON to the specified location.
-pub fn dump_source_files(dump_to: PathBuf) {
+pub fn dump_source_files(dump_to: &PathBuf) {
     println!("cargo::warning=Dumping OS source files...");
 
     // Recursively fetch all cairo files and contents, and convert the paths to relative paths.
@@ -47,6 +47,6 @@ pub fn dump_source_files(dump_to: PathBuf) {
 
     // Serialize and dump the map to the specified location.
     let serialized = serde_json::to_string(&map_without_prefixes).unwrap();
-    std::fs::write(&dump_to, serialized)
+    std::fs::write(dump_to, serialized)
         .unwrap_or_else(|error| panic!("Failed to write to {dump_to:?}: {error:?}."));
 }

crates/apollo_starknet_os_program/build/dump_source.rs

@@ -1,15 +1,22 @@
-#[cfg(feature = "dump_source_files")]
 use std::path::PathBuf;
 
+mod compile_program;
 #[cfg(feature = "dump_source_files")]
 mod dump_source;
 
 /// Build script for the `apollo_starknet_os_program` crate.
 /// Recompiles the OS program if the source files change.
 /// Optionally, also exposes all source cairo files in a mapping from file path to contents.
 fn main() {
+    let out_dir = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR not set."));
+
     #[cfg(feature = "dump_source_files")]
-    let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR not set.");
-    #[cfg(feature = "dump_source_files")]
-    dump_source::dump_source_files(PathBuf::from(out_dir).join("cairo_files_map.json"));
+    dump_source::dump_source_files(&out_dir.join("cairo_files_map.json"));
+
+    println!("cargo::warning=Compiling Starknet OS program...");
+    let starknet_os_bytes = compile_program::compile_starknet_os();
+    println!("cargo::warning=Done. Writing compiled bytes to output directory.");
+    let starknet_os_bytes_path = out_dir.join("starknet_os_bytes");
+    std::fs::write(&starknet_os_bytes_path, &starknet_os_bytes)
+        .expect("Failed to write the compiled OS bytes to the output directory.");
 }

crates/apollo_starknet_os_program/build/main.rs

@@ -1,10 +1,19 @@
 #[cfg(feature = "dump_source_files")]
 use std::collections::HashMap;
-#[cfg(feature = "dump_source_files")]
 use std::sync::LazyLock;
 
+use cairo_vm::types::program::Program;
+
 #[cfg(feature = "dump_source_files")]
 pub static CAIRO_FILES_MAP: LazyLock<HashMap<String, String>> = LazyLock::new(|| {
     serde_json::from_str(include_str!(concat!(env!("OUT_DIR"), "/cairo_files_map.json")))
         .unwrap_or_else(|error| panic!("Failed to deserialize cairo_files_map.json: {error:?}."))
 });
+
+pub static OS_PROGRAM: LazyLock<Program> = LazyLock::new(|| {
+    Program::from_bytes(
+        include_bytes!(concat!(env!("OUT_DIR"), "/starknet_os_bytes")),
+        Some("main"),
+    )
+    .expect("Failed to load the OS bytes.")
+});

crates/apollo_starknet_os_program/src/lib.rs

@@ -8,6 +8,7 @@ FROM ubuntu:24.04 AS base
 
 COPY scripts/install_build_tools.sh .
 COPY scripts/dependencies.sh .
+COPY scripts/requirements.txt .
 COPY rust-toolchain.toml .
 
 RUN apt update && apt -y install \
@@ -24,5 +25,11 @@ RUN ./install_build_tools.sh
 RUN rustup toolchain install
 RUN cargo install cargo-chef
 
+# Define python venv and install python packages.
+ENV VIRTUAL_ENV=/opt/sequencer_venv
+RUN python3 -m venv ${VIRTUAL_ENV}
+RUN ${VIRTUAL_ENV}/bin/pip install -r requirements.txt
+ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
+
 # Cleanup.
-RUN rm -f install_build_tools.sh dependencies.sh
+RUN rm -f install_build_tools.sh dependencies.sh requirements.txt