feat(apollo_starknet_os_program): add feature for exposing cairo file source map (#6259)

* feat(apollo_starknet_os_program): cairo file list test + fixer

* feat(apollo_starknet_os_program): source file dump on build

Signed-off-by: Dori Medini <dori@starkware.co>

---------

Signed-off-by: Dori Medini <dori@starkware.co>

Author: dorimedini-starkware

.gitignore

@@ -12,7 +12,7 @@ CLI_TARGET/
 *.pdb
 
 *.egg-info
-build
+/build
 dist
 target
 */.vscode/*
@@ -30,6 +30,9 @@ __pycache__/
 .idea/
 **/.venv
 
+# Native blockifier artifacts.
+/crates/native_blockifier/build
+
 # Python artifacts.
 scripts/__pycache__
 monitoring_venv/

Cargo.lock

@@ -5,6 +5,17 @@ edition.workspace = true
 repository.workspace = true
 license-file.workspace = true
 description = "The source (Cairo) code of the Starknet OS."
+build = "build/main.rs"
+
+[features]
+dump_source_files = []
 
 [lints]
 workspace = true
+
+[dependencies]
+serde_json.workspace = true
+
+[build-dependencies]
+apollo_infra_utils.workspace = true
+serde_json.workspace = true

crates/apollo_starknet_os_program/Cargo.toml

@@ -0,0 +1,52 @@
+use std::collections::HashMap;
+use std::fs::DirEntry;
+use std::path::PathBuf;
+
+use apollo_infra_utils::compile_time_cargo_manifest_dir;
+
+/// Utility function to recursively find all cairo files.
+fn get_cairo_file_map_recursive(entry: DirEntry) -> HashMap<String, String> {
+    let file_type = entry.file_type().unwrap();
+    let path = entry.path();
+    if file_type.is_dir() {
+        std::fs::read_dir(path)
+            .unwrap()
+            .flat_map(|entry| get_cairo_file_map_recursive(entry.unwrap()).into_iter())
+            .collect()
+    } else {
+        assert!(file_type.is_file());
+        if path.extension().unwrap_or_default() == "cairo" {
+            HashMap::from([(
+                path.to_str().unwrap().to_string(),
+                std::fs::read_to_string(path).unwrap(),
+            )])
+        } else {
+            HashMap::new()
+        }
+    }
+}
+
+/// Find all files with a .cairo extension in the `src` directory, insert them into a map and dump
+/// the map as JSON to the specified location.
+pub fn dump_source_files(dump_to: PathBuf) {
+    println!("cargo::warning=Dumping OS source files...");
+
+    // Recursively fetch all cairo files and contents, and convert the paths to relative paths.
+    let base_path = PathBuf::from(compile_time_cargo_manifest_dir!()).join("src/cairo");
+    let base_path_string = base_path.to_str().unwrap();
+    let map_without_prefixes: HashMap<String, String> = std::fs::read_dir(base_path_string)
+        .unwrap()
+        .flat_map(|entry| get_cairo_file_map_recursive(entry.unwrap()))
+        .map(|(path, contents)| {
+            assert!(path.starts_with(base_path_string));
+            let path =
+                path.strip_prefix(base_path_string).unwrap().strip_prefix("/").unwrap().to_string();
+            (path, contents)
+        })
+        .collect();
+
+    // Serialize and dump the map to the specified location.
+    let serialized = serde_json::to_string(&map_without_prefixes).unwrap();
+    std::fs::write(&dump_to, serialized)
+        .unwrap_or_else(|error| panic!("Failed to write to {dump_to:?}: {error:?}."));
+}

crates/apollo_starknet_os_program/build/dump_source.rs

@@ -0,0 +1,15 @@
+#[cfg(feature = "dump_source_files")]
+use std::path::PathBuf;
+
+#[cfg(feature = "dump_source_files")]
+mod dump_source;
+
+/// Build script for the `apollo_starknet_os_program` crate.
+/// Recompiles the OS program if the source files change.
+/// Optionally, also exposes all source cairo files in a mapping from file path to contents.
+fn main() {
+    #[cfg(feature = "dump_source_files")]
+    let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR not set.");
+    #[cfg(feature = "dump_source_files")]
+    dump_source::dump_source_files(PathBuf::from(out_dir).join("cairo_files_map.json"));
+}

crates/apollo_starknet_os_program/build/main.rs

@@ -1 +1,10 @@
+#[cfg(feature = "dump_source_files")]
+use std::collections::HashMap;
+#[cfg(feature = "dump_source_files")]
+use std::sync::LazyLock;
 
+#[cfg(feature = "dump_source_files")]
+pub static CAIRO_FILES_MAP: LazyLock<HashMap<String, String>> = LazyLock::new(|| {
+    serde_json::from_str(include_str!(concat!(env!("OUT_DIR"), "/cairo_files_map.json")))
+        .unwrap_or_else(|error| panic!("Failed to deserialize cairo_files_map.json: {error:?}."))
+});