Integrated Buck2-Change-Detector to orion-server and orion workflow (#1721)

* feat: integrate buck2-change-detector into orion build progress

Signed-off-by: Acfboy <acfboyu@outlook.com>

* fix: some implementation issues

Signed-off-by: Acfboy <acfboyu@outlook.com>

* fix: clippy

Signed-off-by: Acfboy <acfboyu@outlook.com>

---------

Signed-off-by: Acfboy <acfboyu@outlook.com>
Co-authored-by: Quanyi Ma <eli@patch.sh>

Author: Acfboy

ceres/src/model/change_list.rs

@@ -140,6 +140,7 @@ pub struct ClFilesRes {
 
 impl From<ClDiffFile> for ClFilesRes {
     fn from(value: ClDiffFile) -> Self {
+        // if change, please modify `ceres/src/pack/monorepo.rs` also.
         match value {
             ClDiffFile::New(path, sha) => Self {
                 path: path.to_string_lossy().to_string(),

ceres/src/model/cl.rs

@@ -0,0 +1 @@
+

ceres/src/model/mod.rs

@@ -1,6 +1,7 @@
 pub mod blame;
 pub mod buck;
 pub mod change_list;
+pub mod cl;
 pub mod commit;
 pub mod conversation;
 pub mod dynamic_sidebar;

ceres/src/pack/monorepo.rs

@@ -13,7 +13,11 @@ use std::{
 use tokio::sync::{RwLock, mpsc};
 use tokio_stream::wrappers::ReceiverStream;
 
-use bellatrix::{Bellatrix, orion_client::BuildInfo, orion_client::OrionBuildRequest};
+use bellatrix::{
+    Bellatrix,
+    orion_client::OrionBuildRequest,
+    orion_client::{BuildInfo, ProjectRelativePath, Status},
+};
 use callisto::{
     entity_ext::generate_link, mega_cl, mega_refs, raw_blob, sea_orm_active_enums::ConvTypeEnum,
 };
@@ -587,6 +591,29 @@ impl MonoRepo {
         let mono_stg = self.storage.mono_storage();
         let mono_api_service: MonoApiService = self.into();
 
+        let mut path = Some(cl_path);
+        let mut path_q = Vec::new();
+        while let Some(p) = path {
+            path_q.push(p);
+            path = p.parent();
+        }
+        if path_q.len() > 2 {
+            path_q.pop();
+            path_q.pop();
+
+            let p = path_q[path_q.len() - 1];
+            if p.parent().is_some()
+                && let Some(tree) = tree_ops::search_tree_by_path(&mono_api_service, p, None)
+                    .await
+                    .ok()
+                    .flatten()
+                && let Some(buck) = self.try_extract_buck(tree, cl_path)
+            {
+                return Ok(vec![buck]);
+            };
+            return Ok(vec![]);
+        }
+
         let mut search_trees: Vec<(PathBuf, Tree)> = vec![];
 
         let diff_trees = self.diff_trees_from_cl().await?;
@@ -608,23 +635,6 @@ impl MonoRepo {
             }
         }
 
-        // no buck file found
-        if res.is_empty() {
-            let mut path = Some(cl_path);
-            while let Some(p) = path {
-                if p.parent().is_some()
-                    && let Some(tree) = tree_ops::search_tree_by_path(&mono_api_service, p, None)
-                        .await
-                        .ok()
-                        .flatten()
-                    && let Some(buck) = self.try_extract_buck(tree, cl_path)
-                {
-                    return Ok(vec![buck]);
-                };
-
-                path = p.parent();
-            }
-        }
         Ok(res)
     }
 
@@ -910,7 +920,38 @@ impl MonoRepo {
                     self.path
                 );
             } else {
+                let old_files = self.get_commit_blobs(&cl_info.from_hash).await?;
+                let new_files = self.get_commit_blobs(&cl_info.to_hash).await?;
+                let cl_diff_files = self.cl_files_list(old_files, new_files.clone()).await?;
+
+                let cl_base = PathBuf::from(&cl_info.path);
+                let changes = cl_diff_files
+                    .into_iter()
+                    .map(|m| {
+                        let mut item: crate::model::change_list::ClFilesRes = m.into();
+                        item.path = cl_base.join(item.path).to_string_lossy().to_string();
+                        item
+                    })
+                    .collect::<Vec<_>>();
+
                 for buck_file in buck_files {
+                    let path_str = buck_file.path.to_str().expect("path is not valid UTF-8");
+                    let counter_changes: Vec<_> = changes
+                        .iter()
+                        .filter(|&s| PathBuf::from(&s.path).starts_with(&buck_file.path))
+                        .map(|s| {
+                            let path = ProjectRelativePath::from_abs(&s.path, path_str).unwrap();
+                            if s.action == "new" {
+                                Status::Added(path)
+                            } else if s.action == "deleted" {
+                                Status::Removed(path)
+                            } else if s.action == "modified" {
+                                Status::Modified(path)
+                            } else {
+                                unreachable!()
+                            }
+                        })
+                        .collect();
                     let req = OrionBuildRequest {
                         repo: buck_file.path.to_str().unwrap().to_string(),
                         cl_link: link.to_string(),
@@ -921,6 +962,7 @@ impl MonoRepo {
                             buck_hash: buck_file.buck.to_string(),
                             buckconfig_hash: buck_file.buck_config.to_string(),
                             args: Some(vec![]),
+                            changes: counter_changes,
                         }],
                     };
                     let bellatrix = self.bellatrix.clone();
@@ -935,6 +977,23 @@ impl MonoRepo {
         check_reg.run_checks(cl_info.clone().into()).await?;
         Ok(())
     }
+
+    pub async fn get_commit_blobs(
+        &self,
+        commit_hash: &str,
+    ) -> Result<Vec<(PathBuf, SHA1)>, MegaError> {
+        let api_service: MonoApiService = self.into();
+        api_service.get_commit_blobs(commit_hash).await
+    }
+
+    pub async fn cl_files_list(
+        &self,
+        old_files: Vec<(PathBuf, SHA1)>,
+        new_files: Vec<(PathBuf, SHA1)>,
+    ) -> Result<Vec<crate::model::change_list::ClDiffFile>, MegaError> {
+        let api_service: MonoApiService = self.into();
+        api_service.cl_files_list(old_files, new_files).await
+    }
 }
 
 type DiffResult = Vec<(PathBuf, Option<SHA1>, Option<SHA1>)>;

orion-server/bellatrix/src/orion_client/mod.rs

@@ -1,10 +1,44 @@
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)]
+pub enum Status<Path: Clone> {
+    Modified(Path),
+    Added(Path),
+    Removed(Path),
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Deserialize, Serialize)]
+pub struct ProjectRelativePath(String);
+
+impl ProjectRelativePath {
+    pub fn new(path: &str) -> Self {
+        Self(path.to_owned())
+    }
+
+    pub fn from_abs(abs_path: &str, base: &str) -> Option<Self> {
+        let opt = abs_path
+            .strip_prefix(base)
+            .map(|s| s.trim_start_matches("/"));
+        opt.map(|s| Self(s.to_owned()))
+    }
+}
+
+impl<Path: Clone> Status<Path> {
+    pub fn path(&self) -> Path {
+        match self {
+            Self::Added(p) => p.clone(),
+            Self::Modified(p) => p.clone(),
+            Self::Removed(p) => p.clone(),
+        }
+    }
+}
 
 #[derive(Serialize, Debug)]
 pub struct BuildInfo {
     pub buck_hash: String,
     pub buckconfig_hash: String,
     pub args: Option<Vec<String>>,
+    pub changes: Vec<Status<ProjectRelativePath>>,
 }
 
 #[derive(Serialize, Debug)]

orion-server/src/api.rs

@@ -334,9 +334,6 @@ pub async fn task_handler(
     State(state): State<AppState>,
     Json(req): Json<TaskRequest>,
 ) -> impl IntoResponse {
-    // for now we do not extract from file, just use the fixed build target.
-    let target = "//...".to_string();
-
     // create task id
     let task_id = Uuid::now_v7();
 
@@ -374,7 +371,7 @@ pub async fn task_handler(
                 &req.repo,
                 req.cl,
                 build.clone(),
-                target.clone(),
+                String::new(),
             )
             .await;
             results.push(result);
@@ -386,7 +383,6 @@ pub async fn task_handler(
                     task_id,
                     &req.cl_link,
                     build.clone(),
-                    target.clone(),
                     req.repo.clone(),
                     req.cl,
                 )
@@ -426,6 +422,9 @@ pub async fn task_handler(
 }
 
 /// Handle immediate task dispatch logic (original task_handler logic)
+///
+/// # Note
+/// The `target` field is deprecated, only remain for compatibility reasons.
 async fn handle_immediate_task_dispatch(
     state: AppState,
     task_id: Uuid,
@@ -476,8 +475,8 @@ async fn handle_immediate_task_dispatch(
     // Create build information structure
     let build_info = BuildInfo {
         repo: repo.to_string(),
-        target: target.clone(),
         args: req.args.clone(),
+        changes: req.changes.clone(),
         start_at: chrono::Utc::now(),
         cl: cl.to_string(),
         _worker_id: chosen_id.clone(),
@@ -508,7 +507,7 @@ async fn handle_immediate_task_dispatch(
     let msg: WSMessage = WSMessage::Task {
         id: build_id.to_string(),
         repo: repo.to_string(),
-        target,
+        changes: req.changes.clone(),
         args: req.args.clone(),
         cl_link: cl_link.to_string(),
     };

orion-server/src/scheduler.rs

@@ -1,6 +1,7 @@
 use crate::model::builds;
 use chrono::FixedOffset;
 use dashmap::DashMap;
+use orion::repo::sapling::status::{ProjectRelativePath, Status};
 use orion::ws::WSMessage;
 use rand::Rng;
 use sea_orm::ActiveModelTrait;
@@ -23,6 +24,7 @@ pub struct BuildRequest {
     pub buck_hash: String,
     pub buckconfig_hash: String,
     pub args: Option<Vec<String>>,
+    pub changes: Vec<Status<ProjectRelativePath>>,
 }
 
 /// Pending task waiting for dispatch
@@ -34,7 +36,6 @@ pub struct PendingTask {
     pub repo: String,
     pub cl: i64,
     pub request: BuildRequest,
-    pub target: String,
     pub created_at: Instant,
 }
 
@@ -134,9 +135,9 @@ pub struct TaskQueueStats {
 #[allow(dead_code)]
 pub struct BuildInfo {
     pub repo: String,
-    pub target: String,
     pub args: Option<Vec<String>>,
     pub start_at: DateTimeUtc,
+    pub changes: Vec<Status<ProjectRelativePath>>,
     pub cl: String,
     pub _worker_id: String,
     pub log_file: Arc<Mutex<std::fs::File>>,
@@ -241,7 +242,6 @@ impl TaskScheduler {
         task_id: Uuid,
         cl_link: &str,
         request: BuildRequest,
-        target: String,
         repo: String,
         cl: i64,
     ) -> Result<Uuid, String> {
@@ -252,7 +252,6 @@ impl TaskScheduler {
             cl_link: cl_link.to_string(),
             build_id,
             request,
-            target,
             created_at: Instant::now(),
             repo,
             cl,
@@ -372,9 +371,9 @@ impl TaskScheduler {
         // Create build information
         let build_info = BuildInfo {
             repo: pending_task.repo.clone(),
-            target: pending_task.target.clone(),
             args: pending_task.request.args.clone(),
             start_at: chrono::Utc::now(),
+            changes: pending_task.request.changes.clone(),
             cl: pending_task.cl.to_string(),
             _worker_id: chosen_id.clone(),
             log_file,
@@ -406,9 +405,9 @@ impl TaskScheduler {
         let msg = WSMessage::Task {
             id: pending_task.build_id.to_string(),
             repo: pending_task.repo,
-            target: pending_task.target,
             args: pending_task.request.args,
             cl_link: pending_task.cl_link.to_string(),
+            changes: pending_task.request.changes.clone(),
         };
 
         // Send task to worker
@@ -659,8 +658,8 @@ mod tests {
                 buck_hash: "hash1".to_string(),
                 buckconfig_hash: "config1".to_string(),
                 args: None,
+                changes: vec![],
             },
-            target: "target1".to_string(),
             created_at: Instant::now(),
             repo: "/test/repo".to_string(),
             cl: 123456,
@@ -674,8 +673,8 @@ mod tests {
                 buck_hash: "hash2".to_string(),
                 buckconfig_hash: "config2".to_string(),
                 args: None,
+                changes: vec![],
             },
-            target: "target2".to_string(),
             created_at: Instant::now(),
             repo: "/test2/repo".to_string(),
             cl: 123457,
@@ -712,8 +711,8 @@ mod tests {
                 buck_hash: "hash".to_string(),
                 buckconfig_hash: "config".to_string(),
                 args: None,
+                changes: vec![],
             },
-            target: "target".to_string(),
             created_at: Instant::now(),
             repo: "/test/repo".to_string(),
             cl: 123456,

orion/Cargo.toml

@@ -8,13 +8,28 @@ axum = { workspace = true, features = ["macros"] }
 tokio = { workspace = true, features = ["rt-multi-thread", "fs", "process"] }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
-serde = { workspace = true, features = ["derive"] }
-uuid = { workspace = true, features = ["v7"] }
+serde = { workspace = true, features = ["derive", "rc"] }
 futures-util = { workspace = true }
-dashmap = { workspace = true }
 once_cell = { workspace = true }
 dotenvy = { workspace = true }
 tokio-tungstenite = { workspace = true , features = ["native-tls"]}
 tungstenite = { workspace = true }
 serde_json = { workspace = true }
 reqwest = { workspace = true, features = ["json", "rustls-tls"] }
+uuid = { workspace = true, features = ["v4"] }
+anyhow = "1.0"
+itertools = "0.13.0"
+argfile = "0.1.5"
+clap = {version = "4.1.4"}
+equivalent = "1.0.0"
+futures = "0.3.30"
+parse-display = "0.8.2"
+rayon = "1.6.1"
+tempfile = "3.1.0"
+audit = { path = "./audit" }
+td_util = { path = "./td_util" }
+td_util_buck = { path = "./buck" }
+thiserror = "1.0.36"
+utoipa-axum.workspace = true
+utoipa.workspace = true
+utoipa-swagger-ui = { workspace = true, features = ["axum"] }