Merge pull request #136 from filecoin-project/feat/cache-application-db

Feat/cache-application-db

Author: clriesco

fplus-database/src/database/applications.rs

@@ -1,17 +1,61 @@
-use sea_orm::{entity::*, query::*, DbErr};
+use sea_orm::{entity::*, query::*, DbBackend, DbErr};
 use crate::models::applications::{Column, ActiveModel, Entity as Application, Model as ApplicationModel};
 use crate::get_database_connection;
 use sha1::{Sha1, Digest};
+use chrono::{DateTime, Utc, TimeZone};
 
 /**
  * Get all applications from the database
  * 
  * # Returns
  * @return Result<Vec<ApplicationModel>, sea_orm::DbErr> - The result of the operation
  */
-pub async fn get_applications() ->Result<Vec<ApplicationModel>, sea_orm::DbErr> {
+pub async fn get_applications() -> Result<Vec<ApplicationModel>, sea_orm::DbErr> {
     let conn = get_database_connection().await?;
-    Application::find().all(&conn).await
+
+    //Get all applications from the database. 
+    //Distinct on is not supported in sea_orm yet, so we have to use raw SQL
+    let app_data = JsonValue::find_by_statement(Statement::from_sql_and_values(
+        DbBackend::Postgres,
+        r#"
+            SELECT DISTINCT ON (owner, repo, id) 
+                a.id, 
+                a.owner, 
+                a.repo, 
+                a.pr_number, 
+                a.application, 
+                a.updated_at, 
+                a.sha,
+                a.path
+            FROM 
+                applications a 
+            ORDER BY 
+                a.owner, 
+                a.repo, 
+                a.id, 
+                a.pr_number DESC
+            "#,
+        [],
+    ))
+    .all(&conn)
+    .await?;
+
+    //Iterate over the results and convert them to ApplicationModel
+    let mut applications: Vec<ApplicationModel> = Vec::new();
+    for app in app_data {
+        applications.push(ApplicationModel {
+            id: app.get("id").unwrap().as_str().unwrap().to_string(),
+            owner: app.get("owner").unwrap().as_str().unwrap().to_string(),
+            repo: app.get("repo").unwrap().as_str().unwrap().to_string(),
+            pr_number: app.get("pr_number").unwrap().as_i64().unwrap() as i64,
+            application: Some(app.get("application").unwrap().as_str().unwrap().to_string()),
+            updated_at: Utc.from_utc_datetime(&app.get("updated_at").unwrap().as_str().unwrap().parse::<DateTime<Utc>>().unwrap().naive_utc()),
+            sha: Some(app.get("sha").unwrap().as_str().unwrap().to_string()),
+            path: Some(app.get("path").unwrap().as_str().unwrap().to_string()),
+        });
+    }
+    Ok(applications)
+   
 }
 
 /**
@@ -24,12 +68,22 @@ pub async fn get_applications() ->Result<Vec<ApplicationModel>, sea_orm::DbErr>
  * # Returns
  * @return Result<Vec<ApplicationModel>, sea_orm::DbErr> - The result of the operation
  */
-pub async fn get_merged_applications(owner: String, repo: String) -> Result<Vec<ApplicationModel>, sea_orm::DbErr> {
+pub async fn get_merged_applications(owner: Option<String>, repo: Option<String>) -> Result<Vec<ApplicationModel>, sea_orm::DbErr> {
     let conn = get_database_connection().await?;
-    Application::find()
-        .filter(Column::Owner.contains(owner))
-        .filter(Column::Repo.contains(repo))
-        .filter(Column::PrNumber.eq(0))
+    let mut query = Application::find()
+        .filter(Column::PrNumber.eq(0));
+    if let Some(owner) = owner.clone() {
+        query = query.filter(Column::Owner.contains(owner));
+    }
+    if let Some(repo) = repo {
+        if owner.is_none() {
+            return Err(DbErr::Custom(format!("Owner is required to get merged applications").into()));
+        }
+        query = query.filter(Column::Repo.contains(repo));
+    }
+    query
+        .order_by(Column::Owner, Order::Asc)
+        .order_by(Column::Repo, Order::Asc)
         .all(&conn)
         .await
 }
@@ -44,12 +98,22 @@ pub async fn get_merged_applications(owner: String, repo: String) -> Result<Vec<
  * # Returns
  * @return Result<Vec<ApplicationModel>, sea_orm::DbErr> - The result of the operation
  */
-pub async fn get_active_applications(owner: String, repo: String) -> Result<Vec<ApplicationModel>, sea_orm::DbErr> {
+pub async fn get_active_applications(owner: Option<String>, repo: Option<String>) -> Result<Vec<ApplicationModel>, sea_orm::DbErr> {
     let conn = get_database_connection().await?;
-    Application::find()
-        .filter(Column::Owner.contains(owner))
-        .filter(Column::Repo.contains(repo))
-        .filter(Column::PrNumber.ne(0))
+    let mut query = Application::find()
+        .filter(Column::PrNumber.ne(0));
+    if let Some(owner) = owner.clone() {
+        query = query.filter(Column::Owner.contains(owner));
+    }
+    if let Some(repo) = repo {
+        if owner.is_none() {
+            return Err(DbErr::Custom(format!("Owner is required to get merged applications").into()));
+        }
+        query = query.filter(Column::Repo.contains(repo));
+    }
+    query
+        .order_by(Column::Owner, Order::Asc)
+        .order_by(Column::Repo, Order::Asc)
         .all(&conn)
         .await
 }

fplus-lib/src/core/mod.rs

@@ -290,50 +290,32 @@ impl LDNApplication {
     }
 
     pub async fn all_applications() -> Result<Vec<(ApplicationFile, String, String)>, Vec<LDNError>> {
-        //TODO: Avoid filtering by allocators. Simply get all active & merged applications from the database.
-        let allocators = match database::allocators::get_allocators().await {
-            Ok(allocs) => allocs,
-            Err(e) => return Err(vec![LDNError::Load(format!("Failed to retrieve allocators: {}", e))]),
-        };        let mut all_apps: Vec<(ApplicationFile, String, String)> = Vec::new();
-        let mut errors: Vec<LDNError> = Vec::new();
-    
-        for allocator in allocators {
-            match Self::active(allocator.owner.clone(), allocator.repo.clone(), None).await {
-                Ok(apps) => {
-                    for app in apps {
-                        all_apps.push((app, allocator.owner.clone(), allocator.repo.clone()));
-                    }
-                },
-                Err(e) => {
-                    errors.push(e);
-                    eprintln!("Failed to process active applications for allocator {}:{}", allocator.repo, allocator.owner);
-                },
-            }
-    
-            match Self::merged(allocator.owner.clone(), allocator.repo.clone()).await {
-                Ok(merged_apps) => {
-                    for (_, app) in merged_apps {
-                        all_apps.push((app, allocator.owner.clone(), allocator.repo.clone()));
-                    }
-                },
-                Err(e) => {
-                    errors.push(e);
-                    eprintln!("Failed to process merged applications for allocator {}:{}", allocator.repo, allocator.owner);
-                },
-            }
-        }
-    
-        if errors.is_empty() {
-            Ok(all_apps)
-        } else {
-            Err(errors)
+        let db_apps = database::applications::get_applications().await;
+        let mut all_apps: Vec<(ApplicationFile, String, String)> = Vec::new();
+        match db_apps {
+            Ok(apps) => {
+                for app in apps {
+                    let app_file = match ApplicationFile::from_str(&app.application.unwrap()) {
+                        Ok(app) => app,
+                        Err(e) => {
+                            return Err(vec![LDNError::Load(format!("Failed to parse application file from DB /// {}", e))]);
+                        }
+                    };
+                    all_apps.push((app_file, app.owner, app.repo));
+                }
+                return Ok(all_apps);
+            },
+            Err(e) => {
+                return Err(vec![LDNError::Load(format!("Failed to retrieve applications from the database /// {}", e))]);
+            },
         }
+
     }
 
 
     pub async fn active(owner: String, repo: String, filter: Option<String>) -> Result<Vec<ApplicationFile>, LDNError> {
         // Get all active applications from the database.
-        let active_apps_result = database::applications::get_active_applications(owner, repo).await;
+        let active_apps_result = database::applications::get_active_applications(Some(owner), Some(repo)).await;
 
         // Handle errors in getting active applications.
         let active_apps = match active_apps_result {
@@ -355,7 +337,8 @@ impl LDNApplication {
             if let Some(app_json) = app_model.application {
                 match from_str::<ApplicationFile>(&app_json) {
                     Ok(app) => apps.push(app),
-                    Err(e) => return Err(LDNError::Load(format!("Failed to parse application file: {}", e))),
+                    //if error, don't push into apps
+                    Err(_) => {}
                 }
             }
         }
@@ -934,7 +917,7 @@ impl LDNApplication {
 
     pub async fn merged(owner: String, repo: String) -> Result<Vec<(ApplicationGithubInfo, ApplicationFile)>, LDNError> {
         // Retrieve all applications in the main branch from the database.
-        let merged_apps_result = database::applications::get_merged_applications(owner.clone(), repo.clone()).await;
+        let merged_apps_result = database::applications::get_merged_applications(Some(owner.clone()), Some(repo.clone())).await;
 
         // Handle errors in getting applications from the main branch.
         let merged_app_models = match merged_apps_result {
@@ -950,7 +933,7 @@ impl LDNApplication {
             if let Some(app_json) = app_model.application {
                 match from_str::<ApplicationFile>(&app_json) {
                     Ok(app) => merged_apps.push((ApplicationGithubInfo {sha: app_model.sha.unwrap(), path: app_model.path.unwrap()}, app)),
-                    Err(e) => return Err(LDNError::Load(format!("Failed to parse application file: {}", e))),
+                    Err(_) => {},
                 }
             }
         }
@@ -1807,7 +1790,7 @@ Your Datacap Allocation Request has been {} by the Notary
 
     pub async fn cache_renewal_active(owner: String, repo: String) -> Result<(), LDNError> {
         let active_from_gh: Vec<ApplicationFileWithDate> = LDNApplication::active_apps_with_last_update(owner.clone(), repo.clone(), None).await?;
-        let active_from_db: Vec<ApplicationModel> = database::applications::get_active_applications(owner.clone(), repo.clone()).await.unwrap();
+        let active_from_db: Vec<ApplicationModel> = database::applications::get_active_applications(Some(owner.clone()), Some(repo.clone())).await.unwrap();
 
         let mut db_apps_set: HashSet<String> = HashSet::new();
         let mut processed_gh_apps: HashSet<String> = HashSet::new();
@@ -1864,7 +1847,7 @@ Your Datacap Allocation Request has been {} by the Notary
 
     pub async fn cache_renewal_merged(owner: String, repo: String) -> Result<(), LDNError> {
         let merged_from_gh: Vec<ApplicationFileWithDate> = LDNApplication::merged_apps_with_last_update(owner.clone(), repo.clone(), None).await?;
-        let merged_from_db: Vec<ApplicationModel> = database::applications::get_merged_applications(owner.clone(), repo.clone()).await.unwrap();
+        let merged_from_db: Vec<ApplicationModel> = database::applications::get_merged_applications(Some(owner.clone()), Some(repo.clone())).await.unwrap();
 
         let mut db_apps_set: HashSet<String> = HashSet::new();
         let mut processed_gh_apps: HashSet<String> = HashSet::new();