Merge branch 'main' into feat/multisig-support

kokal33 · kokal33 · commit db4fc67ea595 · 2024-03-08T15:59:50.000+01:00
diff --git a/.github/workflows/deploy_to_prod.yml b/.github/workflows/deploy_to_prod.yml
@@ -8,6 +8,7 @@ on:
 jobs:
   deploy_to_prod:
     runs-on: ubuntu-latest
+    environment: production
 
     steps:
     - name: Checkout code
@@ -49,13 +50,12 @@ jobs:
             },
             "environment": { 
               "GH_PRIVATE_KEY": "${{secrets.GH_PRIVATE_KEY}}",
-              "GITHUB_OWNER": "${{secrets.GH_OWNER}}",
-              "GITHUB_REPO": "${{secrets.GH_REPO}}",
-              "GITHUB_APP_ID": "${{secrets.GH_APP_ID}}",
-              "GITHUB_INSTALLATION_ID": "${{secrets.GH_INSTALLATION_ID}}",
+              "GITHUB_APP_ID": "${{vars.GH_APP_ID}}",
               "FILPLUS_ENV": "prod",
               "RUST_LOG": "debug",
-              "BOT_USER": "${{secrets.BOT_USER}}"
+              "BOT_USER": "${{vars.BOT_USER}}",
+              "BACKEND_URL": "${{vars.BACKEND_URL}}",
+              "DB_URL": "${{secrets.DB_URL}}"
             }
           }
         }' > containers.json
diff --git a/.github/workflows/deploy_to_staging.yml b/.github/workflows/deploy_to_staging.yml
@@ -24,6 +24,7 @@ jobs:
     name: Deploy to staging
     needs: end_to_end_tests
     runs-on: ubuntu-latest
+    environment: staging
 
     steps:
     - name: Checkout code
@@ -64,8 +65,14 @@ jobs:
               "8080": "HTTP"
             },
             "environment": { 
-                "GH_PRIVATE_KEY": "${{secrets.GH_STAGING_PRIVATE_KEY}}",
-                "DB_URL": "${{secrets.STAGING_DB_URL}}"
+                "GH_PRIVATE_KEY": "${{secrets.GH_PRIVATE_KEY}}",
+                "DB_URL": "${{secrets.DB_URL}}"
+                "GITHUB_APP_ID": "${{vars.GH_APP_ID}}",
+                "FILPLUS_ENV": "prod",
+                "RUST_LOG": "debug",
+                "BOT_USER": "${{vars.BOT_USER}}",
+                "BACKEND_URL": "${{vars.BACKEND_URL}}",
+                "DB_URL": "${{secrets.DB_URL}}"
             }
           }
         }' > containers.json
diff --git a/fplus-database/src/lib.rs b/fplus-database/src/lib.rs
@@ -50,21 +50,22 @@ pub async fn get_database_connection() -> Result<DatabaseConnection, DbErr> {
         Err(DbErr::Custom("Database connection is not established".into()))
     }
 }
+
+/**
+* Sets up the initial test environment (database connection and env variables)
+*/
+pub async fn setup_test_environment() {
+   init();
+   setup().await.expect("Failed to setup database connection.");
+}
+
 #[cfg(test)]
 mod tests {
     
     use super::*;
     use tokio;
     use serial_test::serial;
 
-    /**
-     * Sets up the initial test environment (database connection and env variables)
-     */
-    async fn setup_test_environment() {
-        init();
-        setup().await.expect("Failed to setup database connection.");
-    }
-
     /**
      * Test the establish_connection function
      * 
diff --git a/fplus-http-server/src/router/allocator.rs b/fplus-http-server/src/router/allocator.rs
@@ -1,6 +1,6 @@
 use actix_web::{get, post, put, delete, web, HttpResponse, Responder};
 use fplus_database::database::allocators as allocators_db;
-use fplus_lib::core::{allocator::process_allocator_file, AllocatorUpdateInfo, ChangedAllocator};
+use fplus_lib::core::{allocator::{process_allocator_file, is_allocator_repo_created, create_allocator_repo}, AllocatorUpdateInfo, ChangedAllocator};
 
 /**
  * Get all allocators
@@ -44,15 +44,30 @@ pub async fn create_from_json(file: web::Json<ChangedAllocator>) -> actix_web::R
                 Some(model.application.verifiers_gh_handles.join(", ")) // Join verifiers in a string if exists
             };
             
-            match allocators_db::create_or_update_allocator(
-                model.owner,
-                model.repo,
+            let allocator_model = match allocators_db::create_or_update_allocator(
+                model.owner.clone(),
+                model.repo.clone(),
                 Some(model.installation_id as i64), 
                 Some(model.multisig_address),      
                 verifiers_gh_handles,
                 model.multisig_threshold
             ).await {
-                Ok(allocator_model) => Ok(HttpResponse::Ok().json(allocator_model)),
+                Ok(allocator_model) => allocator_model,
+                Err(e) => return Ok(HttpResponse::BadRequest().body(e.to_string())),
+            };
+
+            match is_allocator_repo_created(&model.owner, &model.repo).await {
+                Ok(true) => Ok(HttpResponse::Ok().json(allocator_model)),
+                Ok(false) => {
+                    //Create allocator repo. If it fails, return http error
+                    match create_allocator_repo(&model.owner, &model.repo).await {
+                        Ok(files_list) => {
+                            log::info!("Allocator repo created successfully: {:?}", files_list);
+                            Ok(HttpResponse::Ok().json(allocator_model))
+                        }
+                        Err(e) => Ok(HttpResponse::BadRequest().body(e.to_string())),
+                    }
+                },
                 Err(e) => Ok(HttpResponse::BadRequest().body(e.to_string())),
             }
         },
diff --git a/fplus-lib/src/config.rs b/fplus-lib/src/config.rs
@@ -9,7 +9,7 @@ pub fn default_env_vars() -> &'static HashMap<&'static str, &'static str> {
         m.insert("GITHUB_OWNER", "filecoin-project");
         m.insert("GITHUB_REPO", "filecoin-plus-falcon");
         m.insert("GITHUB_APP_ID", "826129");
-        m.insert("GITHUB_INSTALLATION_ID", "47207972");
+        m.insert("GITHUB_INSTALLATION_ID", "48206379");
         m.insert("RUST_LOG", "info");
         m.insert("RUST_BACKTRACE", "1");
         m.insert("DB_URL", "");
diff --git a/fplus-lib/src/core/allocator/mod.rs b/fplus-lib/src/core/allocator/mod.rs
@@ -2,8 +2,9 @@ use octocrab::models::repos::ContentItems;
 
 use crate::config::get_env_var_or_default;
 use crate::external_services::filecoin::get_multisig_threshold_for_actor;
+use crate::external_services::github::{github_async_new, GithubWrapper};
 use crate::{
-    base64::decode_allocator_model, error::LDNError, external_services::github::GithubWrapper,
+    base64::decode_allocator_model, error::LDNError,
 };
 
 use self::file::AllocatorModel;
@@ -13,16 +14,13 @@ pub mod file;
 pub async fn process_allocator_file(file_name: &str) -> Result<AllocatorModel, LDNError> {
     let owner = get_env_var_or_default("ALLOCATOR_GOVERNANCE_OWNER");
     let repo = get_env_var_or_default("ALLOCATOR_GOVERNANCE_REPO");
+    let installation_id = get_env_var_or_default("GITHUB_INSTALLATION_ID");
     let branch = "main";
     let path = file_name.to_string();
 
-    let gh = GithubWrapper::new(owner.to_string(), repo.to_string());
-    let content_items = gh
-        .get_file(&path, branch)
-        .await
-        .map_err(|e| LDNError::Load(e.to_string()))?;
-    let mut model = content_items_to_allocator_model(content_items)
-        .map_err(|e| LDNError::Load(e.to_string()))?;
+    let gh = GithubWrapper::new(owner.clone(), repo.clone(), installation_id);
+    let content_items = gh.get_file(&path, branch).await.map_err(|e| LDNError::Load(e.to_string()))?;
+    let mut model = content_items_to_allocator_model(content_items).map_err(|e| LDNError::Load(e.to_string()))?;
 
     // Get multisig threshold from the blockchain if multisig address is available
     if let Ok(blockchain_threshold) = get_multisig_threshold_for_actor(&model.multisig_address).await {
@@ -63,3 +61,88 @@ fn content_items_to_allocator_model(file: ContentItems) -> Result<AllocatorModel
         }
     }
 }
+
+pub async fn is_allocator_repo_created(owner: &str, repo: &str) -> Result<bool, LDNError> {
+    let repo_flag_file = "invalisd.md";
+    let applications_directory = "applications";
+    let gh = github_async_new(owner.to_string(), repo.to_string()).await;
+    let all_files_result = gh.get_files(applications_directory).await.map_err(|e| {
+        LDNError::Load(format!("Failed to retrieve all files from GitHub. Reason: {}", e))
+    });
+
+    match all_files_result {
+        Ok(content_items) => {
+            let mut is_repo_created = false;
+            for file in content_items.items.iter() {
+                if file.name == repo_flag_file {
+                    is_repo_created = true;
+                    break;
+                }
+            }
+            Ok(is_repo_created)
+        },
+        Err(e) => {
+            if e.to_string().contains("GitHub: This repository is empty") || e.to_string().contains("GitHub: Not Found"){
+                Ok(false)
+            } else {
+                Err(e)
+            }
+        },
+    }
+}
+
+pub async fn create_allocator_repo(owner: &str, repo: &str) -> Result<(), LDNError> {
+    let gh = github_async_new(owner.to_string(), repo.to_string()).await;
+    let mut dirs = Vec::new();
+    dirs.push("".to_string());
+    
+    while dirs.len() > 0 {
+        let dir = dirs.pop().unwrap();
+        let files_list = gh.get_files_from_public_repo("clriesco", "filplus-allocator-template", Some(&dir)).await.map_err(|e| {
+            LDNError::Load(format!("Failed to retrieve all files from GitHub. Reason: {}", e))
+        })?;
+
+        for file in files_list.items.iter() {
+            let file_path = file.path.clone();
+            if file.r#type == "dir" {
+                dirs.push(file_path);
+                continue;
+            }
+            let file = reqwest::Client::new()
+            .get(&file.download_url.clone().unwrap())
+            .send()
+            .await
+            .map_err(|e| LDNError::Load(format!("here {}", e)))?;
+            let file = file
+                .text()
+                .await
+                .map_err(|e| LDNError::Load(format!("here1 {}", e)))?;
+
+            //Get file from target repo. If file does not exist or fails to retrieve, create it
+            let target_file = gh.get_file(&file_path, "main").await.map_err(|e| {
+                LDNError::Load(format!("Failed to retrieve file from GitHub. Reason: {} in file {}", e, file_path))
+            });
+
+            match target_file {
+                Ok(target_file) => {
+                    if target_file.items.is_empty() {
+                        log::info!("Creating file in target repo: {}", file_path);
+                        gh.add_file(&file_path, &file, "first commit", "main").await.map_err(|e| {
+                            LDNError::Load(format!("Failed to create file in GitHub. Reason: {} in file {}", e, file_path))
+                        })?;
+                    } else {
+                        log::info!("File already exists in target repo: {}", file_path);
+                    }
+                },
+                Err(_) => {
+                    log::info!("Creating file in target repo: {}", file_path);
+                    gh.add_file(&file_path, &file, "first commit", "main").await.map_err(|e| {
+                        LDNError::Load(format!("Failed to create file in GitHub. Reason: {} in file {}", e, file_path))
+                    })?;
+                },
+            }
+        }
+    }
+
+    Ok(())
+}
diff --git a/fplus-lib/src/core/mod.rs b/fplus-lib/src/core/mod.rs
diff --git a/fplus-lib/src/external_services/github.rs b/fplus-lib/src/external_services/github.rs
diff --git a/fplus-lib/src/parsers.rs b/fplus-lib/src/parsers.rs
