Merge pull request #1206 from input-output-hk/jpraynaud/986-add-cloudflare-support

jpraynaud · web-flow · commit 6086c8aa4230 · 2023-09-18T16:46:53.000+02:00
Add aggregator CDN support in infrastructure
diff --git a/.github/workflows/actions/deploy-terraform-infrastructure/action.yml b/.github/workflows/actions/deploy-terraform-infrastructure/action.yml
@@ -60,6 +60,13 @@ inputs:
   mithril_era_reader_secret_key: 
     description: Mithril era reader secret key (onlye for test networks).
     required: false
+  mithril_aggregator_cdn_cname:
+    description: Mithril aggregator uses CDN domain for constructing snapshot url.
+    required: true
+  mithril_aggregator_snapshot_use_cdn_domain:
+    description: Mithril aggregator CNAME field used for the snapshot CDN.
+    required: false
+    default: "false"
   mithril_aggregator_snapshot_compression_algorithm: 
     description: Mithril aggregator snapshot compression algorithm.
     required: false
@@ -136,6 +143,8 @@ runs:
         mithril_era_reader_address_url                    = "${{ inputs.mithril_era_reader_address_url }}"
         mithril_era_reader_verification_key_url           = "${{ inputs.mithril_era_reader_verification_key_url }}"
         mithril_era_reader_secret_key                     = "${{ inputs.mithril_era_reader_secret_key }}"
+        mithril_aggregator_cdn_cname                      = "${{ inputs.mithril_aggregator_cdn_cname }}"
+        mithril_aggregator_snapshot_use_cdn_domain        = ${{ inputs.mithril_aggregator_snapshot_use_cdn_domain }}
         mithril_aggregator_snapshot_compression_algorithm = "${{ inputs.mithril_aggregator_snapshot_compression_algorithm }}"
         mithril_aggregator_zstandard_parameters_level     = "${{ inputs.mithril_aggregator_zstandard_parameters_level }}"
         mithril_aggregator_zstandard_parameters_workers   = "${{ inputs.mithril_aggregator_zstandard_parameters_workers }}"
@@ -162,7 +171,7 @@ runs:
     shell: bash
     working-directory: mithril-infra
     run: |
-      GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform init -backend-config="bucket=${{ inputs.terraform_backend_bucket }}" -backend-config="prefix=terraform/mithril-${{ inputs.environment }}"
+      GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform init -backend-config="bucket=${{ inputs.terraform_backend_bucket }}" -backend-config="prefix=terraform/mithril-${{ inputs.environment }}" -upgrade
 
   - name: Check Terraform
     shell: bash
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -476,6 +476,8 @@ jobs:
           mithril_era_reader_address_url: ${{ vars.ERA_READER_ADDRESS_URL }}
           mithril_era_reader_verification_key_url: ${{ vars.ERA_READER_VERIFICATION_KEY_URL }}
           mithril_era_reader_secret_key: ${{ secrets.ERA_READER_SECRET_KEY }}
+          mithril_aggregator_cdn_cname: ${{ vars.AGGREGATOR_CDN_CNAME }}
+          mithril_aggregator_snapshot_use_cdn_domain: ${{ vars.AGGREGATOR_USE_CDN_DOMAIN }}
           mithril_aggregator_snapshot_compression_algorithm: ${{ vars.AGGREGATOR_SNAPSHOT_COMPRESSION_ALGORITHM }}
           mithril_aggregator_zstandard_parameters_level: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_LEVEL }}
           mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml
@@ -201,6 +201,8 @@ jobs:
           mithril_era_reader_address_url: ${{ vars.ERA_READER_ADDRESS_URL }}
           mithril_era_reader_verification_key_url: ${{ vars.ERA_READER_VERIFICATION_KEY_URL }}
           mithril_era_reader_secret_key: ${{ secrets.ERA_READER_SECRET_KEY }}
+          mithril_aggregator_cdn_cname: ${{ vars.AGGREGATOR_CDN_CNAME }}
+          mithril_aggregator_snapshot_use_cdn_domain: ${{ vars.AGGREGATOR_USE_CDN_DOMAIN }}
           mithril_aggregator_snapshot_compression_algorithm: ${{ vars.AGGREGATOR_SNAPSHOT_COMPRESSION_ALGORITHM }}
           mithril_aggregator_zstandard_parameters_level: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_LEVEL }}
           mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -191,6 +191,8 @@ jobs:
           mithril_genesis_verification_key_url: ${{ vars.GENESIS_VERIFICATION_KEY_URL }}
           mithril_era_reader_address_url: ${{ vars.ERA_READER_ADDRESS_URL }}
           mithril_era_reader_verification_key_url: ${{ vars.ERA_READER_VERIFICATION_KEY_URL }}
+          mithril_aggregator_cdn_cname: ${{ vars.AGGREGATOR_CDN_CNAME }}
+          mithril_aggregator_snapshot_use_cdn_domain: ${{ vars.AGGREGATOR_USE_CDN_DOMAIN }}
           mithril_aggregator_snapshot_compression_algorithm: ${{ vars.AGGREGATOR_SNAPSHOT_COMPRESSION_ALGORITHM }}
           mithril_aggregator_zstandard_parameters_level: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_LEVEL }}
           mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/docs/website/root/manual/developer-docs/nodes/mithril-aggregator.md b/docs/website/root/manual/developer-docs/nodes/mithril-aggregator.md
@@ -437,8 +437,8 @@ Here is a list of the available parameters:
 | `snapshot_store_type` | - | - | `SNAPSHOT_STORE_TYPE` | Type of snapshot store to use | - | `gcp` or `local` | :heavy_check_mark: |
 | `snapshot_uploader_type` | - | - | `SNAPSHOT_UPLOADER_TYPE` | Type of snapshot uploader to use | - | `gcp` or `local` | :heavy_check_mark: |
 | `snapshot_bucket_name` | - | - | `SNAPSHOT_BUCKET_NAME` | Name of the bucket where the snapshots are stored  | - | `snapshot-bucket` | :heavy_check_mark: | Required if `snapshot_uploader_type` is `gcp`
+| `snapshot_use_cdn_domain` | - | - | `SNAPSHOT_USE_CDN_DOMAIN` | Use CDN domain for constructing snapshot url  | `false` | - | - | To be used if `snapshot_uploader_type` is `gcp`
 | `run_interval` | - | - | `RUN_INTERVAL` | Interval between two runtime cycles in ms | - | `60000` | :heavy_check_mark: |
-| `url_snapshot_manifest` | - | - | `URL_SNAPSHOT_MANIFEST` | Snapshots manifest location | - | Only if `snapshot_store_type` is `gcp`, else it should be `` | :heavy_check_mark: |
 | `era_reader_adapter_type` | `--era-reader-adapter-type` | - | `ERA_READER_ADAPTER_TYPE` | Era reader adapter type that can be `cardano-chain`, `file` or `bootstrap`. | `bootstrap` | - | - |
 | `era_reader_adapter_params` | `--era-reader-adapter-params` | - | `ERA_READER_ADAPTER_PARAMS` | Era reader adapter params that is an optional JSON encoded parameters structure that is expected depending on the `era_reader_adapter_type` parameter | - | - | - |
 | `snapshot_compression_algorithm` | `--snapshot-compression-algorithm` | - | `SNAPSHOT_COMPRESSION_ALGORITHM` | Compression algorithm of the snapshot archive | `zstandard` | `gzip` or `zstandard` | - |
diff --git a/docs/website/root/manual/getting-started/run-mithril-devnet.md b/docs/website/root/manual/getting-started/run-mithril-devnet.md
@@ -165,7 +165,7 @@ Creating artifacts_mithril-aggregator_1        ... done
 Creating artifacts_mithril-signer-node-pool1_1 ... done
 Creating artifacts_mithril-signer-node-pool2_1 ... done
 Creating artifacts_mithril-aggregator-genesis_run ... done
-{"msg":"Started","v":0,"name":"slog-rs","level":20,"time":"2022-09-06T09:24:31.925641527Z","hostname":"e932dff845aa","pid":1,"config":"Configuration { cardano_cli_path: \"/app/bin/cardano-cli\", cardano_node_socket_path: \"/data/ipc/node.sock\", network_magic: Some(42), network: \"devnet\", protocol_parameters: ProtocolParameters { k: 5, m: 100, phi_f: 0.65 }, url_snapshot_manifest: \"\", snapshot_store_type: Local, snapshot_uploader_type: Local, server_url: \"http://0.0.0.0:8080/\", run_interval: 1000, db_directory: \"/data/db\", snapshot_directory: \"/data/mithril/aggregator\", data_stores_directory: \"/data/mithril/aggregator/stores\", genesis_verification_key: \"5b33322c3235332c3138362c3230312c3137372c31312c3131372c3133352c3138372c3136372c3138312c3138382c32322c35392c3230362c3130352c3233312c3135302c3231352c33302c37382c3231322c37362c31362c3235322c3138302c37322c3133342c3133372c3234372c3136312c36385d\" }","run_mode":"dev"}
+{"msg":"Started","v":0,"name":"slog-rs","level":20,"time":"2022-09-06T09:24:31.925641527Z","hostname":"e932dff845aa","pid":1,"config":"Configuration { cardano_cli_path: \"/app/bin/cardano-cli\", cardano_node_socket_path: \"/data/ipc/node.sock\", network_magic: Some(42), network: \"devnet\", protocol_parameters: ProtocolParameters { k: 5, m: 100, phi_f: 0.65 }, snapshot_store_type: Local, snapshot_uploader_type: Local, server_url: \"http://0.0.0.0:8080/\", run_interval: 1000, db_directory: \"/data/db\", snapshot_directory: \"/data/mithril/aggregator\", data_stores_directory: \"/data/mithril/aggregator/stores\", genesis_verification_key: \"5b33322c3235332c3138362c3230312c3137372c31312c3131372c3133352c3138372c3136372c3138312c3138382c32322c35392c3230362c3130352c3233312c3135302c3231352c33302c37382c3231322c37362c31362c3235322c3138302c37322c3133342c3133372c3234372c3136312c36385d\" }","run_mode":"dev"}
 {"msg":"New LocalSnapshotUploader created","v":0,"name":"slog-rs","level":20Genesis bootstrap for test only
 ,"time":"2022-09-06T09:24:31.925683285Z","hostname":"e932dff845aa","pid":1,"snapshot_server_url":"http://0.0.0.0:8080/"}
 {"msg":"New MultiSignerImpl created","v":0,"name":"slog-rs","level":20,"time":"2022-09-06T09:24:31.925711468Z","hostname":"e932dff845aa","pid":1}
diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.3.86"
+version = "0.3.87"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }
diff --git a/mithril-aggregator/config/dev.json b/mithril-aggregator/config/dev.json
@@ -10,10 +10,9 @@
         "m": 100,
         "phi_f": 0.65
     },
-    "url_snapshot_manifest": "https://storage.googleapis.com/cardano-testnet/snapshots.json",
     "snapshot_store_type": "local",
     "snapshot_uploader_type": "local",
     "data_stores_directory": "/tmp/mithril-aggregator/stores",
     "genesis_verification_key": "5b33322c3235332c3138362c3230312c3137372c31312c3131372c3133352c3138372c3136372c3138312c3138382c32322c35392c3230362c3130352c3233312c3135302c3231352c33302c37382c3231322c37362c31362c3235322c3138302c37322c3133342c3133372c3234372c3136312c36385d",
     "era_adapter_type": "bootstrap"
-}
+}
diff --git a/mithril-aggregator/config/preview.json b/mithril-aggregator/config/preview.json
@@ -9,12 +9,11 @@
         "m": 100,
         "phi_f": 0.65
     },
-    "url_snapshot_manifest": "https://storage.googleapis.com/cardano-preview/snapshots.json",
     "snapshot_store_type": "local",
     "snapshot_uploader_type": "gcp",
     "snapshot_bucket_name": "cardano-testnet",
     "stores_directory": "./mithril-aggregator/stores",
     "genesis_verification_key": "5b3132372c37332c3132342c3136312c362c3133372c3133312c3231332c3230372c3131372c3139382c38352c3137362c3139392c3136322c3234312c36382c3132332c3131392c3134352c31332c3233322c3234332c34392c3232392c322c3234392c3230352c3230352c33392c3233352c34345d",
     "limit_keys_in_stores": 5,
     "era_adapter_type": "bootstrap"
-}
+}
diff --git a/mithril-aggregator/src/configuration.rs b/mithril-aggregator/src/configuration.rs
@@ -76,6 +76,9 @@ pub struct Configuration {
     /// Bucket name where the snapshots are stored if snapshot_uploader_type is Gcp
     pub snapshot_bucket_name: Option<String>,
 
+    /// Use CDN domain to construct snapshot urls if snapshot_uploader_type is Gcp
+    pub snapshot_use_cdn_domain: bool,
+
     /// Server listening IP
     pub server_ip: String,
 
@@ -173,6 +176,7 @@ impl Configuration {
             },
             snapshot_uploader_type: SnapshotUploaderType::Local,
             snapshot_bucket_name: None,
+            snapshot_use_cdn_domain: false,
             server_ip: "0.0.0.0".to_string(),
             server_port: 8000,
             run_interval: 5000,
@@ -205,6 +209,7 @@ impl Configuration {
                 Ok(Arc::new(RemoteSnapshotUploader::new(
                     Box::new(GcpFileUploader::new(bucket.clone())),
                     bucket,
+                    self.snapshot_use_cdn_domain,
                 )))
             }
             SnapshotUploaderType::Local => Ok(Arc::new(LocalSnapshotUploader::new(
@@ -277,6 +282,9 @@ pub struct DefaultConfiguration {
 
     /// Snapshot compression algorithm default setting
     pub snapshot_compression_algorithm: String,
+
+    /// Use CDN domain to construct snapshot urls default setting (if snapshot_uploader_type is Gcp)
+    pub snapshot_use_cdn_domain: String,
 }
 
 impl Default for DefaultConfiguration {
@@ -293,6 +301,7 @@ impl Default for DefaultConfiguration {
             reset_digests_cache: "false".to_string(),
             disable_digests_cache: "false".to_string(),
             snapshot_compression_algorithm: "zstandard".to_string(),
+            snapshot_use_cdn_domain: "false".to_string(),
         }
     }
 }
@@ -377,6 +386,13 @@ impl Source for DefaultConfiguration {
                 ValueKind::from(myself.snapshot_compression_algorithm),
             ),
         );
+        result.insert(
+            "snapshot_use_cdn_domain".to_string(),
+            Value::new(
+                Some(&namespace),
+                ValueKind::from(myself.snapshot_use_cdn_domain),
+            ),
+        );
 
         Ok(result)
     }
diff --git a/mithril-aggregator/src/dependency_injection/builder.rs b/mithril-aggregator/src/dependency_injection/builder.rs
@@ -314,6 +314,7 @@ impl DependenciesBuilder {
                     Ok(Arc::new(RemoteSnapshotUploader::new(
                         Box::new(GcpFileUploader::new(bucket.clone())),
                         bucket,
+                        self.configuration.snapshot_use_cdn_domain,
                     )))
                 }
                 SnapshotUploaderType::Local => Ok(Arc::new(LocalSnapshotUploader::new(
diff --git a/mithril-aggregator/src/snapshot_uploaders/remote_snapshot_uploader.rs b/mithril-aggregator/src/snapshot_uploaders/remote_snapshot_uploader.rs
@@ -10,15 +10,21 @@ use crate::tools::RemoteFileUploader;
 pub struct RemoteSnapshotUploader {
     bucket: String,
     file_uploader: Box<dyn RemoteFileUploader>,
+    use_cdn_domain: bool,
 }
 
 impl RemoteSnapshotUploader {
     /// GCPSnapshotUploader factory
-    pub fn new(file_uploader: Box<dyn RemoteFileUploader>, bucket: String) -> Self {
+    pub fn new(
+        file_uploader: Box<dyn RemoteFileUploader>,
+        bucket: String,
+        use_cdn_domain: bool,
+    ) -> Self {
         debug!("New GCPSnapshotUploader created");
         Self {
             bucket,
             file_uploader,
+            use_cdn_domain,
         }
     }
 }
@@ -27,10 +33,14 @@ impl RemoteSnapshotUploader {
 impl SnapshotUploader for RemoteSnapshotUploader {
     async fn upload_snapshot(&self, snapshot_filepath: &Path) -> StdResult<SnapshotLocation> {
         let archive_name = snapshot_filepath.file_name().unwrap().to_str().unwrap();
-        let location = format!(
-            "https://storage.googleapis.com/{}/{}",
-            self.bucket, archive_name
-        );
+        let location = if self.use_cdn_domain {
+            format!("https://{}/{}", self.bucket, archive_name)
+        } else {
+            format!(
+                "https://storage.googleapis.com/{}/{}",
+                self.bucket, archive_name
+            )
+        };
 
         self.file_uploader.upload_file(snapshot_filepath).await?;
 
@@ -47,11 +57,15 @@ mod tests {
     use std::path::Path;
 
     #[tokio::test]
-    async fn test_upload_snapshot_ok() {
+    async fn test_upload_snapshot_not_using_cdn_domain_ok() {
+        let use_cdn_domain = false;
         let mut file_uploader = MockRemoteFileUploader::new();
         file_uploader.expect_upload_file().returning(|_| Ok(()));
-        let snapshot_uploader =
-            RemoteSnapshotUploader::new(Box::new(file_uploader), "cardano-testnet".to_string());
+        let snapshot_uploader = RemoteSnapshotUploader::new(
+            Box::new(file_uploader),
+            "cardano-testnet".to_string(),
+            use_cdn_domain,
+        );
         let snapshot_filepath = Path::new("test/snapshot.xxx.tar.gz");
         let expected_location =
             "https://storage.googleapis.com/cardano-testnet/snapshot.xxx.tar.gz".to_string();
@@ -64,14 +78,35 @@ mod tests {
         assert_eq!(expected_location, location);
     }
 
+    #[tokio::test]
+    async fn test_upload_snapshot_using_cdn_domain_ok() {
+        let use_cdn_domain = true;
+        let mut file_uploader = MockRemoteFileUploader::new();
+        file_uploader.expect_upload_file().returning(|_| Ok(()));
+        let snapshot_uploader = RemoteSnapshotUploader::new(
+            Box::new(file_uploader),
+            "cdn.mithril.network".to_string(),
+            use_cdn_domain,
+        );
+        let snapshot_filepath = Path::new("test/snapshot.xxx.tar.gz");
+        let expected_location = "https://cdn.mithril.network/snapshot.xxx.tar.gz".to_string();
+
+        let location = snapshot_uploader
+            .upload_snapshot(snapshot_filepath)
+            .await
+            .expect("remote upload should not fail");
+
+        assert_eq!(expected_location, location);
+    }
+
     #[tokio::test]
     async fn test_upload_snapshot_ko() {
         let mut file_uploader = MockRemoteFileUploader::new();
         file_uploader
             .expect_upload_file()
             .returning(|_| Err(anyhow!("unexpected error")));
         let snapshot_uploader =
-            RemoteSnapshotUploader::new(Box::new(file_uploader), "".to_string());
+            RemoteSnapshotUploader::new(Box::new(file_uploader), "".to_string(), false);
         let snapshot_filepath = Path::new("test/snapshot.xxx.tar.gz");
 
         let result = snapshot_uploader
diff --git a/mithril-infra/assets/docker/docker-compose-aggregator-auth.yaml b/mithril-infra/assets/docker/docker-compose-aggregator-auth.yaml
@@ -52,10 +52,10 @@ services:
       - PROTOCOL_PARAMETERS__M=${PROTOCOL_PARAMETERS__M}
       - PROTOCOL_PARAMETERS__PHI_F=${PROTOCOL_PARAMETERS__PHI_F}
       - RUN_INTERVAL=60000
-      - URL_SNAPSHOT_MANIFEST=https://storage.googleapis.com/${SNAPSHOT_BUCKET_NAME}/snapshots.json
       - SNAPSHOT_STORE_TYPE=local
       - SNAPSHOT_UPLOADER_TYPE=gcp
       - SNAPSHOT_BUCKET_NAME=${SNAPSHOT_BUCKET_NAME}
+      - SNAPSHOT_USE_CDN_DOMAIN=${SNAPSHOT_USE_CDN_DOMAIN}
       - SNAPSHOT_COMPRESSION_ALGORITHM=${SNAPSHOT_COMPRESSION_ALGORITHM}
       - ZSTANDARD_PARAMETERS__LEVEL=${ZSTANDARD_PARAMETERS__LEVEL}
       - ZSTANDARD_PARAMETERS__NUMBER_OF_WORKERS=${ZSTANDARD_PARAMETERS__NUMBER_OF_WORKERS}
diff --git a/mithril-infra/assets/docker/docker-compose-aggregator-noauth.yaml b/mithril-infra/assets/docker/docker-compose-aggregator-noauth.yaml
@@ -52,10 +52,10 @@ services:
       - PROTOCOL_PARAMETERS__M=${PROTOCOL_PARAMETERS__M}
       - PROTOCOL_PARAMETERS__PHI_F=${PROTOCOL_PARAMETERS__PHI_F}
       - RUN_INTERVAL=60000
-      - URL_SNAPSHOT_MANIFEST=https://storage.googleapis.com/${SNAPSHOT_BUCKET_NAME}/snapshots.json
       - SNAPSHOT_STORE_TYPE=local
       - SNAPSHOT_UPLOADER_TYPE=gcp
       - SNAPSHOT_BUCKET_NAME=${SNAPSHOT_BUCKET_NAME}
+      - SNAPSHOT_USE_CDN_DOMAIN=${SNAPSHOT_USE_CDN_DOMAIN}
       - SNAPSHOT_COMPRESSION_ALGORITHM=${SNAPSHOT_COMPRESSION_ALGORITHM}
       - ZSTANDARD_PARAMETERS__LEVEL=${ZSTANDARD_PARAMETERS__LEVEL}
       - ZSTANDARD_PARAMETERS__NUMBER_OF_WORKERS=${ZSTANDARD_PARAMETERS__NUMBER_OF_WORKERS}
diff --git a/mithril-infra/assets/infra.version b/mithril-infra/assets/infra.version
@@ -1 +1 @@
-0.1.12
+0.1.13
diff --git a/mithril-infra/main.cloud-storage.tf b/mithril-infra/main.cloud-storage.tf
@@ -1,7 +1,9 @@
 resource "google_storage_bucket" "cloud_storage" {
-  name          = "${local.environment_name}-cs"
+  name          = local.mithril_aggregator_cdn_host
   location      = var.google_region
-  force_destroy = true
+  force_destroy = var.google_storage_bucket_force_destroy
+
+  depends_on = [googlesiteverification_dns.mithril-aggregator]
 
   lifecycle_rule {
     condition {
diff --git a/mithril-infra/main.dns.tf b/mithril-infra/main.dns.tf
diff --git a/mithril-infra/main.vm.tf b/mithril-infra/main.vm.tf
diff --git a/mithril-infra/mithril.aggregator.tf b/mithril-infra/mithril.aggregator.tf
diff --git a/mithril-infra/variables.tf b/mithril-infra/variables.tf
