infra: configure forge-test environment (#44)

* use custom storoku v0.5.1_co cost-optimization version

* add test network

* add references to test network services

* forge-test upload-service has been deployed as w3infra

* latest storoku

* deploy to forge-test on releases

Author: volmedo

.github/workflows/deploy.yml

@@ -20,6 +20,7 @@ on:
         options:
           - warm-staging
           - forge-production
+          - forge-test
 
 permissions:
   id-token: write # This is required for requesting the JWT
@@ -48,7 +49,7 @@ jobs:
       cloudflare-zone-id: ${{ secrets.WARM_STAGING_CLOUDFLARE_ZONE_ID }}
       cloudflare-api-token: ${{ secrets.WARM_STAGING_CLOUDFLARE_API_TOKEN }}
 
-  # apply prod on successful release, plan otherwise
+  # apply prod and test on successful release, plan otherwise
   forge-production:
     uses: ./.github/workflows/terraform.yml
     with:
@@ -69,3 +70,24 @@ jobs:
       admin-dashboard-password: ${{ secrets.FORGE_PROD_ADMIN_DASHBOARD_PASSWORD }}
       cloudflare-zone-id: ${{ secrets.FORGE_PROD_CLOUDFLARE_ZONE_ID }}
       cloudflare-api-token: ${{ secrets.FORGE_PROD_CLOUDFLARE_API_TOKEN }}
+
+  forge-test:
+    uses: ./.github/workflows/terraform.yml
+    with:
+      env: forge-test
+      workspace: forge-test
+      network: test
+      did: did:web:etracker.test.storacha.network
+      client-egress-usd-per-tib: ${{ vars.FORGE_TEST_CLIENT_EGRESS_USD_PER_TIB }}
+      provider-egress-usd-per-tib: ${{ vars.FORGE_TEST_PROVIDER_EGRESS_USD_PER_TIB }}
+      apply: ${{ (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'forge-test') }}
+    secrets:
+      aws-account-id: ${{ secrets.FORGE_TEST_AWS_ACCOUNT_ID }}
+      aws-region: ${{ secrets.FORGE_TEST_AWS_REGION }}
+      region: ${{ secrets.FORGE_TEST_AWS_REGION }}
+      private-key: ${{ secrets.FORGE_TEST_PRIVATE_KEY }}
+      metrics-auth-token: ${{ secrets.FORGE_TEST_METRICS_AUTH_TOKEN }}
+      admin-dashboard-user: ${{ secrets.FORGE_TEST_ADMIN_DASHBOARD_USER }}
+      admin-dashboard-password: ${{ secrets.FORGE_TEST_ADMIN_DASHBOARD_PASSWORD }}
+      cloudflare-zone-id: ${{ secrets.FORGE_TEST_CLOUDFLARE_ZONE_ID }}
+      cloudflare-api-token: ${{ secrets.FORGE_TEST_CLOUDFLARE_API_TOKEN }}

.github/workflows/terraform.yml

@@ -86,9 +86,6 @@ jobs:
           aws-region: ${{ env.AWS_REGION }}
           role-to-assume: arn:aws:iam::${{ env.AWS_ACCOUNT_ID }}:role/terraform-ci
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
       - uses: opentofu/setup-opentofu@v1
 
       - name: Tofu Init
@@ -97,17 +94,24 @@ jobs:
           make init
         working-directory: deploy
 
-      - name: Build + Push Docker ECR
-        run: |
-          make docker-push
-        working-directory: deploy
-
+      # just plan if !inputs.apply
       - name: Terraform Plan
         if: ${{ !inputs.apply }}
         run: |
           make plan
         working-directory: deploy
 
+      # build and push docker image and apply if inputs.apply
+      - name: Set up Docker Buildx
+        if: ${{ inputs.apply }}
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build + Push Docker ECR
+        if: ${{ inputs.apply }}
+        run: |
+          make docker-push
+        working-directory: deploy
+
       - name: Terraform Apply
         if: ${{ inputs.apply }}
         run: |

.storoku.json

@@ -14,15 +14,18 @@
   "secrets": [
     {
       "name": "ETRACKER_METRICS_AUTH_TOKEN",
-      "variable": true
+      "variable": true,
+      "external": false
     },
     {
       "name": "ETRACKER_ADMIN_DASHBOARD_USER",
-      "variable": true
+      "variable": true,
+      "external": false
     },
     {
       "name": "ETRACKER_ADMIN_DASHBOARD_PASSWORD",
-      "variable": true
+      "variable": true,
+      "external": false
     }
   ],
   "tables": [
@@ -101,7 +104,8 @@
   ],
   "networks": [
     "warm",
-    "forge"
+    "forge",
+    "test"
   ],
   "writeToContainer": false
 }

deploy/.env.production.local.tpl

@@ -12,6 +12,19 @@ if [ "$TF_WORKSPACE" == "forge-prod" ]; then
   CONSUMER_CUSTOMER_INDEX_NAME="customer"
 
   TRUSTED_AUTHORITIES="did:web:up.forge.storacha.network"
+elif [ "$TF_WORKSPACE" == "forge-test" ]; then
+  STORAGE_PROVIDER_TABLE_NAME="forge-test-w3infra-storage-provider"
+  STORAGE_PROVIDER_TABLE_REGION="us-west-2"
+
+  CUSTOMER_TABLE_NAME="forge-test-w3infra-customer"
+  CUSTOMER_TABLE_REGION="us-west-2"
+
+  CONSUMER_TABLE_NAME="forge-test-w3infra-consumer"
+  CONSUMER_TABLE_REGION="us-west-2"
+  CONSUMER_CONSUMER_INDEX_NAME="consumer"
+  CONSUMER_CUSTOMER_INDEX_NAME="customer"
+
+  TRUSTED_AUTHORITIES="did:web:up.test.storacha.network"
 else
   STORAGE_PROVIDER_TABLE_NAME="staging-warm-upload-api-storage-provider"
   STORAGE_PROVIDER_TABLE_REGION="us-east-2"

deploy/app/.terraform.lock.hcl

@@ -1,12 +1,12 @@
 locals {
-    storage_provider_table_name = "${terraform.workspace == "forge-prod" ? "forge-prod-upload-api-storage-provider" : "staging-warm-upload-api-storage-provider"}"
-    storage_provider_table_region = "${terraform.workspace == "forge-prod" ? "us-west-2" : "us-east-2"}"
+    storage_provider_table_name = "${terraform.workspace == "forge-test" ? "forge-test-w3infra-storage-provider" : (terraform.workspace == "forge-prod" ? "forge-prod-upload-api-storage-provider" : "staging-warm-upload-api-storage-provider")}"
+    storage_provider_table_region = "${(terraform.workspace == "forge-prod" || terraform.workspace == "forge-test") ? "us-west-2" : "us-east-2"}"
 
-    customer_table_name = "${terraform.workspace == "forge-prod" ? "forge-prod-upload-api-customer" : "staging-warm-upload-api-customer"}"
-    customer_table_region = "${terraform.workspace == "forge-prod" ? "us-west-2" : "us-east-2"}"
+    customer_table_name = "${terraform.workspace == "forge-test" ? "forge-test-w3infra-customer" : (terraform.workspace == "forge-prod" ? "forge-prod-upload-api-customer" : "staging-warm-upload-api-customer")}"
+    customer_table_region = "${(terraform.workspace == "forge-test" || terraform.workspace == "forge-prod") ? "us-west-2" : "us-east-2"}"
 
-    consumer_table_name = "${terraform.workspace == "forge-prod" ? "forge-prod-upload-api-consumer" : "staging-warm-upload-api-consumer"}"
-    consumer_table_region = "${terraform.workspace == "forge-prod" ? "us-west-2" : "us-east-2"}"
+    consumer_table_name = "${terraform.workspace == "forge-test" ? "forge-test-w3infra-consumer" : (terraform.workspace == "forge-prod" ? "forge-prod-upload-api-consumer" : "staging-warm-upload-api-consumer")}"
+    consumer_table_region = "${(terraform.workspace == "forge-test" || terraform.workspace == "forge-prod") ? "us-west-2" : "us-east-2"}"
 }
 
 provider "aws" {

deploy/app/external.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 5.86.0"
+      version = ">= 6.0.0"
     }
     archive = {
       source = "hashicorp/archive"
@@ -31,14 +31,10 @@ provider "aws" {
   }
 }
 
-# CloudFront is a global service. Certs must be created in us-east-1, where the core ACM infra lives
-provider "aws" {
-  region = "us-east-1"
-  alias = "acm"
-}
+
 
 module "app" {
-  source = "github.com/storacha/storoku//app?ref=v0.5.1"
+  source = "github.com/storacha/storoku//app?ref=v0.6.2"
   private_key = var.private_key
   private_key_env_var = "ETRACKER_PRIVATE_KEY"
   httpport = 8080
@@ -72,6 +68,8 @@ module "app" {
     "ETRACKER_ADMIN_DASHBOARD_USER" = var.admin_dashboard_user
     "ETRACKER_ADMIN_DASHBOARD_PASSWORD" = var.admin_dashboard_password
   }
+  # enter external secrets (provisioned out-of-band) here
+  external_secrets = []
   # enter any sqs queues you want to create here
   queues = []
   caches = []
@@ -145,10 +143,6 @@ module "app" {
   ]
   buckets = [
   ]
-  providers = {
-    aws = aws
-    aws.acm = aws.acm
-  }
   env_files = var.env_files
   domain_base = var.domain_base
 }