infra: configure forge-test environment (#295)

Ref. storacha/project-tracking#667

Deploy the `indexer` to the new `forge-test` demo environment.

Author: volmedo

.github/workflows/deploy.yml

@@ -20,6 +20,7 @@ on:
         options:
           - production
           - forge-production
+          - forge-test
 
 permissions:
   id-token: write # This is required for requesting the JWT
@@ -71,7 +72,7 @@ jobs:
       cloudflare-api-token: ${{ secrets.WARM_STAGING_CLOUDFLARE_API_TOKEN }}
       sentry-dsn: ${{ secrets.WARM_STAGING_SENTRY_DSN }}
 
-  # apply prod on successful release, plan otherwise
+  # apply prod and test on successful release, plan otherwise
   production:
     uses: ./.github/workflows/terraform.yml
     with:
@@ -115,3 +116,25 @@ jobs:
       cloudflare-zone-id: ${{ secrets.FORGE_PROD_CLOUDFLARE_ZONE_ID }}
       cloudflare-api-token: ${{ secrets.FORGE_PROD_CLOUDFLARE_API_TOKEN }}
       sentry-dsn: ${{ secrets.FORGE_PROD_SENTRY_DSN }}
+
+  forge-test:
+    uses: ./.github/workflows/terraform.yml
+    with:
+      env: forge-test
+      workspace: forge-test
+      network: test
+      did: did:web:indexer.test.storacha.network
+      ipni-endpoint: ${{ vars.FORGE_TEST_IPNI_ENDPOINT }}
+      ipni-fallback-endpoints: ${{ vars.FORGE_TEST_IPNI_FALLBACK_ENDPOINTS }}
+      ipni-announce-urls: ${{ vars.FORGE_TEST_IPNI_ANNOUNCE_URLS }}
+      ipni-format-peer-id: ${{ vars.FORGE_TEST_IPNI_FORMAT_PEER_ID }}
+      ipni-format-endpoint: ${{ vars.FORGE_TEST_IPNI_FORMAT_ENDPOINT }}
+      apply: ${{ (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || (github.event_name == 'workflow_dispatch' && contains(github.ref, 'refs/tags')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'forge-test') }}
+    secrets:
+      aws-account-id: ${{ secrets.FORGE_TEST_AWS_ACCOUNT_ID }}
+      aws-region: ${{ secrets.FORGE_TEST_AWS_REGION }}
+      region: ${{ secrets.FORGE_TEST_AWS_REGION }}
+      private-key: ${{ secrets.FORGE_TEST_PRIVATE_KEY }}
+      cloudflare-zone-id: ${{ secrets.FORGE_TEST_CLOUDFLARE_ZONE_ID }}
+      cloudflare-api-token: ${{ secrets.FORGE_TEST_CLOUDFLARE_API_TOKEN }}
+      sentry-dsn: ${{ secrets.FORGE_TEST_SENTRY_DSN }}

.github/workflows/terraform.yml

@@ -96,9 +96,6 @@ jobs:
         with:
           aws-region: ${{ env.AWS_REGION }}
           role-to-assume: arn:aws:iam::${{ env.AWS_ACCOUNT_ID }}:role/terraform-ci
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
 
       - uses: opentofu/setup-opentofu@v1
 
@@ -108,17 +105,24 @@ jobs:
           make init
         working-directory: deploy
 
-      - name: Build + Push Docker ECR
-        run: |
-          make docker-push
-        working-directory: deploy
-
+      # just plan if !inputs.apply
       - name: Terraform Plan
         if: ${{ !inputs.apply }}
         run: |
           make plan
         working-directory: deploy
 
+      # build and push docker image and apply if inputs.apply
+      - name: Set up Docker Buildx
+        if: ${{ inputs.apply }}
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build + Push Docker ECR
+        if: ${{ inputs.apply }}
+        run: |
+          make docker-push
+        working-directory: deploy
+
       - name: Terraform Apply
         if: ${{ inputs.apply }}
         run: |

.storoku.json

@@ -93,7 +93,8 @@
   ],
   "networks": [
     "warm",
-    "forge"
+    "forge",
+    "test"
   ],
   "writeToContainer": false
 }

deploy/.env.production.local.tpl

@@ -1,5 +1,5 @@
 <%
-if [[ "$TF_WORKSPACE" == "prod" || "$TF_WORKSPACE" == "forge-prod" || ${TF_VAR_use_prod_vars:-""} == "true" ]]; then
+if [[ "$TF_WORKSPACE" == "prod" || "$TF_WORKSPACE" == "forge-prod" || "$TF_WORKSPACE" == "forge-test" || ${TF_VAR_use_prod_vars:-""} == "true" ]]; then
   PROVIDERS_CACHE_EXPIRATION_SECONDS=$((30 * 24 * 60 * 60))
   NO_PROVIDERS_CACHE_EXPIRATION_SECONDS=$((24 * 60 * 60))
   INDEXES_CACHE_EXPIRATION_SECONDS=$((24 * 60 * 60))
@@ -47,7 +47,7 @@ else
   BASE_TRACE_SAMPLE_RATIO="1.0"
 fi
 
-if [[ "$TF_WORKSPACE" == "forge-prod" || "$TF_WORKSPACE" == "forge-staging" || "$TF_WORKSPACE" == "warm-staging" ]]; then
+if [[ "$TF_WORKSPACE" == "forge-prod" || "$TF_WORKSPACE" == "forge-test" || "$TF_WORKSPACE" == "forge-staging" || "$TF_WORKSPACE" == "warm-staging" ]]; then
   SUPPORT_LEGACY_SERVICES="false"
 else
   SUPPORT_LEGACY_SERVICES="true"

deploy/app/.terraform.lock.hcl

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 5.86.0"
+      version = ">= 6.0.0"
     }
     archive = {
       source = "hashicorp/archive"
@@ -31,16 +31,10 @@ provider "aws" {
   }
 }
 
-# CloudFront is a global service. Certs must be created in us-east-1, where the core ACM infra lives
-provider "aws" {
-  region = "us-east-1"
-  alias  = "acm"
-}
-
 
 
 module "app" {
-  source             = "github.com/storacha/storoku//app?ref=v0.5.1"
+  source             = "github.com/storacha/storoku//app?ref=v0.6.2"
   private_key        = var.private_key
   httpport           = 8080
   principal_mapping  = var.principal_mapping
@@ -61,6 +55,8 @@ module "app" {
   # as env vars in the container at runtime
   secrets = {
   }
+  # enter external secrets (provisioned out-of-band) here
+  external_secrets = []
   # enter any sqs queues you want to create here
   queues = [
     {
@@ -84,7 +80,7 @@ module "app" {
       message_retention_seconds = 86400
     },
   ]
-  caches = ["providers", "no-providers", "indexes", "claims", ]
+  caches = ["providers","no-providers","indexes","claims",]
   topics = []
   tables = [
     {
@@ -142,10 +138,6 @@ module "app" {
       object_expiration_days = 14
     },
   ]
-  providers = {
-    aws     = aws
-    aws.acm = aws.acm
-  }
   env_files   = var.env_files
   domain_base = var.domain_base
 }

deploy/app/main.tf

@@ -49,7 +49,7 @@ provider "aws" {
 }
 
 module "shared" {
-  source = "github.com/storacha/storoku//shared?ref=v0.5.1"
+  source = "github.com/storacha/storoku//shared?ref=v0.6.2"
   providers = {
     aws = aws
     aws.dev = aws.dev