Merge pull request #187 from American-Institutes-for-Research/INF-526/migrate-air-githuborg

rhunwicks · web-flow · commit 15bf8bb19d6c · 2025-10-22T10:24:34.000-04:00
INF-526 Update to use the AIR Github setup
diff --git a/.github/workflows/01-build-then-test.yml b/.github/workflows/01-build-then-test.yml
@@ -29,7 +29,7 @@ env:
 
 jobs:
   lint:
-    runs-on: [ data ]
+    runs-on: [ fewsnet ]
     container: "python:3.11"
 
     steps:
@@ -51,7 +51,7 @@ jobs:
   start-runner-test_branch:
     if: ${{ !(startsWith(github.ref, 'refs/tags/')) && !(github.ref == 'refs/heads/main') }}
     needs: lint
-    runs-on: [ data ]
+    runs-on: [ fewsnet ]
     outputs:
       label: ${{ steps.start-ec2-runner.outputs.label }}
       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
@@ -62,24 +62,14 @@ jobs:
           aws-region: ${{ vars.AWS_REGION }}
       - name: Start EC2 runner test_branch
         id: start-ec2-runner
-        uses: FEWS-NET/ec2-github-runner@1f687d4b786b21bed4ad6f420daddc7665bff857
+        uses: American-Institutes-for-Research/gitops-actions/fewsnet-start-runner@main
         with:
-          mode: start
-          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-          ec2-image-id: ami-092d88ab9dba1ddc0
-          ec2-instance-type: m5a.large
-          subnet-id: subnet-0f50e5fa11397a87b
-          security-group-id: sg-0c844cef0ffc13cde
-          iam-role-name: data-prod-github-runner-role # optional, requires additional permissions
-          aws-resource-tags: > # optional, requires additional permissions
-            [
-              {"Key": "Name", "Value": "HEA-github-autoscale-runner"},
-              {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}
-            ]
+          gh_personal_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          project-key: "HEA"
 
   start-runner-test_keepdb:
     needs: lint
-    runs-on: [ data ]
+    runs-on: [ fewsnet ]
     outputs:
       label: ${{ steps.start-ec2-runner.outputs.label }}
       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
@@ -90,25 +80,15 @@ jobs:
           aws-region: ${{ env.AWS_REGION }}
       - name: Start EC2 runner test_keepdb
         id: start-ec2-runner
-        uses: FEWS-NET/ec2-github-runner@1f687d4b786b21bed4ad6f420daddc7665bff857
+        uses: American-Institutes-for-Research/gitops-actions/fewsnet-start-runner@main
         with:
-          mode: start
-          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-          ec2-image-id: ami-092d88ab9dba1ddc0
-          ec2-instance-type: m5a.large
-          subnet-id: subnet-0f50e5fa11397a87b
-          security-group-id: sg-0c844cef0ffc13cde
-          iam-role-name: data-prod-github-runner-role # optional, requires additional permissions
-          aws-resource-tags: > # optional, requires additional permissions
-            [
-              {"Key": "Name", "Value": "HEA-github-autoscale-runner"},
-              {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}
-            ]
+          gh_personal_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          project-key: "HEA"
 
   start-runner-test_main:
     if: github.ref == 'refs/heads/main'
     needs: lint
-    runs-on: [ data ]
+    runs-on: [ fewsnet ]
     outputs:
       label: ${{ steps.start-ec2-runner.outputs.label }}
       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
@@ -119,25 +99,15 @@ jobs:
           aws-region: ${{ vars.AWS_REGION }}
       - name: Start EC2 runner test_main
         id: start-ec2-runner
-        uses: FEWS-NET/ec2-github-runner@1f687d4b786b21bed4ad6f420daddc7665bff857
+        uses: American-Institutes-for-Research/gitops-actions/fewsnet-start-runner@main
         with:
-          mode: start
-          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-          ec2-image-id: ami-092d88ab9dba1ddc0
-          ec2-instance-type: m5a.large
-          subnet-id: subnet-0f50e5fa11397a87b
-          security-group-id: sg-0c844cef0ffc13cde
-          iam-role-name: data-prod-github-runner-role # optional, requires additional permissions
-          aws-resource-tags: > # optional, requires additional permissions
-            [
-              {"Key": "Name", "Value": "HEA-github-autoscale-runner"},
-              {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}
-            ]
+          gh_personal_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          project-key: "HEA"
 
   start-runner-test_tag:
     if: startsWith(github.ref, 'refs/tags/')
     needs: lint
-    runs-on: [ data ]
+    runs-on: [ fewsnet ]
     outputs:
       label: ${{ steps.start-ec2-runner.outputs.label }}
       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
@@ -148,20 +118,10 @@ jobs:
           aws-region: ${{ env.AWS_REGION }}
       - name: Start EC2 runner test_tag
         id: start-ec2-runner
-        uses: FEWS-NET/ec2-github-runner@1f687d4b786b21bed4ad6f420daddc7665bff857
+        uses: American-Institutes-for-Research/gitops-actions/fewsnet-start-runner@main
         with:
-          mode: start
-          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-          ec2-image-id: ami-092d88ab9dba1ddc0
-          ec2-instance-type: m5a.large
-          subnet-id: subnet-0f50e5fa11397a87b
-          security-group-id: sg-0c844cef0ffc13cde
-          iam-role-name: data-prod-github-runner-role # optional, requires additional permissions
-          aws-resource-tags: > # optional, requires additional permissions
-            [
-              {"Key": "Name", "Value": "HEA-github-autoscale-runner"},
-              {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}
-            ]
+          gh_personal_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          project-key: "HEA"
 
   test_branch:
     if: ${{ !(startsWith(github.ref, 'refs/tags/')) && !(github.ref == 'refs/heads/main') }}
@@ -173,10 +133,10 @@ jobs:
     needs:
       - start-runner-test_branch
     container:
-      image: ghcr.io/fews-net/inf/docker:latest
+      image: airorg/fewsnet-inf:docker-latest
       credentials:
-        username: ${{ github.repository_owner }}
-        password: ${{ secrets.GHCR_PAT }}
+        username: ${{ secrets.DOCKER_HUB_USERNAME }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
     defaults:
       run:
         shell: ash --noprofile --norc -eo pipefail {0} # Fail on any non-zero exit code, even in piped commands
@@ -207,7 +167,7 @@ jobs:
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
       - name: "Pull previous images to speed up builds"
         run: |
           for SERVICE in ${BUILD_IMAGES}; do (docker pull ${CI_REGISTRY_IMAGE}/${SERVICE}:edge | grep -i -e 'Pulling from' -e Digest -e Status -e Error) || true; done
@@ -316,10 +276,10 @@ jobs:
     needs:
       - start-runner-test_keepdb
     container:
-      image: ghcr.io/fews-net/inf/docker:latest
+      image: airorg/fewsnet-inf:docker-latest
       credentials:
-        username: ${{ github.repository_owner }}
-        password: ${{ secrets.GHCR_PAT }}
+        username: ${{ secrets.DOCKER_HUB_USERNAME }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
     defaults:
       run:
         shell: ash --noprofile --norc -eo pipefail {0} # Fail on any non-zero exit code, even in piped commands
@@ -350,7 +310,7 @@ jobs:
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
       - name: "Pull previous images to speed up builds"
         run: |
           for SERVICE in ${BUILD_IMAGES}; do (docker pull ${CI_REGISTRY_IMAGE}/${SERVICE}:edge | grep -i -e 'Pulling from' -e Digest -e Status -e Error) || true; done
@@ -441,10 +401,10 @@ jobs:
     needs:
       - start-runner-test_main
     container:
-      image: ghcr.io/fews-net/inf/docker:latest
+      image: airorg/fewsnet-inf:docker-latest
       credentials:
-        username: ${{ github.repository_owner }}
-        password: ${{ secrets.GHCR_PAT }}
+        username: ${{ secrets.DOCKER_HUB_USERNAME }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
     defaults:
       run:
         shell: ash --noprofile --norc -eo pipefail {0} # Fail on any non-zero exit code, even in piped commands
@@ -475,7 +435,7 @@ jobs:
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
       - name: "Pull previous images to speed up builds"
         run: |
           echo Using registry image ${CI_REGISTRY_IMAGE}
@@ -569,10 +529,10 @@ jobs:
     needs:
       - start-runner-test_tag
     container:
-      image: ghcr.io/fews-net/inf/docker:latest
+      image: airorg/fewsnet-inf:docker-latest
       credentials:
-        username: ${{ github.repository_owner }}
-        password: ${{ secrets.GHCR_PAT }}
+        username: ${{ secrets.DOCKER_HUB_USERNAME }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
     defaults:
       run:
         shell: ash --noprofile --norc -eo pipefail {0} # Fail on any non-zero exit code, even in piped commands
@@ -603,7 +563,7 @@ jobs:
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
       - uses: "actions/checkout@v4"
         with:
           ssh-key: ${{ secrets.GIT_SSH_PRIVATE_KEY }}
@@ -694,15 +654,15 @@ jobs:
     needs:
       - start-runner-test_branch # required to get output from the start-runner-test_branch job
       - test_branch # required to wait until the main jobs are done
-    runs-on: [ data ]
+    runs-on: [ fewsnet ]
 
     steps:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: ${{ env.AWS_REGION }}
       - name: Stop EC2 runner
-        uses: FEWS-NET/ec2-github-runner@1f687d4b786b21bed4ad6f420daddc7665bff857
+        uses: American-Institutes-for-Research/ec2-github-runner@598310b0225c539534b6ef47575f36404dedf3b1
         with:
           mode: stop
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
@@ -714,15 +674,15 @@ jobs:
     needs:
       - start-runner-test_keepdb # required to get output from the start-runner-test_keepdb job
       - test_keepdb
-    runs-on: [ data ]
+    runs-on: [ fewsnet ]
 
     steps:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: ${{ env.AWS_REGION }}
       - name: Stop EC2 runner
-        uses: FEWS-NET/ec2-github-runner@1f687d4b786b21bed4ad6f420daddc7665bff857
+        uses: American-Institutes-for-Research/ec2-github-runner@598310b0225c539534b6ef47575f36404dedf3b1
         with:
           mode: stop
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
@@ -734,15 +694,15 @@ jobs:
     needs:
       - start-runner-test_main # required to get output from the start-runner-test_main job
       - test_main
-    runs-on: [ data ]
+    runs-on: [ fewsnet ]
 
     steps:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: ${{ env.AWS_REGION }}
       - name: Stop EC2 runner
-        uses: FEWS-NET/ec2-github-runner@1f687d4b786b21bed4ad6f420daddc7665bff857
+        uses: American-Institutes-for-Research/ec2-github-runner@598310b0225c539534b6ef47575f36404dedf3b1
         with:
           mode: stop
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
@@ -754,15 +714,15 @@ jobs:
     needs:
       - start-runner-test_tag # required to get output from the start-runner-test_tag job
       - test_tag
-    runs-on: [ data ]
+    runs-on: [ fewsnet ]
 
     steps:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: ${{ env.AWS_REGION }}
       - name: Stop EC2 runner
-        uses: FEWS-NET/ec2-github-runner@1f687d4b786b21bed4ad6f420daddc7665bff857
+        uses: American-Institutes-for-Research/ec2-github-runner@598310b0225c539534b6ef47575f36404dedf3b1
         with:
           mode: stop
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
diff --git a/.github/workflows/02-deploy-env.yml b/.github/workflows/02-deploy-env.yml
@@ -24,7 +24,7 @@ jobs:
   deploy_env:
     # this will allow us to access secrets defined for this environment
     environment: ${{ inputs.environment }}
-    runs-on: [ data ]
+    runs-on: [ fewsnet ]
     container:
       image: alpine:3.17
     defaults:
diff --git a/.github/workflows/03-mirror-to-fdw.yml b/.github/workflows/03-mirror-to-fdw.yml
@@ -0,0 +1,41 @@
+name: Mirror to FDW
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches:
+      - "main"
+    tags:
+      - "*"
+
+jobs:
+  mirror:
+    runs-on: [ fewsnet ]
+    # run in container to avoid file permission issues like EACCES: permission denied
+    container: "python:3.12"
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          # we don't want the current commit only,
+          # but also previous commits in case those haven't been synced
+          fetch-depth: 0
+
+        # since we are running this on the runner controller (like lint), action/checkout
+        # will not create a new repo for subsequent runs. This lead to
+        # error: remote mirror already exists.
+        # We will just remove the remote here if it already exists, so mirror-action works
+      - name: Remove existing remote
+        run: |
+          # seems to be necessary to run git commands
+          git config --global --add safe.directory /__w/FEWSNET-HEA-Database-Development/FEWSNET-HEA-Database-Development
+          git remote remove mirror || echo "No remote 'mirror' existed"
+
+      - uses: yesolutions/mirror-action@v0.7.0
+        with:
+          REMOTE: "ssh://git@github.com/FEWS-NET/HEA-Database-Development.git"
+          GIT_SSH_PRIVATE_KEY: ${{ secrets.FDW_GIT_SSH_PRIVATE_KEY }}
+          # @TODO: change this to secrets.GIT_SSH_KNOWN_HOSTS
+          GIT_SSH_NO_VERIFY_HOST: "true"
+          PUSH_ALL_REFS: "false"
