gitcoinco · 0xkenj1 · Jan 24, 2025 · Jan 24, 2025 · Jan 30, 2025 · Feb 3, 2025
diff --git a/.github/README.md b/.github/README.md
@@ -0,0 +1,82 @@
+# GitHub Workflows
+
+This directory contains the GitHub Actions workflows for the Grants Stack Indexer project. These workflows handle continuous integration, testing, and deployment processes.
+
+## Available Workflows
+
+| Workflow               | Description                                        |
+| ---------------------- | -------------------------------------------------- |
+| `main-workflow.yml`    | Main CI pipeline triggered on PR to dev/main       |
+| `build.yml`            | Handles project building and type checking         |
+| `build-image.yml`      | Builds and validates Docker images                 |
+| `lint.yml`             | Runs code linting and commit message validation    |
+| `test.yml`             | Executes unit tests with coverage                  |
+| `test-integration.yml` | Runs integration tests                             |
+| `deploy.yaml`          | Handles terraform deployment to staging/production |
+| `push-to-ecr.yaml`     | Pushes Docker images to Amazon ECR                 |
+
+## Main Workflow
+
+The main workflow (`main-workflow.yml`) is triggered on pull requests to `dev` and `main` branches. It orchestrates the following jobs in sequence:
+
+1. Build
+2. Build Image
+3. Lint (after Build)
+4. Tests (after Lint)
+5. Integration Tests (after Lint)
+
+## Deployment
+
+The deployment workflow (`deploy.yaml`) is manually triggered and includes:
+
+-   Environment validation (staging/production)
+-   Admin permission checks
+-   Terraform deployment steps
+
+### Usage
+
+To trigger a deployment:
+
+1. Go to Actions > Terraform Deployment with Docker Tag
+2. Click "Run workflow"
+3. Enter:
+    - Docker image tag
+    - Deployment environment (staging/production)
+
+### Required Secrets
+
+For deployment to work, the following secrets must be configured:
+
+#### Staging Environment
+
+-   `STAGING_AWS_REGION`
+-   `STAGING_TF_BACKEND_BUCKET`
+-   `STAGING_TF_BACKEND_KEY`
+
+#### Production Environment
+
+-   `PROD_AWS_REGION`
+-   `PROD_TF_BACKEND_BUCKET`
+-   `PROD_TF_BACKEND_KEY`
+
+## Docker Image Management
+
+### Building Images
+
+The `build-image.yml` workflow:
+
+-   Uses Docker Buildx
+-   Implements layer caching
+-   Targets the processing stage
+
+### ECR Push
+
+The `push-to-ecr.yaml` workflow automatically pushes images to Amazon ECR when changes are pushed to the `dev` branch.
+
+Required secrets for ECR:
+
+-   `ECR_REGISTRY`
+-   `ECR_REPOSITORY`
+-   `AWS_ACCESS_KEY_ID`
+-   `AWS_SECRET_ACCESS_KEY`
+-   `AWS_REGION`
diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml
@@ -0,0 +1,24 @@
+name: Build Image
+
+on:
+    workflow_call:
+
+jobs:
+    build-image:
+        name: Build Image
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout Repository
+              uses: actions/checkout@v3
+
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v2
+
+            - name: Build Docker Image
+              uses: docker/build-push-action@v3
+              with:
+                  context: .
+                  target: processing
+                  push: false # change to true if you want to push the image
+                  cache-from: type=gha
+                  cache-to: type=gha,mode=max
diff --git a/.github/workflows/deployment/deploy.yaml b/.github/workflows/deployment/deploy.yaml
@@ -0,0 +1,72 @@
+name: Deploy to AWS
+
+on:
+    workflow_dispatch:
+        inputs:
+            processing_image_tag:
+                description: "Processing image tag"
+                required: true
+            deploy_environment:
+                description: "Deployment environment (e.g., staging, production)"
+                required: true
+
+env:
+    TF_VAR_processing_image_tag: ${{ github.event.inputs.processing_image_tag }}
+
+jobs:
+    check-admin-permissions:
+        name: Set deployment environment
+        runs-on: ubuntu-latest
+        steps:
+            - id: check-admin
+              name: Check if user is an admin
+              run: |
+                  if [ "${{ github.event_name }}" = 'workflow_dispatch' ]; then
+                    RESPONSE=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+                    https://api.github.com/repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission)
+
+                    PERMISSION=$(echo $RESPONSE | jq -r .permission)
+
+                    if [ "$PERMISSION" != "admin" ]; then
+                      echo "::error::This workflow can only be run by admin users." && exit 1
+                    fi
+                  fi
+            - id: check-valid-environment
+              name: Check if the environment is valid
+              run: |
+                  if [ "${{ github.event.inputs.deploy_environment }}" != "staging" && "${{ github.event.inputs.deploy_environment }}" != "production" ]; then
+                    echo "::error::Invalid environment. Please choose either 'staging' or 'production'." && exit 1
+                  fi
+
+    deploy:
+        name: Terraform Deployment
+        runs-on: ubuntu-latest
+        environment:
+            name: ${{ github.event.inputs.deploy_environment }}
+        steps:
+            - name: Checkout Repository
+              uses: actions/checkout@v3
+
+            - name: Setup Terraform
+              uses: hashicorp/setup-terraform@v1
+              with:
+                  terraform_version: "1.4.6"
+
+            - name: Set up AWS CLI
+              uses: aws-actions/configure-aws-credentials@v1
+              with:
+                  aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
+                  aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+                  aws-region: us-east-2
+
+            - name: Terraform Init
+              working-directory: deployment/environments/${{ github.event.inputs.deploy_environment }}
+              run: terraform init
+
+            - name: Terraform Plan
+              working-directory: deployment/environments/${{ github.event.inputs.deploy_environment }}
+              run: terraform plan
+
+            - name: Terraform Apply with Auto Confirm
+              working-directory: deployment/environments/${{ github.event.inputs.deploy_environment }}
+              run: terraform apply -auto-approve
diff --git a/.github/workflows/deployment/push-to-ecr.yaml b/.github/workflows/deployment/push-to-ecr.yaml
@@ -0,0 +1,38 @@
+name: Push Docker Image to ECR
+
+on:
+    push:
+        branches:
+            - dev
+
+jobs:
+    push-to-ecr:
+        runs-on: ubuntu-latest
+        env:
+            ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }}
+            ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }}
+
+        steps:
+            - name: Checkout Repository
+              uses: actions/checkout@v3
+
+            - name: Configure AWS Credentials
+              uses: aws-actions/configure-aws-credentials@v4
+              with:
+                  aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
+                  aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+                  aws-region: us-east-2
+
+            - name: Log in to Amazon ECR
+              uses: aws-actions/amazon-ecr-login@v2
+
+            - name: Build Docker Image
+              run: |
+
+                  IMAGE_TAG=${{ github.sha }}
+                  docker build -t ${{ secrets.ECR_REGISTRY }}/${{ secrets.ECR_REPOSITORY }}:${IMAGE_TAG} .
+
+            - name: Push Docker Image to ECR
+              run: |
+                  IMAGE_TAG=${{ github.sha }}
+                  docker push ${{ secrets.ECR_REGISTRY }}/${{ secrets.ECR_REPOSITORY }}:${IMAGE_TAG}
diff --git a/.github/workflows/main-workflow.yml b/.github/workflows/main-workflow.yml
@@ -15,6 +15,9 @@ jobs:
     build:
         uses: ./.github/workflows/build.yml
 
+    build-image:
+        uses: ./.github/workflows/build-image.yml
+
     lint:
         uses: ./.github/workflows/lint.yml
         needs: build

diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
@@ -1,4 +1,4 @@
-name: Tests
+name: Integration Tests
 
 on:
     workflow_call:

diff --git a/deployment/.gitignore b/deployment/.gitignore
@@ -0,0 +1,23 @@
+# Directorio de dependencias locales de Terraform
+.terraform/
+
+# Archivos de estado de Terraform (contienen información sensible)
+terraform.tfstate
+terraform.tfstate.backup
+
+# Logs y archivos de crash
+crash.log
+
+# Archivos de override (no versionables)
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Archivos de configuración local de Terraform CLI
+.terraformrc
+terraform.rc
+
+# Archivos de variables sensibles
+*.tfvars
+*.tfvars.json
diff --git a/deployment/README.md b/deployment/README.md
@@ -0,0 +1,127 @@
+# Deployment Module
+
+## Overview
+
+The `deployment` module is responsible for managing the infrastructure deployment for the Grants Stack Indexer project. It utilizes Terraform to provision and manage AWS resources, ensuring a scalable and reliable environment for the application.
+
+## 🚀 Getting Started
+
+### Module Structure
+
+The `deployment` module is organized into several submodules, each responsible for different aspects of the infrastructure:
+
+-   **Networking**: Manages VPC, subnets, and security groups.
+-   **IAM**: Handles IAM roles and policies for the application services.
+-   **Compute**: Configures ECS clusters and services for running the application.
+-   **Storage**: Manages RDS instances and other storage solutions.
+-   **Load Balancer**: Sets up application load balancers for routing traffic.
+-   **Container Registry**: Manages ECR repositories for storing Docker images.
+-   **Bastion**: Provides a bastion host for secure access to the VPC.
+
+### Prerequisites
+
+-   [AWS CLI](https://aws.amazon.com/cli/) configured with appropriate permissions
+-   [Docker](https://www.docker.com/) installed
+
+### Pre conditions
+
+1. Create S3 bucket for storing terraform state
+2. Deploy docker image to the ECR repository using aws cli
+
+```bash
+aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${ECR_REPOSITORY_URL}
+docker build -t gitcoin-data-layer:latest .
+docker tag gitcoin-data-layer:latest ${ECR_REPOSITORY_URL}/gitcoin-data-layer:latest
+docker push ${ECR_REPOSITORY_URL}/gitcoin-data-layer:latest
+```
+
+### Usage
+
+To use the deployment module, follow these steps:
+
+1. Copy the `terraform.tfvars.example` file to `terraform.tfvars` and update the variables with your desired values.
+
+```bash
+cp terraform.tfvars.example terraform.tfvars
+```
+
+2. Initialize the Terraform workspace:
+
+```bash
+terraform init
+```
+
+3. Review the deployment plan:
+
+```bash
+terraform plan
+```
+
+4. Apply the changes:
+
+```bash
+terraform apply
+```
+
+5. Destroy the infrastructure (when needed):
+
+```bash
+terraform destroy
+```
+
+## Variables
+
+| Variable Name                                    | Description                                                                                                              | Sensitivity   |
+| ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------ | ------------- |
+| `app_name`                                       | The name of the application.                                                                                             | Non-sensitive |
+| `app_environment`                                | The environment in which the application is running (e.g., dev, staging, prod).                                          | Non-sensitive |
+| `region`                                         | The AWS region where the resources will be deployed.                                                                     | Non-sensitive |
+| `db_name`                                        | The name of the RDS database to be created.                                                                              | Non-sensitive |
+| `rds_username`                                   | The username for the RDS database.                                                                                       | Sensitive     |
+| `rds_password`                                   | The password for the RDS database.                                                                                       | Sensitive     |
+| `rds_security_group_id`                          | The ID of the RDS security group.                                                                                        | Non-sensitive |
+| `rds_subnet_ids`                                 | The subnet IDs for the RDS instance.                                                                                     | Non-sensitive |
+| `rds_subnet_group_name`                          | The name of the RDS subnet group.                                                                                        | Non-sensitive |
+| `processing_image_tag`                           | The tag of the processing Docker image.                                                                                  | Non-sensitive |
+| `api_repository_url`                             | The URL of the ECR repository where the API Docker image is stored.                                                      | Non-sensitive |
+| `api_image_tag`                                  | The tag of the Docker image for the API.                                                                                 | Non-sensitive |
+| `NODE_ENV`                                       | The Node environment.                                                                                                    | Non-sensitive |
+| `RETRY_MAX_ATTEMPTS`                             | The maximum number of retry attempts.                                                                                    | Non-sensitive |
+| `RETRY_BASE_DELAY_MS`                            | The base delay in milliseconds for retries.                                                                              | Non-sensitive |
+| `RETRY_MAX_DELAY_MS`                             | The maximum delay in milliseconds for retries.                                                                           | Non-sensitive |
+| `RETRY_FACTOR`                                   | The factor to increase the delay for retries.                                                                            | Non-sensitive |
+| `CHAINS`                                         | Chains to be indexed, defined as a list of objects containing `id`, `name`, `rpcUrls`, `fetchLimit`, and `fetchDelayMs`. | Sensitive     |
+| `INDEXER_GRAPHQL_URL`                            | The URL for the Indexer GraphQL API.                                                                                     | Non-sensitive |
+| `METADATA_SOURCE`                                | The source of metadata.                                                                                                  | Non-sensitive |
+| `PUBLIC_GATEWAY_URLS`                            | A list of public gateway URLs.                                                                                           | Non-sensitive |
+| `PRICING_SOURCE`                                 | The source for pricing data.                                                                                             | Non-sensitive |
+| `COINGECKO_API_KEY`                              | The API key for Coingecko.                                                                                               | Sensitive     |
+| `COINGECKO_API_TYPE`                             | The type of Coingecko API to use (e.g., pro).                                                                            | Non-sensitive |
+| `LOG_LEVEL`                                      | The logging level for the application.                                                                                   | Non-sensitive |
+| `DATALAYER_PG_USER`                              | The username for the PostgreSQL database.                                                                                | Sensitive     |
+| `DATALAYER_PG_PASSWORD`                          | The password for the PostgreSQL database.                                                                                | Sensitive     |
+| `DATALAYER_PG_DB_NAME`                           | The name of the PostgreSQL database.                                                                                     | Non-sensitive |
+| `DATALAYER_PG_EXPOSED_PORT`                      | The port on which the PostgreSQL database is exposed.                                                                    | Non-sensitive |
+| `DATALAYER_HASURA_EXPOSED_PORT`                  | The port on which the Hasura GraphQL engine is exposed.                                                                  | Non-sensitive |
+| `DATALAYER_HASURA_ENABLE_CONSOLE`                | Whether to enable the Hasura console.                                                                                    | Non-sensitive |
+| `DATALAYER_HASURA_ADMIN_SECRET`                  | The admin secret for Hasura.                                                                                             | Sensitive     |
+| `DATALAYER_HASURA_UNAUTHORIZED_ROLE`             | The unauthorized role for Hasura.                                                                                        | Non-sensitive |
+| `DATALAYER_HASURA_CORS_DOMAIN`                   | The CORS domain for Hasura.                                                                                              | Non-sensitive |
+| `DATALAYER_HASURA_ENABLE_TELEMETRY`              | Whether to enable telemetry for Hasura.                                                                                  | Non-sensitive |
+| `DATALAYER_HASURA_DEV_MODE`                      | Whether to enable development mode for Hasura.                                                                           | Non-sensitive |
+| `DATALAYER_HASURA_ADMIN_INTERNAL_ERRORS`         | Whether to enable internal errors for Hasura.                                                                            | Non-sensitive |
+| `DATALAYER_HASURA_CONSOLE_ASSETS_DIR`            | The directory for console assets in Hasura.                                                                              | Non-sensitive |
+| `DATALAYER_HASURA_EXPERIMENTAL_FEATURES`         | The experimental features for Hasura.                                                                                    | Non-sensitive |
+| `DATALAYER_HASURA_DEFAULT_NAMING_CONVENTION`     | The default naming convention for Hasura.                                                                                | Non-sensitive |
+| `DATALAYER_HASURA_BIGQUERY_STRING_NUMERIC_INPUT` | Whether to enable BigQuery string numeric input for Hasura.                                                              | Non-sensitive |
+
+## Github Secrets
+
+For the following variables, you need to create secrets in the Github repository settings for the environments production and staging.
+
+-   `CHAINS`
+-   `COINGECKO_API_KEY`
+-   `DATALAYER_PG_USER`
+-   `DATALAYER_PG_PASSWORD`
+-   `DATALAYER_PG_DB_NAME`
+-   `DATALAYER_HASURA_ADMIN_SECRET`