feat: terraform deployment

.env.example

@@ -61,8 +61,6 @@ HASURA_GRAPHQL_UNAUTHORIZED_ROLE=public
 RPC_URLS=["https://optimism.llamarpc.com","https://rpc.ankr.com/optimism","https://optimism.gateway.tenderly.co","https://optimism.blockpi.network/v1/rpc/public","https://mainnet.optimism.io","https://opt-mainnet.g.alchemy.com/v2/demo"]
 CHAIN_ID=10
 
-FETCH_LIMIT=1000
-FETCH_DELAY_MS=3000
 
 DATABASE_URL=postgresql://postgres:testing@datalayer-postgres-db:5432/datalayer-postgres-db
 DATABASE_SCHEMA=chain_data_schema_1

Dockerfile

@@ -9,9 +9,9 @@ COPY . /usr/src/app
 WORKDIR /usr/src/app
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
 RUN pnpm run -r build
-RUN pnpm deploy --filter=./apps/processing --prod /prod/processing
+RUN pnpm deploy --filter=./apps/processing  /prod/processing
 
 FROM base AS processing
 COPY --from=build /prod/processing /prod/processing
 WORKDIR /prod/processing
-CMD [ "pnpm", "start" ]
+CMD [ "pnpm", "start" ]

apps/processing/src/config/env.ts

@@ -23,6 +23,7 @@ const chainSchema = z.object({
 });
 
 const baseSchema = z.object({
+    NODE_ENV: z.enum(["development", "staging", "production"]).default("development"),
     CHAINS: stringToJSONSchema.pipe(z.array(chainSchema).nonempty()).refine((chains) => {
         const ids = chains.map((chain) => chain.id);
         const uniqueIds = new Set(ids);
@@ -31,14 +32,12 @@ const baseSchema = z.object({
     DATABASE_URL: z.string(),
     DATABASE_SCHEMA: z.string().default("public"),
     INDEXER_GRAPHQL_URL: z.string().url(),
-    INDEXER_ADMIN_SECRET: z.string(),
+    INDEXER_ADMIN_SECRET: z.string().optional(),
     PRICING_SOURCE: z.enum(["dummy", "coingecko"]).default("coingecko"),
-    IPFS_GATEWAYS_URL: stringToJSONSchema
-        .pipe(z.array(z.string().url()))
-        .default('["https://ipfs.io"]'),
+    METADATA_SOURCE: z.enum(["dummy", "pinata", "public-gateway"]).default("dummy"),
     RETRY_MAX_ATTEMPTS: z.coerce.number().int().min(1).default(3),
     RETRY_BASE_DELAY_MS: z.coerce.number().int().min(1).default(3000), // 3 seconds
-    RETRY_FACTOR: z.coerce.number().int().min(1).default(2),
+    RETRY_FACTOR: z.coerce.number().min(1).default(2),
     RETRY_MAX_DELAY_MS: z.coerce.number().int().min(1).optional(), // 5 minute
 });
 
@@ -53,6 +52,21 @@ const coingeckoPricingSchema = baseSchema.extend({
     COINGECKO_API_TYPE: z.enum(["demo", "pro"]).default("pro"),
 });
 
+const dummyMetadataSchema = baseSchema.extend({
+    METADATA_SOURCE: z.literal("dummy"),
+});
+
+const pinataMetadataSchema = baseSchema.extend({
+    METADATA_SOURCE: z.literal("pinata"),
+    PINATA_JWT: z.string().min(1),
+    PINATA_GATEWAY_URL: z.string().url(),
+});
+
+const publicGatewayMetadataSchema = baseSchema.extend({
+    METADATA_SOURCE: z.literal("public-gateway"),
+    PUBLIC_GATEWAY_URLS: stringToJSONSchema.pipe(z.array(z.string().url())),
+});
+
 const validationSchema = z
     .discriminatedUnion("PRICING_SOURCE", [dummyPricingSchema, coingeckoPricingSchema])
     .transform((val) => {
@@ -66,7 +80,36 @@ const validationSchema = z
             apiType: val.COINGECKO_API_TYPE,
             ...val,
         };
-    });
+    })
+    .and(
+        z
+            .discriminatedUnion("METADATA_SOURCE", [
+                dummyMetadataSchema,
+                pinataMetadataSchema,
+                publicGatewayMetadataSchema,
+            ])
+            .transform((val) => {
+                if (val.METADATA_SOURCE === "dummy") {
+                    return { metadataSource: val.METADATA_SOURCE, ...val };
+                }
+                if (val.METADATA_SOURCE === "pinata") {
+                    return {
+                        metadataSource: val.METADATA_SOURCE,
+                        jwt: val.PINATA_JWT,
+                        gateway: val.PINATA_GATEWAY_URL,
+                        ...val,
+                    };
+                }
+                if (val.METADATA_SOURCE === "public-gateway") {
+                    return {
+                        metadataSource: val.METADATA_SOURCE,
+                        gateways: val.PUBLIC_GATEWAY_URLS,
+                        ...val,
+                    };
+                }
+                throw new Error("Invalid metadata source");
+            }),
+    );
 
 const env = validationSchema.safeParse(process.env);
 

apps/processing/src/services/sharedDependencies.service.ts

@@ -1,6 +1,6 @@
 import { CoreDependencies } from "@grants-stack-indexer/data-flow";
 import { EnvioIndexerClient } from "@grants-stack-indexer/indexer-client";
-import { CachingMetadataProvider, IpfsProvider } from "@grants-stack-indexer/metadata";
+import { CachingMetadataProvider, MetadataProviderFactory } from "@grants-stack-indexer/metadata";
 import { CachingPricingProvider, PricingProviderFactory } from "@grants-stack-indexer/pricing";
 import {
     createKyselyDatabase,
@@ -52,6 +52,7 @@ export class SharedDependenciesService {
         const kyselyDatabase = createKyselyDatabase(
             {
                 connectionString: env.DATABASE_URL,
+                isProduction: env.NODE_ENV === "production" || env.NODE_ENV === "staging",
             },
             logger,
         );
@@ -89,7 +90,11 @@ export class SharedDependenciesService {
         );
 
         const metadataRepository = new KyselyMetadataCache(kyselyDatabase, env.DATABASE_SCHEMA);
-        const internalMetadataProvider = new IpfsProvider(env.IPFS_GATEWAYS_URL, logger);
+
+        const internalMetadataProvider = MetadataProviderFactory.create(env, {
+            logger,
+        });
+
         const dbCachedMetadataProvider = new CachingMetadataProvider(
             internalMetadataProvider,
             metadataRepository,

apps/processing/test/unit/sharedDependencies.service.spec.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it, vi } from "vitest";
 
 import { EnvioIndexerClient } from "@grants-stack-indexer/indexer-client";
-import { IpfsProvider } from "@grants-stack-indexer/metadata";
+import { PublicGatewayProvider } from "@grants-stack-indexer/metadata";
 import { PricingProviderFactory } from "@grants-stack-indexer/pricing";
 import { createKyselyDatabase } from "@grants-stack-indexer/repository";
 
@@ -103,17 +103,17 @@ describe("SharedDependenciesService", () => {
         Environment,
         | "DATABASE_URL"
         | "DATABASE_SCHEMA"
-        | "IPFS_GATEWAYS_URL"
         | "INDEXER_GRAPHQL_URL"
         | "INDEXER_ADMIN_SECRET"
         | "PRICING_SOURCE"
+        | "METADATA_SOURCE"
     > = {
         DATABASE_URL: "postgresql://localhost:5432/test",
         DATABASE_SCHEMA: "public",
-        IPFS_GATEWAYS_URL: ["https://ipfs.io"],
         INDEXER_GRAPHQL_URL: "http://localhost:8080",
         INDEXER_ADMIN_SECRET: "secret",
         PRICING_SOURCE: "dummy",
+        METADATA_SOURCE: "public-gateway",
     };
 
     it("initializes all dependencies correctly", async () => {
@@ -131,7 +131,7 @@ describe("SharedDependenciesService", () => {
         expect(PricingProviderFactory.create).toHaveBeenCalledWith(mockEnv, {
             logger: mocks.logger,
         });
-        expect(IpfsProvider).toHaveBeenCalledWith(mockEnv.IPFS_GATEWAYS_URL, mocks.logger);
+        expect(PublicGatewayProvider).toHaveBeenCalledWith(mockEnv, mocks.logger);
 
         // Verify indexer client initialization
         expect(EnvioIndexerClient).toHaveBeenCalledWith(

deployment/.gitignore

@@ -0,0 +1,23 @@
+# Directorio de dependencias locales de Terraform
+.terraform/
+
+# Archivos de estado de Terraform (contienen información sensible)
+terraform.tfstate
+terraform.tfstate.backup
+
+# Logs y archivos de crash
+crash.log
+
+# Archivos de override (no versionables)
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Archivos de configuración local de Terraform CLI
+.terraformrc
+terraform.rc
+
+# Archivos de variables sensibles
+*.tfvars
+*.tfvars.json

deployment/environments/staging/main.tf

@@ -0,0 +1,113 @@
+terraform {
+  backend "s3" {
+    bucket  = "gitcoin-datalayer-staging-terraform-state"
+    key     = "state"
+    region  = "us-east-2"
+    encrypt = true
+  }
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "5.84.0"
+    }
+  }
+}
+
+provider "aws" {
+  region = "us-east-2"
+}
+
+
+data "aws_caller_identity" "current" {}
+
+module "container_registry" {
+  source   = "../../modules/container-registry"
+  app_name = var.app_name
+}
+
+
+
+module "networking" {
+  source          = "../../modules/networking"
+  app_environment = var.app_environment
+  app_name        = var.app_name
+  region          = var.region
+}
+
+module "iam" {
+  source          = "../../modules/iam"
+  app_name        = var.app_name
+  app_environment = var.app_environment
+  region          = var.region
+  account_id      = data.aws_caller_identity.current.account_id
+  db_name         = var.DATALAYER_PG_DB_NAME
+}
+
+module "storage" {
+  source                = "../../modules/storage"
+  app_name              = var.app_name
+  app_environment       = var.app_environment
+  region                = var.region
+  db_name               = var.DATALAYER_PG_DB_NAME
+  rds_username          = var.DATALAYER_PG_USER
+  rds_password          = var.DATALAYER_PG_PASSWORD
+  rds_security_group_id = module.networking.rds_security_group_id
+  rds_subnet_ids        = module.networking.private_subnets
+  rds_subnet_group_name = module.networking.rds_subnet_group_name
+}
+
+module "bastion" {
+  source                        = "../../modules/bastion"
+  app_environment               = var.app_environment
+  app_name                      = var.app_name
+  subnet_id                     = module.networking.private_subnets[0]
+  bastion_instance_profile_name = module.iam.bastion_instance_profile_name
+}
+
+module "compute" {
+  source                                         = "../../modules/compute"
+  app_name                                       = var.app_name
+  app_environment                                = var.app_environment
+  region                                         = var.region
+  processing_repository_url                      = module.container_registry.processing_repository_url
+  processing_service_role_arn                    = module.iam.processing_service_role_arn
+  processing_image_tag                           = var.processing_image_tag
+  processing_security_group_id                   = module.networking.processing_security_group_id
+  api_image_tag                                  = var.api_image_tag
+  api_repository_url                             = var.api_repository_url
+  api_service_role_arn                           = module.iam.api_service_role_arn
+  api_security_group_id                          = module.networking.api_security_group_id
+  NODE_ENV                                       = var.NODE_ENV
+  RETRY_BASE_DELAY_MS                            = var.RETRY_BASE_DELAY_MS
+  RETRY_MAX_DELAY_MS                             = var.RETRY_MAX_DELAY_MS
+  RETRY_FACTOR                                   = var.RETRY_FACTOR
+  RETRY_MAX_ATTEMPTS                             = var.RETRY_MAX_ATTEMPTS
+  DATALAYER_HASURA_DATABASE_URL                  = "postgresql://${var.DATALAYER_PG_USER}:${var.DATALAYER_PG_PASSWORD}@${module.storage.rds_endpoint}/${var.DATALAYER_PG_DB_NAME}"
+  DATALAYER_HASURA_EXPOSED_PORT                  = var.DATALAYER_HASURA_EXPOSED_PORT
+  DATALAYER_HASURA_ENABLE_CONSOLE                = var.DATALAYER_HASURA_ENABLE_CONSOLE
+  DATALAYER_HASURA_ADMIN_SECRET                  = var.DATALAYER_HASURA_ADMIN_SECRET
+  DATALAYER_HASURA_UNAUTHORIZED_ROLE             = var.DATALAYER_HASURA_UNAUTHORIZED_ROLE
+  DATALAYER_HASURA_CORS_DOMAIN                   = var.DATALAYER_HASURA_CORS_DOMAIN
+  DATALAYER_HASURA_ENABLE_TELEMETRY              = var.DATALAYER_HASURA_ENABLE_TELEMETRY
+  DATALAYER_HASURA_DEV_MODE                      = var.DATALAYER_HASURA_DEV_MODE
+  DATALAYER_HASURA_ADMIN_INTERNAL_ERRORS         = var.DATALAYER_HASURA_ADMIN_INTERNAL_ERRORS
+  DATALAYER_HASURA_CONSOLE_ASSETS_DIR            = var.DATALAYER_HASURA_CONSOLE_ASSETS_DIR
+  DATALAYER_HASURA_ENABLED_LOG_TYPES             = var.DATALAYER_HASURA_ENABLED_LOG_TYPES
+  DATALAYER_HASURA_DEFAULT_NAMING_CONVENTION     = var.DATALAYER_HASURA_DEFAULT_NAMING_CONVENTION
+  DATALAYER_HASURA_BIGQUERY_STRING_NUMERIC_INPUT = var.DATALAYER_HASURA_BIGQUERY_STRING_NUMERIC_INPUT
+  DATALAYER_HASURA_EXPERIMENTAL_FEATURES         = var.DATALAYER_HASURA_EXPERIMENTAL_FEATURES
+  CHAINS                                         = var.CHAINS
+
+  DATABASE_URL        = "postgresql://${var.DATALAYER_PG_USER}:${var.DATALAYER_PG_PASSWORD}@${module.storage.rds_endpoint}/${var.DATALAYER_PG_DB_NAME}"
+  INDEXER_GRAPHQL_URL = var.INDEXER_GRAPHQL_URL
+  # INDEXER_ADMIN_SECRET                           = var.INDEXER_ADMIN_SECRET
+  PUBLIC_GATEWAY_URLS = var.PUBLIC_GATEWAY_URLS
+  METADATA_SOURCE     = var.METADATA_SOURCE
+  PRICING_SOURCE      = var.PRICING_SOURCE
+  COINGECKO_API_KEY   = var.COINGECKO_API_KEY
+  COINGECKO_API_TYPE  = var.COINGECKO_API_TYPE
+  LOG_LEVEL           = var.LOG_LEVEL
+  public_subnets      = module.networking.public_subnets
+  private_subnets     = module.networking.private_subnets
+}