Add cubestore script

Author: singhals

.github/workflows/deploy.yml

@@ -26,13 +26,13 @@ jobs:
         with:
           mask-password: "false"
 
-      - name: Build, tag, and push docker image to Amazon ECR
+      - name: Build, tag, and push cube api docker image to Amazon ECR
         env:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           REPOSITORY: prod-sync-cube-ecr
           IMAGE_TAG: "${{ github.sha }}"
         run: |
-          docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG .
+          docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG -f docker/cube/Dockerfile .
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
 
       - name: Update cube-api Task Definition with latest image

deploy_cubestore.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+AWS_REGION="us-east-1"
+ECR_REPOSITORY="prod-sync-cubestore-ecr"
+
+AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text)
+REGISTRY="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com"
+IMAGE_TAG=$(git rev-parse --short HEAD 2>/dev/null || date +%s)
+
+aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $REGISTRY
+
+docker build --platform linux/amd64 -t $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f docker/cubestore/Dockerfile .
+docker push $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+
+echo "New cubestore image pushed to ECR: $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG. Please update terraform cubestore services task definitions accordingly."

Dockerfile renamed to docker/cube/Dockerfile

@@ -0,0 +1,3 @@
+FROM cubejs/cubestore:latest
+
+RUN apt-get update && apt-get install -y curl

docker/cubestore/Dockerfile

@@ -29,4 +29,37 @@ resource "aws_ecr_lifecycle_policy" "cube_lf_policy" {
     ]
 }
 EOF
+}
+
+resource "aws_ecr_repository" "cubestore_repo" {
+  name                 = "${var.cluster_prefix}-cubestore-ecr"
+  image_tag_mutability = "IMMUTABLE"
+
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+}
+
+resource "aws_ecr_lifecycle_policy" "cubestore_lf_policy" {
+  repository = aws_ecr_repository.cubestore_repo.name
+
+  policy = <<EOF
+{
+    "rules": [
+        {
+            "rulePriority": 1,
+            "description": "Keep last 30 images",
+            "selection": {
+                "tagStatus": "tagged",
+                "tagPrefixList": ["v"],
+                "countType": "imageCountMoreThan",
+                "countNumber": 30
+            },
+            "action": {
+                "type": "expire"
+            }
+        }
+    ]
+}
+EOF
 }

terraform/modules/cube-cluster/ecr.tf

@@ -380,7 +380,7 @@ resource "aws_ecs_task_definition" "cubestore_router" {
   container_definitions = jsonencode([
     {
       name      = "cubestore-router"
-      image     = "${var.cubestore_image}"
+      image     = "${aws_ecr_repository.cubestore_repo.repository_url}:8311306"
       cpu       = tonumber(var.cubestore_router_resources.cpu)
       memory    = tonumber(var.cubestore_router_resources.memory)
       essential = true
@@ -502,7 +502,7 @@ resource "aws_ecs_task_definition" "cubestore" {
   container_definitions = jsonencode([
     {
       name      = "cubestore"
-      image     = "${var.cubestore_image}"
+      image     = "${aws_ecr_repository.cubestore_repo.repository_url}:8311306"
       cpu       = tonumber(var.cubestore_worker_resources.cpu)
       memory    = tonumber(var.cubestore_worker_resources.memory)
       essential = true

terraform/modules/cube-cluster/ecs.tf

@@ -27,7 +27,7 @@ resource "aws_iam_policy" "cube_repo_ecr_policy" {
           "ecr:PutImage",
           "ecr:UploadLayerPart"
         ],
-        "Resource" : aws_ecr_repository.cube_repo.arn
+        "Resource" : [aws_ecr_repository.cube_repo.arn, aws_ecr_repository.cubestore_repo.arn]
       },
       {
         "Sid" : "AllowEcsServiceDeploys",

terraform/modules/cube-cluster/iam.tf

@@ -7,25 +7,43 @@ variable "vpc" {
   type        = any
 }
 
+data "aws_vpc" "selected" {
+  id = var.vpc.vpc_id
+}
+
+data "aws_nat_gateways" "selected" {
+  filter {
+    name   = "vpc-id"
+    values = [var.vpc.vpc_id]
+  }
+}
+
+data "aws_internet_gateway" "selected" {
+  filter {
+    name   = "attachment.vpc-id"
+    values = [var.vpc.vpc_id]
+  }
+}
+
 resource "null_resource" "validate_vpc" {
   lifecycle {
     precondition {
-      condition     = var.vpc.enable_dns_support
+      condition     = data.aws_vpc.selected.enable_dns_support
       error_message = "The VPC must have enable_dns_support = true"
     }
 
     precondition {
-      condition     = var.vpc.enable_dns_hostnames
+      condition     = data.aws_vpc.selected.enable_dns_hostnames
       error_message = "The VPC must have enable_dns_hostnames = true"
     }
 
     precondition {
-      condition     = var.vpc.enable_nat_gateway
+      condition     = length(data.aws_nat_gateways.selected.ids) > 0
       error_message = "The VPC must have at least one NAT Gateway"
     }
 
     precondition {
-      condition     = var.vpc.create_igw
+      condition     = can(data.aws_internet_gateway.selected.id)
       error_message = "The VPC must have an Internet Gateway"
     }
   }