write tests + revamp ci & makefile

.github/workflows/keepalive.yml

@@ -0,0 +1,17 @@
+name: Keep Alive
+on:
+  schedule:
+    - cron: "0 0 * * *"
+jobs:
+  main-job:
+    name: Main Job
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+  workflow-keepalive:
+    if: github.event_name == 'schedule'
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+    steps:
+      - uses: liskin/gh-workflow-keepalive@v1

.github/workflows/main.yml

@@ -1,4 +1,4 @@
-name: LocalStack Setup
+name: Deploy infrastructure using CloudFormation
 
 on:
   push:
@@ -16,44 +16,42 @@ on:
 
 jobs:
   cloudformation:
-    name: Deploy infrastructure using CloudFormation
+    name: Run integration tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
+      - name: Set up Python
         id: setup-python
         uses: actions/setup-python@v2
         with:
-          python-version: "3.10"
+          python-version: "3.12"
 
       - name: Start LocalStack
+        uses: LocalStack/setup-localstack@main
+        with:
+          image-tag: 'latest'
+          use-pro: 'true'
+          configuration: LS_LOG=trace
+          install-awslocal: 'true'
         env:
-          LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }}
-          DNS_ADDRESS: 0
-        run: |
-          pip install localstack awscli-local[ver1]
-          docker pull localstack/localstack-pro:latest
-
-          # Start LocalStack in the background
-          localstack start -d
-
-          # Wait 30 seconds for the LocalStack container to become ready before timing out
-          echo "Waiting for LocalStack startup..."
-          localstack wait -t 15
-
-          echo "Startup complete"
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
+          IMAGE_NAME: 'localstack/localstack-pro:latest-bigdata'
 
       - name: Deploy on CloudFormation
         run: |
-          make setup
+          make deploy
+
+      - name: Run tests
+        run: |
+          make test
 
-      - name: Check deployed resources
+      - name: Show LocalStack logs
+        if: always()
         run: |
-          sleep 20
-          awslocal cloudformation describe-stacks --stack-name covid-lake-stack
-
+          localstack logs
+
       - name: Send a Slack notification
         if: failure() || github.event_name != 'pull_request'
         uses: ravsamhq/notify-slack-action@v2

.gitignore

@@ -7,3 +7,4 @@ volume/
 .DS_Store
 .idea/
 .vscode/
+__pycache__

Makefile

@@ -3,32 +3,61 @@ export AWS_SECRET_ACCESS_KEY ?= test
 export AWS_DEFAULT_REGION=us-east-1
 SHELL := /bin/bash
 
-usage:    ## Show this help
-		@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//'
-
-install:  ## Install dependencies
-		@which localstack || pip install localstack
-		@which awslocal || pip install awscli-local
-		@which tflocal || pip install terraform-local
-
-setup:    ## Setup the architecture
-		awslocal s3 mb s3://covid19-lake; \
-		awslocal s3 cp CovidLakeStack.template.json s3://covid19-lake/cfn/CovidLakeStack.template.json; \
-		awslocal s3 sync ./covid19-lake-data/ s3://covid19-lake/; \
-		awslocal cloudformation create-stack --stack-name covid-lake-stack --template-url https://covid19-lake.s3.us-east-2.amazonaws.com/cfn/CovidLakeStack.template.json
-
-start:    ## Start LocalStack in detached mode
-		localstack start -d
-
-stop:     ## Stop the Running LocalStack container
-		@echo
-		localstack stop
-
-ready:    ## Make sure the LocalStack container is up
-		@echo Waiting on the LocalStack container...
-		@localstack wait -t 30 && echo LocalStack is ready to use! || (echo Gave up waiting on LocalStack, exiting. && exit 1)
+usage:			## Show this help in table format
+	@echo "| Target                 | Description                                                       |"
+	@echo "|------------------------|-------------------------------------------------------------------|"
+	@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/:.*##\s*/##/g' | awk -F'##' '{ printf "| %-22s | %-65s |\n", $$1, $$2 }'
+
+check:			## Check if all required prerequisites are installed
+	@command -v docker > /dev/null 2>&1 || { echo "Docker is not installed. Please install Docker and try again."; exit 1; }
+	@command -v aws > /dev/null 2>&1 || { echo "AWS CLI is not installed. Please install AWS CLI and try again."; exit 1; }
+	@command -v awslocal > /dev/null 2>&1 || { echo "AWS CLI Local is not installed. Please install awslocal and try again."; exit 1; }
+	@command -v localstack > /dev/null 2>&1 || { echo "LocalStack is not installed. Please install LocalStack and try again."; exit 1; }
+	@command -v python > /dev/null 2>&1 || { echo "Python is not installed. Please install Python and try again."; exit 1; }
+	@echo "All required prerequisites are available."
+
+deploy:		## Setup the architecture
+	@echo "Deploying the architecture..."
+	@echo "Create S3 bucket and upload the CloudFormation template..."
+	awslocal s3 mb s3://covid19-lake; \
+	awslocal s3 cp cloudformation-templates/CovidLakeStack.template.json s3://covid19-lake/cfn/CovidLakeStack.template.json; \
+	awslocal s3 sync ./covid19-lake-data/ s3://covid19-lake/; \
+	awslocal cloudformation create-stack --stack-name covid-lake-stack --template-url http://s3.localhost.localstack.cloud:4566/covid19-lake/cfn/CovidLakeStack.template.json
+	@counter=0; \
+	while [ $$counter -lt 30 ]; do \
+		status=$$(awslocal cloudformation describe-stacks --stack-name covid-lake-stack | grep StackStatus | cut -d'"' -f4); \
+		echo "Attempt $$counter: Stack status: $$status"; \
+		if [ "$$status" = "CREATE_COMPLETE" ]; then \
+			echo "Stack creation completed successfully!"; \
+			exit 0; \
+		fi; \
+		counter=$$((counter+1)); \
+		if [ $$counter -lt 30 ]; then sleep 2; fi; \
+	done; \
+	echo "Stack creation timed out after 30 attempts"; \
+	exit 1
+
+test:		## Run the tests
+	@echo "Installing dependencies..."
+	@pip install -r tests/requirements.txt
+	@echo "Running tests..."
+	pytest -s -v --disable-warnings tests/
+	@echo "All tests completed successfully."
+
+start:		## Start LocalStack
+	@echo "Starting LocalStack..."
+	@LOCALSTACK_AUTH_TOKEN=$(LOCALSTACK_AUTH_TOKEN) IMAGE_NAME=localstack/localstack-pro:latest-bigdata localstack start -d
+	@echo "LocalStack started successfully."
+
+stop:		## Stop the Running LocalStack container
+	@echo "Stopping LocalStack..."
+	localstack stop
+
+ready:		## Make sure the LocalStack container is up
+	@echo Waiting on the LocalStack container...
+	@localstack wait -t 30 && echo LocalStack is ready to use! || (echo Gave up waiting on LocalStack, exiting. && exit 1)
 
 logs:     ## Save the LocalStack logs in a separate file
-		@localstack logs > logs.txt
+	@localstack logs > logs.txt
 
 .PHONY: usage install run start stop ready logs

CovidLakeStack.template.json → cloudformation-templates/CovidLakeStack.template.json

@@ -7,7 +7,7 @@
           "Ref": "AWS::AccountId"
         },
         "DatabaseInput": {
-          "Name": "covid-19"
+          "Name": "covid_19"
         }
       },
       "Metadata": {

tests/requirements.txt

@@ -0,0 +1,3 @@
+pandas
+boto3
+pytest