create a cloud pods workflow (#10)

* create a cloud pods workflow

* fix

* fix

* last fix

* run the workflow only on pushes

* add a note about cloud pods in README

Author: HarshCasper

.github/workflows/ci.yml

@@ -80,4 +80,4 @@ jobs:
       - name: Run tests
         run: |
           pip3 install pytest pandas numpy
-          pytest tests/
+          make test

.github/workflows/cloud-pods.yml

@@ -0,0 +1,139 @@
+name: Create and Test Cloud Pods
+
+on:
+  schedule:
+    # At 00:00 on Saturday.
+    - cron: "0 0 * * 6"
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  actions: read
+
+jobs:
+  create-cloud-pod:
+    name: Create Cloud Pods
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12' 
+
+      - name: Install Snow CLI
+        run: |
+          pip install snowflake-cli==3.7.2
+          snow --version
+          snow connection add \
+            --connection-name localstack \
+            --user test \
+            --password test \
+            --account test \
+            --role test \
+            --warehouse test \
+            --database test \
+            --schema test \
+            --port 4566 \
+            --host snowflake.localhost.localstack.cloud \
+            --no-interactive
+          snow connection list
+
+      - name: Install dependencies
+        run: |
+          make install
+
+      - name: Start LocalStack
+        uses: LocalStack/setup-localstack@main
+        with:
+          image-tag: 'latest'
+          use-pro: 'true'
+          configuration: LS_LOG=trace
+          install-awslocal: 'true'
+        env:
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
+          IMAGE_NAME: localstack/snowflake:latest
+
+      - name: Create Snowflake resources
+        run: |
+          make seed
+
+      - name: Create AWS resources
+        run: |
+          make aws
+          make upload
+
+      - name: Run dbt models
+        run: |
+          make dbt
+
+      - name: Deploy Native App
+        run: |
+          make app
+
+      - name: Create Cloud Pod
+        env:
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
+        run: |
+          message="Cloud Pod created: smart-factory-app on $(date) with workflow run id: ${{ github.run_id }}"
+          localstack pod save smart-factory-app --message "$message"
+
+      - name: Show LocalStack Logs
+        if: always()
+        run: |
+          localstack logs
+
+  test-cloud-pod:
+    name: Test Cloud Pod
+    needs: create-cloud-pod
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Start LocalStack
+        uses: LocalStack/setup-localstack@main
+        with:
+          use-pro: 'true'
+          install-awslocal: 'true'
+        env:
+          DEBUG: 1
+          IMAGE_NAME: localstack/snowflake:latest
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
+
+      - name: Load Cloud Pod
+        env:
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
+        run: |
+          localstack pod load smart-factory-app
+
+      - name: Install dependencies
+        run: |
+          make install
+
+      - name: Run Tests
+        run: |
+          pip3 install pytest pandas numpy snowflake-connector-python
+          pytest tests/
+
+      - name: Show LocalStack Logs
+        if: always()
+        run: |
+          localstack logs
+
+      - name: Generate a Diagnostic Report
+        if: failure()
+        run: |
+          curl -s localhost:4566/_localstack/diagnose | gzip -cf > diagnose.json.gz
+
+      - name: Upload the Diagnostic Report
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diagnose.json.gz
+          path: ./diagnose.json.gz

Makefile

@@ -1,6 +1,7 @@
 export AWS_ACCESS_KEY_ID ?= test
 export AWS_SECRET_ACCESS_KEY ?= test
 export AWS_DEFAULT_REGION=us-east-1
+export DBT_WARN_ERROR_OPTIONS={"silence": ["UnusedResourceConfigPath", "NoNodesForSelectionCriteria"]}
 SHELL := /bin/bash
 
 # Flag to control which batch file to upload (default: false = use batch 1, true = use latest batch)
@@ -59,7 +60,7 @@ pipeline:		## Setup Dagster pipeline
 
 dbt:			## Run dbt models
 	@echo "Running dbt..."
-	bash -c "source env/bin/activate && dbt build --profile localstack"
+	bash -c "source env/bin/activate && dbt run --profile localstack"
 	@echo "Dbt run successfully."
 
 app:			## Run Snowflake Native App
@@ -70,6 +71,11 @@ app:			## Run Snowflake Native App
 	@printf " └─ URL: \033[4;94mhttps://snowflake.localhost.localstack.cloud:4566/apps/test/test/FACTORY_APP_$$(whoami | tr '[:lower:]' '[:upper:]')/\033[0m\n"
 	@echo ""
 
+streamlit:		## Run Streamlit app
+	@echo "Running Streamlit app..."
+	cd app/src/module-ui/src && streamlit run ui.py
+	@echo "Streamlit app run successfully."
+
 deploy:			## Deploy the entire stack
 	@echo "Deploying the entire stack..."
 	make seed
@@ -95,6 +101,9 @@ alerts:			## Setup alerts
 
 test:			## Run tests
 	@echo "Running tests..."
+	@echo "Running dbt tests..."
+	bash -c "source env/bin/activate && dbt test --profile localstack"
+	@echo "Running pytest integration tests..."
 	bash -c "source env/bin/activate && pytest tests/"
 	@echo "Tests run successfully."
 

README.md

@@ -201,6 +201,32 @@ This showcases:
 - Point-in-time recovery scenarios
 - Disaster recovery best practices
 
+### Test Data Management with Cloud Pods
+
+Cloud Pods enable persistent state snapshots of your LocalStack instance that can be easily stored, versioned, shared, and restored across development teams and CI/CD environments.
+
+To save a Cloud Pod, run the following command:
+
+```bash
+localstack pod save smart-factory-app
+```
+
+To load a Cloud Pod, run the following command:
+
+```bash
+# Restore the saved state in a fresh LocalStack instance
+localstack pod load smart-factory-app
+```
+
+The key advantages are:
+
+- Pre-seed CI environments with complex factory data, Snowflake schemas, and S3 assets in seconds
+- Share identical test scenarios across development teams for consistent debugging
+- Guarantee the same test data state across different pipeline runs and environments  
+- Capture problematic states for collaborative debugging and issue reproduction
+
+For automated Cloud Pods management in CI/CD pipelines, check out the sample workflow in [`.github/workflows/cloud-pods.yml`](.github/workflows/cloud-pods.yml).
+
 ## License
 
 This project is licensed under the [Apache License 2.0](LICENSE).