feat(ci): Implement mechanism to release new versions of ETL (#316)

Author: iambriccardo

.github/workflows/audit.yml

@@ -16,11 +16,11 @@ jobs:
   security_audit:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
+      - name: Checkout
         uses: actions/checkout@v4
 
       - name: Install cargo-deny
         uses: taiki-e/install-action@cargo-deny
 
-      - name: Scan for vulnerabilities
+      - name: Scan for Vulnerabilities
         run: cargo deny check advisories

.github/workflows/ci.yml

@@ -15,35 +15,35 @@ permissions:
 
 jobs:
   check:
-    name: Check
+    name: Lint
     runs-on: ubuntu-latest
     strategy:
       matrix:
         check: [fmt, clippy]
     steps:
-      - name: Checkout repository
+      - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Install Rust toolchain
+      - name: Set up Rust
         uses: dtolnay/rust-toolchain@stable
         with:
           components: ${{ matrix.check == 'fmt' && 'rustfmt' || 'clippy' }}
 
-      - name: Setup Rust cache
+      - name: Cache Cargo
         uses: Swatinem/rust-cache@v2
         with:
           key: ${{ matrix.check }}
 
-      - name: Run cargo fmt
+      - name: Run Cargo Fmt
         if: matrix.check == 'fmt'
         run: cargo fmt --check
 
-      - name: Run cargo clippy
+      - name: Run Cargo Clippy
         if: matrix.check == 'clippy'
         run: cargo clippy --all-targets --all-features --no-deps -- -D warnings
 
   test-partial:
-    name: Test Partial
+    name: Tests (Partial)
     runs-on: ubuntu-latest
     # Run on forks.
     if: github.event.pull_request.head.repo.fork == true
@@ -53,18 +53,18 @@ jobs:
       matrix:
         postgres_version: [17, 16, 15, 14]
     steps:
-      - name: Checkout repository
+      - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Install Rust toolchain
+      - name: Set up Rust
         uses: dtolnay/rust-toolchain@stable
 
-      - name: Setup Rust cache
+      - name: Cache Cargo
         uses: Swatinem/rust-cache@v2
         with:
           key: test-partial
 
-      - name: Start Docker Compose Environment (Postgres ${{ matrix.postgres_version }})
+      - name: Start Docker Compose (Postgres ${{ matrix.postgres_version }})
         run: |
           POSTGRES_VERSION=${{ matrix.postgres_version }} docker compose -f ./scripts/docker-compose.yaml up -d
 
@@ -75,7 +75,7 @@ jobs:
             --no-default-features \
             --locked
 
-      - name: Migrate database
+      - name: Migrate Database
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -86,7 +86,7 @@ jobs:
           sudo apt-get install libpq-dev -y
           ./etl-api/scripts/run_migrations.sh
 
-      - name: Run tests
+      - name: Run Tests
         run: |
           cargo test \
             --workspace \
@@ -101,7 +101,7 @@ jobs:
             --features iceberg
 
   test-full:
-    name: Test Full
+    name: Tests (Full)
     runs-on: ubuntu-latest
     # Run on non-forks.
     if: github.event.pull_request.head.repo.fork == false
@@ -112,18 +112,18 @@ jobs:
       matrix:
         postgres_version: [17, 16, 15, 14]
     steps:
-      - name: Checkout repository
+      - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Install Rust toolchain
+      - name: Set up Rust
         uses: dtolnay/rust-toolchain@stable
 
-      - name: Setup Rust cache
+      - name: Cache Cargo
         uses: Swatinem/rust-cache@v2
         with:
           key: test-full
 
-      - name: Start Docker Compose Environment (Postgres ${{ matrix.postgres_version }})
+      - name: Start Docker Compose (Postgres ${{ matrix.postgres_version }})
         run: |
           POSTGRES_VERSION=${{ matrix.postgres_version }} docker compose -f ./scripts/docker-compose.yaml up -d
 
@@ -134,7 +134,7 @@ jobs:
             --no-default-features \
             --locked
 
-      - name: Migrate database
+      - name: Migrate Database
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -148,21 +148,21 @@ jobs:
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
 
-      - name: Set up BigQuery environment variables and credentials
+      - name: Set up BigQuery Credentials
         run: |
           printf '%s' '${{ secrets.TESTS_BIGQUERY_SA_KEY_JSON }}' > /tmp/bigquery-sa-key.json
           echo "TESTS_BIGQUERY_PROJECT_ID=${{ secrets.TESTS_BIGQUERY_PROJECT_ID }}" >> $GITHUB_ENV
           echo "TESTS_BIGQUERY_SA_KEY_PATH=/tmp/bigquery-sa-key.json" >> $GITHUB_ENV
 
-      - name: Generate code coverage
+      - name: Generate Code Coverage
         id: coverage
         run: |
           cargo llvm-cov test \
             --workspace --no-fail-fast \
             --all-features \
             --lcov --output-path lcov.info
 
-      - name: Upload coverage to Coveralls
+      - name: Upload Coverage to Coveralls
         uses: coverallsapp/github-action@v2
         with:
           fail-on-error: false

.github/workflows/docker-build.yml

@@ -1,51 +1,80 @@
 name: Docker Build
 
 on:
-  push:
-    branches: [main]
-  workflow_dispatch:
-
-env:
-  CARGO_TERM_COLOR: always
+  workflow_call:
+    inputs:
+      image:
+        description: "Docker image (e.g. docker.io/<org>/<repo>)"
+        type: string
+        required: true
+      context:
+        description: "Docker build context"
+        type: string
+        default: "."
+      push:
+        description: "Push image to registry"
+        type: boolean
+        default: false
+      tag_with_version:
+        description: "Also tag the image with version"
+        type: boolean
+        default: false
+      version:
+        description: "Version to tag when tag_with_version=true"
+        type: string
+        required: false
 
 permissions:
   contents: read
 
 jobs:
   docker:
-    name: Build and Push Docker Images
+    name: Build Docker Image
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        image: [etl-api, etl-replicator]
     steps:
-      - name: Checkout repository
+      - name: Checkout
         uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Login to Docker Hub
+      - name: Log in to Docker Hub
+        if: inputs.push == true
         uses: docker/login-action@v3
         with:
           username: ${{ vars.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Build and push ${{ matrix.image }} image
-        uses: docker/build-push-action@v6
-        id: build
+      - name: Docker Metadata
+        id: meta
+        uses: docker/metadata-action@v5
         with:
-          file: ./${{ matrix.image }}/Dockerfile
-          push: true
-          cache-from: type=gha,scope=${{ matrix.image }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.image }}
+          images: ${{ inputs.image }}
           tags: |
-            ${{ vars.DOCKERHUB_USERNAME }}/${{ matrix.image }}:latest
-            ${{ vars.DOCKERHUB_USERNAME }}/${{ matrix.image }}:${{ github.ref_name }}
-            ${{ vars.DOCKERHUB_USERNAME }}/${{ matrix.image }}:${{ github.sha }}
-
-      - name: Output image information
-        run: |          
-          echo "✅ Successfully built and pushed ${{ matrix.image }} image"
-          echo "🏷️ Tags: latest, ${{ github.ref_name }}, ${{ github.sha }}"
-          echo "🔗 View at: https://hub.docker.com/r/${{ vars.DOCKERHUB_USERNAME }}/${{ matrix.image }}"
+            type=raw,value=latest
+            type=sha,format=short,prefix=
+            ${{ inputs.tag_with_version && format('type=raw,value=v{0}', inputs.version) || '' }}
+
+      - name: Build and Push Image
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ inputs.context }}
+          push: ${{ inputs.push }}
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Output Image Information
+        run: |
+          IMAGE="${{ inputs.image }}"
+          NAME="${IMAGE##*/}"
+          SHORT_SHA="${GITHUB_SHA::7}"
+          TAGS="latest, ${SHORT_SHA}"
+          if [ "${{ inputs.tag_with_version }}" = "true" ] && [ -n "${{ inputs.version }}" ]; then
+            TAGS="${TAGS}, v${{ inputs.version }}"
+          fi
+          HUB_PATH=$(echo "${IMAGE}" | sed -E 's@^docker\.io/@@')
+          echo "✅ Successfully built and pushed ${NAME} image"
+          echo "🏷️ Tags: ${TAGS}"
+          echo "🔗 View at: https://hub.docker.com/r/${HUB_PATH}"

.github/workflows/docker-ci.yml

@@ -0,0 +1,23 @@
+name: Docker CI
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build and Push Images
+    strategy:
+      matrix:
+        image: [etl-api, etl-replicator]
+    uses: ./.github/workflows/docker-build.yml
+    secrets: inherit
+    with:
+      image: docker.io/${{ vars.DOCKERHUB_USERNAME }}/${{ matrix.image }}
+      context: ./${{ matrix.image }}
+      push: true
+      tag_with_version: false

.github/workflows/docs.yml

@@ -1,15 +1,15 @@
 name: Docs
 
+concurrency:
+  group: docs
+  cancel-in-progress: false
+
 on:
   push:
     branches: [main]
   pull_request:
     branches: [main]
 
-concurrency:
-  group: docs
-  cancel-in-progress: false
-
 permissions:
   contents: write
 
@@ -18,38 +18,50 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'push'
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout
+        uses: actions/checkout@v4
       - name: Configure Git Credentials
         run: |
           git config user.name github-actions[bot]
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
-      - uses: actions/setup-python@v5
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
           python-version: 3.x
-      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
-      - uses: actions/cache@v4
+      - name: Compute Cache Key
+        run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - name: Restore Docs Cache
+        uses: actions/cache@v4
         with:
           key: mkdocs-material-${{ env.cache_id }}
           path: ~/.cache
           restore-keys: |
             mkdocs-material-
-      - run: pip install -r docs/requirements_docs.txt
-      - run: mkdocs gh-deploy --force
+      - name: Install Docs Requirements
+        run: pip install -r docs/requirements_docs.txt
+      - name: Deploy Docs
+        run: mkdocs gh-deploy --force
 
   build:
     runs-on: ubuntu-latest
     if: github.event_name == 'pull_request'
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
           python-version: 3.x
-      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
-      - uses: actions/cache@v4
+      - name: Compute Cache Key
+        run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - name: Restore Docs Cache
+        uses: actions/cache@v4
         with:
           key: mkdocs-material-${{ env.cache_id }}
           path: ~/.cache
           restore-keys: |
             mkdocs-material-
-      - run: pip install -r docs/requirements_docs.txt
-      - run: mkdocs build --strict
+      - name: Install Docs Requirements
+        run: pip install -r docs/requirements_docs.txt
+      - name: Build Docs
+        run: mkdocs build --strict