Merge pull request #1821 from elementary-data/release/v0.16.2

.github/workflows/build-docker-image.yml

@@ -0,0 +1,46 @@
+name: Build Docker image
+on:
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-docker-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout Elementary
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU for multi-platform support
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx for multi-platform support
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: false
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/bump-version.yml

@@ -51,7 +51,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Create release branch
         run: git checkout -b release/v${{ inputs.cli-version }}
       - name: Initial config
@@ -80,7 +80,7 @@ jobs:
     needs: bump-version
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: create pull request
         uses: repo-sync/pull-request@v2
         with:

.github/workflows/close_pylon_issue.yml

@@ -0,0 +1,38 @@
+name: Close Pylon Ticket on Issue or Pull Request Closure
+
+on:
+  issues:
+    types: [closed]
+  pull_request_target:
+    types: [closed]
+
+jobs:
+  close_pylon_ticket:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Extract Pylon Ticket ID
+        id: extract_ticket_id
+        run: |
+          if [[ "${{ github.event_name }}" == 'issues' ]]; then
+            ISSUE_BODY=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+                         "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}" | jq -r '.body')
+            pylon_ticket_id=$(echo "$ISSUE_BODY" | grep -oP '(?<=<!-- pylon-ticket-id: )\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b(?= -->)')
+          elif [[ "${{ github.event_name }}" =~ ^pull_request ]]; then
+            PR_BODY=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+                     "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" | jq -r '.body')
+            pylon_ticket_id=$(echo "$PR_BODY" | grep -oP '(?<=<!-- pylon-ticket-id: )\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b(?= -->)')
+          fi
+          echo "pylon_ticket_id=$pylon_ticket_id" >> "$GITHUB_OUTPUT"
+
+      - name: Close Pylon Ticket
+        if: steps.extract_ticket_id.outputs.pylon_ticket_id != ''
+        run: |
+          pylon_ticket_id=${{ steps.extract_ticket_id.outputs.pylon_ticket_id }}
+          echo "Closing Pylon Ticket ID: $pylon_ticket_id"
+          curl --request PATCH \
+            --url "https://api.usepylon.com/issues/$pylon_ticket_id" \
+            --header "Authorization: ${{ secrets.PYLON_API_KEY }}" \
+            --header 'Content-Type: application/json' \
+            --data '{
+              "state": "closed"
+            }'

.github/workflows/create_pylon_issue.yml

@@ -3,40 +3,70 @@ name: Create Pylon Issue
 on:
   issues:
     types: [opened]
-  pull_request:
+  pull_request_target:
     types: [opened]
 
 jobs:
   create_pylon_issue:
     runs-on: ubuntu-latest
     steps:
-      - name: Check out the repository
-        uses: actions/checkout@v2
+      - name: Install jq
+        run: sudo apt-get install -y jq
 
       - name: Create Pylon Issue for GitHub Issue
         if: github.event_name == 'issues'
         run: |
-          curl -X POST "https://api.usepylon.com/v1/issues" \
-          -H "Authorization: Bearer ${{ secrets.PYLON_API_KEY }}" \
-          -H "Content-Type: application/json" \
-          -d '{
+          response=$(curl --request POST \
+            --url https://api.usepylon.com/issues \
+            --header 'Authorization: ${{ secrets.PYLON_API_KEY }}' \
+            --header 'Content-Type: application/json' \
+            --data '{
+            "account_id": "${{ secrets.PYLON_ACCOUNT_ID }}",
+            "requester_id": "${{ secrets.PYLON_REQUESTER_ID }}",
+            "priority": "medium",
             "title": "${{ github.event.issue.title }}",
-            "description": "${{ github.event.issue.body }}",
-            "repository": "${{ github.repository }}",
-            "type": "GitHub Issue",
-            "url": "${{ github.event.issue.html_url }}"
-          }'
+            "body_html": "<html lang=en><head><title>GitHub Issue Details</title></head><body><h1>GitHub Issue Details</h1><p><strong>Repository:</strong> ${{ github.repository }}</p><p><strong>Type:</strong> Github Issue</p><p><strong>URL:</strong> <a href=${{ github.event.issue.html_url }}>${{ github.event.issue.html_url }}</a></p></body></html>"
+          }')
+          ticket_id=$(echo $response | jq -r '.data.id')
+          echo "ticket_id=$ticket_id" >> $GITHUB_ENV
+
+      - name: Add Pylon ticket ID to issue body
+        if: success()
+        run: |
+          issue_body=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}" | jq -r '.body')
+          new_body="$issue_body<!-- pylon-ticket-id: ${{ env.ticket_id }} -->"
+          json_body=$(jq -R -s --arg body "$new_body" '{"body": $body}' <<< "$new_body")
+          curl -X PATCH -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d "$json_body" \
+            "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}"
 
       - name: Create Pylon Issue for Pull Request
-        if: github.event_name == 'pull_request'
+        if: contains(github.event_name, 'pull_request')
         run: |
-          curl -X POST "https://api.usepylon.com/v1/issues" \
-          -H "Authorization: Bearer ${{ secrets.PYLON_API_KEY }}" \
-          -H "Content-Type: application/json" \
-          -d '{
+          response=$(curl --request POST \
+            --url https://api.usepylon.com/issues \
+            --header 'Authorization: ${{ secrets.PYLON_API_KEY }}' \
+            --header 'Content-Type: application/json' \
+            --data '{
+            "account_id": "${{ secrets.PYLON_ACCOUNT_ID }}",
+            "requester_id": "${{ secrets.PYLON_REQUESTER_ID }}",
+            "priority": "high",
             "title": "${{ github.event.pull_request.title }}",
-            "description": "${{ github.event.pull_request.body }}",
-            "repository": "${{ github.repository }}",
-            "type": "Pull Request",
-            "url": "${{ github.event.pull_request.html_url }}"
-          }'
+            "body_html": "<html lang=en><head><title>GitHub Pull Request Details</title></head><body><h1>GitHub Pull Request Details</h1><p><strong>Repository:</strong> ${{ github.repository }}</p><p><strong>Type:</strong> Pull Request</p><p><strong>URL:</strong> <a href=${{ github.event.pull_request.html_url }}>${{ github.event.pull_request.html_url }}</a></p></body></html>"
+          }')
+          ticket_id=$(echo $response | jq -r '.data.id')
+          echo "ticket_id=$ticket_id" >> $GITHUB_ENV
+
+      - name: Add Pylon ticket ID to PR body
+        if: success()
+        run: |
+          pr_body=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+                     "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" | jq -r '.body')
+          new_body="$pr_body<!-- pylon-ticket-id: ${{ env.ticket_id }} -->"
+          json_body=$(jq -R -s --arg body "$new_body" '{"body": $body}' <<< "$new_body")
+          curl -X PATCH -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d "$json_body" \
+            "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}"

.github/workflows/release.yml

@@ -10,15 +10,15 @@ env:
 
 jobs:
   publish-to-pypi:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install 'build' package
         run: pip install build
@@ -27,7 +27,7 @@ jobs:
         run: python -m build --sdist --wheel --outdir dist .
 
       - name: Upload build artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: build
           path: dist
@@ -46,7 +46,13 @@ jobs:
 
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU for multi-platform support
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx for multi-platform support
+        uses: docker/setup-buildx-action@v3
 
       - name: Log in to the container registry
         uses: docker/login-action@v2
@@ -66,13 +72,14 @@ jobs:
         with:
           context: .
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
   merge-to-docs:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: PR master to docs
         uses: repo-sync/pull-request@v2
         with:

.github/workflows/remind-docs-and-tests.yml

@@ -1,6 +1,6 @@
 name: Remind docs and tests
 on:
-  pull_request:
+  pull_request_target:
     branches: ["master"]
 jobs:
   run:

.github/workflows/run-precommit.yml

@@ -8,12 +8,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/[email protected]
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       # mainly needed so mypy will have the dependencies it needs
       - name: Install elementary

.github/workflows/test-all-warehouses.yml

@@ -25,16 +25,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dbt-version: ${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) || fromJSON('["1.3.0", null]') }}
+        dbt-version: ${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) || fromJSON('["1.8.0", null]') }}
         warehouse-type:
-          [
-            postgres,
-            snowflake,
-            bigquery,
-            redshift,
-            databricks,
-            databricks_catalog,
-          ]
+          [postgres, snowflake, bigquery, redshift, databricks_catalog, athena]
     uses: ./.github/workflows/test-warehouse.yml
     with:
       warehouse-type: ${{ matrix.warehouse-type }}

.github/workflows/test-github-action.yml

@@ -22,19 +22,19 @@ env:
 
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: elementary
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: elementary
           ref: ${{ inputs.elementary-ref }}
 
       - name: Checkout dbt package
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: elementary-data/dbt-data-reliability
           path: dbt-data-reliability
@@ -51,7 +51,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install dbt
         run: pip install --pre
@@ -90,14 +90,14 @@ jobs:
             --file-path "report.html"
 
       - name: Upload report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: report.html
           path: edr_target/report.html
 
       - name: Upload log
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: edr.log
           path: edr_target/edr.log

.github/workflows/test-main-warehouse.yml

@@ -1,9 +1,10 @@
 name: Test main warehouse platform
 on:
-  pull_request:
+  pull_request_target:
     branches: ["master"]
     paths:
       - elementary/**
+      - .github/**
       - pyproject.toml
   workflow_dispatch:
 
@@ -12,4 +13,5 @@ jobs:
     uses: ./.github/workflows/test-warehouse.yml
     with:
       warehouse-type: postgres
+      elementary-ref: ${{ (github.event_name == 'pull_request_target' && github.event.pull_request.head.sha) || '' }}
     secrets: inherit

.github/workflows/test-release.yml

@@ -47,14 +47,7 @@ jobs:
       fail-fast: false
       matrix:
         warehouse-type:
-          [
-            postgres,
-            snowflake,
-            bigquery,
-            redshift,
-            databricks,
-            databricks_catalog,
-          ]
+          [postgres, snowflake, bigquery, redshift, databricks_catalog, athena]
     needs: get-latest-release-tags
     uses: ./.github/workflows/test-warehouse.yml
     with: