Unstructured data tests docs

.github/workflows/build-docker-image.yml

@@ -0,0 +1,46 @@
+name: Build Docker image
+on:
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-docker-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout Elementary
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU for multi-platform support
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx for multi-platform support
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: false
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/bump-version.yml

@@ -51,7 +51,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Create release branch
         run: git checkout -b release/v${{ inputs.cli-version }}
       - name: Initial config
@@ -80,7 +80,7 @@ jobs:
     needs: bump-version
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: create pull request
         uses: repo-sync/pull-request@v2
         with:

.github/workflows/close_pylon_issue.yml

@@ -0,0 +1,41 @@
+name: Close Pylon Ticket on Issue or Pull Request Closure
+
+on:
+  issues:
+    types: [closed]
+  pull_request:
+    types: [closed]
+
+jobs:
+  close_pylon_ticket:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+
+      - name: Extract Pylon Ticket ID
+        id: extract_ticket_id
+        run: |
+          if [[ "${{ github.event_name }}" == 'issues' ]]; then
+            ISSUE_BODY=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+                         "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}" | jq -r '.body')
+            pylon_ticket_id=$(echo "$ISSUE_BODY" | grep -oP '(?<=<!-- pylon-ticket-id: )\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b(?= -->)')
+          elif [[ "${{ github.event_name }}" == 'pull_request' ]]; then
+            PR_BODY=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+                     "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" | jq -r '.body')
+            pylon_ticket_id=$(echo "$PR_BODY" | grep -oP '(?<=<!-- pylon-ticket-id: )\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b(?= -->)')
+          fi
+          echo "::set-output name=pylon_ticket_id::$pylon_ticket_id"
+
+      - name: Close Pylon Ticket
+        if: steps.extract_ticket_id.outputs.pylon_ticket_id != ''
+        run: |
+          pylon_ticket_id=${{ steps.extract_ticket_id.outputs.pylon_ticket_id }}
+          echo "Closing Pylon Ticket ID: $pylon_ticket_id"
+          curl --request PATCH \
+            --url "https://api.usepylon.com/issues/$pylon_ticket_id" \
+            --header "Authorization: ${{ secrets.PYLON_API_KEY }}" \
+            --header 'Content-Type: application/json' \
+            --data '{
+              "state": "closed"
+            }'

.github/workflows/create_pylon_issue.yml

@@ -13,30 +13,63 @@ jobs:
       - name: Check out the repository
         uses: actions/checkout@v2
 
+      - name: Install jq
+        run: sudo apt-get install -y jq
+
       - name: Create Pylon Issue for GitHub Issue
         if: github.event_name == 'issues'
         run: |
-          curl -X POST "https://api.usepylon.com/v1/issues" \
-          -H "Authorization: Bearer ${{ secrets.PYLON_API_KEY }}" \
-          -H "Content-Type: application/json" \
-          -d '{
+          response=$(curl --request POST \
+            --url https://api.usepylon.com/issues \
+            --header 'Authorization: ${{ secrets.PYLON_API_KEY }}' \
+            --header 'Content-Type: application/json' \
+            --data '{
+            "account_id": "${{ secrets.PYLON_ACCOUNT_ID }}",
+            "requester_id": "${{ secrets.PYLON_REQUESTER_ID }}",
+            "priority": "medium",
             "title": "${{ github.event.issue.title }}",
-            "description": "${{ github.event.issue.body }}",
-            "repository": "${{ github.repository }}",
-            "type": "GitHub Issue",
-            "url": "${{ github.event.issue.html_url }}"
-          }'
+            "body_html": "<html lang=en><head><title>GitHub Issue Details</title></head><body><h1>GitHub Issue Details</h1><p><strong>Repository:</strong> ${{ github.repository }}</p><p><strong>Type:</strong> Github Issue</p><p><strong>URL:</strong> <a href=${{ github.event.issue.html_url }}>${{ github.event.issue.html_url }}</a></p></body></html>"
+          }')
+          ticket_id=$(echo $response | jq -r '.data.id')
+          echo "ticket_id=$ticket_id" >> $GITHUB_ENV
+
+      - name: Add Pylon ticket ID to issue body
+        if: success()
+        run: |
+          issue_body=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}" | jq -r '.body')
+          new_body="$issue_body<!-- pylon-ticket-id: ${{ env.ticket_id }} -->"
+          json_body=$(jq -R -s --arg body "$new_body" '{"body": $body}' <<< "$new_body")
+          curl -X PATCH -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d "$json_body" \
+            "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}"
 
       - name: Create Pylon Issue for Pull Request
         if: github.event_name == 'pull_request'
         run: |
-          curl -X POST "https://api.usepylon.com/v1/issues" \
-          -H "Authorization: Bearer ${{ secrets.PYLON_API_KEY }}" \
-          -H "Content-Type: application/json" \
-          -d '{
+          response=$(curl --request POST \
+            --url https://api.usepylon.com/issues \
+            --header 'Authorization: ${{ secrets.PYLON_API_KEY }}' \
+            --header 'Content-Type: application/json' \
+            --data '{
+            "account_id": "${{ secrets.PYLON_ACCOUNT_ID }}",
+            "requester_id": "${{ secrets.PYLON_REQUESTER_ID }}",
+            "priority": "high",
             "title": "${{ github.event.pull_request.title }}",
-            "description": "${{ github.event.pull_request.body }}",
-            "repository": "${{ github.repository }}",
-            "type": "Pull Request",
-            "url": "${{ github.event.pull_request.html_url }}"
-          }'
+            "body_html": "<html lang=en><head><title>GitHub Pull Request Details</title></head><body><h1>GitHub Pull Request Details</h1><p><strong>Repository:</strong> ${{ github.repository }}</p><p><strong>Type:</strong> Pull Request</p><p><strong>URL:</strong> <a href=${{ github.event.pull_request.html_url }}>${{ github.event.pull_request.html_url }}</a></p></body></html>"
+          }')
+          ticket_id=$(echo $response | jq -r '.data.id')
+          echo "ticket_id=$ticket_id" >> $GITHUB_ENV
+
+      - name: Add Pylon ticket ID to PR body
+        if: success()
+        run: |
+          pr_body=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+                     "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" | jq -r '.body')
+          new_body="$pr_body<!-- pylon-ticket-id: ${{ env.ticket_id }} -->"
+          json_body=$(jq -R -s --arg body "$new_body" '{"body": $body}' <<< "$new_body")
+          curl -X PATCH -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d "$json_body" \
+            "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}"

.github/workflows/release.yml

@@ -10,10 +10,10 @@ env:
 
 jobs:
   publish-to-pypi:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v4
@@ -27,7 +27,7 @@ jobs:
         run: python -m build --sdist --wheel --outdir dist .
 
       - name: Upload build artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: build
           path: dist
@@ -46,7 +46,13 @@ jobs:
 
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU for multi-platform support
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx for multi-platform support
+        uses: docker/setup-buildx-action@v3
 
       - name: Log in to the container registry
         uses: docker/login-action@v2
@@ -66,13 +72,14 @@ jobs:
         with:
           context: .
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
   merge-to-docs:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: PR master to docs
         uses: repo-sync/pull-request@v2
         with:

.github/workflows/run-precommit.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/[email protected]

.github/workflows/test-github-action.yml

@@ -22,19 +22,19 @@ env:
 
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: elementary
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: elementary
           ref: ${{ inputs.elementary-ref }}
 
       - name: Checkout dbt package
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: elementary-data/dbt-data-reliability
           path: dbt-data-reliability
@@ -90,14 +90,14 @@ jobs:
             --file-path "report.html"
 
       - name: Upload report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: report.html
           path: edr_target/report.html
 
       - name: Upload log
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: edr.log
           path: edr_target/edr.log

.github/workflows/test-main-warehouse.yml

@@ -4,6 +4,7 @@ on:
     branches: ["master"]
     paths:
       - elementary/**
+      - .github/**
       - pyproject.toml
   workflow_dispatch:
 

.github/workflows/test-warehouse.yml

@@ -76,7 +76,7 @@ env:
 
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: elementary
@@ -86,13 +86,13 @@ jobs:
       cancel-in-progress: true
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: elementary
           ref: ${{ inputs.elementary-ref }}
 
       - name: Checkout dbt package
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: elementary-data/dbt-data-reliability
           path: dbt-data-reliability
@@ -101,12 +101,12 @@ jobs:
       - name: Start Postgres
         if: inputs.warehouse-type == 'postgres'
         working-directory: ${{ env.DBT_PKG_INTEG_TESTS_DIR }}
-        run: docker-compose up -d postgres
+        run: docker compose up -d postgres
 
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install Spark requirements
         if: inputs.warehouse-type == 'spark'
@@ -182,9 +182,9 @@ jobs:
           --project-profile-target "${{ inputs.warehouse-type }}"
 
       - name: Upload report artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}.html
+          name: report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}_dbt_${{ inputs.dbt-version }}.html
           path: elementary/edr_target/elementary_report.html
 
       - name: Write GCS keyfile
@@ -210,17 +210,17 @@ jobs:
           --aws-access-key-id "$AWS_ACCESS_KEY_ID"
           --aws-secret-access-key "$AWS_SECRET_ACCESS_KEY"
           --s3-bucket-name elementary-ci-artifacts
-          --google-service-account-path /tmp/gcs_keyfile.json
-          --gcs-bucket-name elementary_ci_artifacts
+          # --google-service-account-path /tmp/gcs_keyfile.json
+          # --gcs-bucket-name elementary_ci_artifacts
           --azure-connection-string "$AZURE_CONNECTION_STRING"
           --azure-container-name reports
           --update-bucket-website true
 
       - name: Upload edr log
         if: ${{ always() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: edr.log
+          name: edr_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}_dbt_${{ inputs.dbt-version }}.log
           path: elementary/edr_target/edr.log
 
       - name: Run Python package e2e tests