Merge branch 'main' into bot-team-strip-fix

Author: joaopamaral

.github/dependabot.yml

@@ -0,0 +1,67 @@
+version: 2
+
+# NOTE: This file controls Dependabot version-update PRs only.
+# It does NOT suppress Dependabot security alerts on the Security tab.
+# To auto-dismiss transitive (indirect) alerts, configure auto-triage rules at
+# Settings -> Code security -> Dependabot -> "Manage rules".
+
+updates:
+  - package-ecosystem: "pip"
+    directory: "/ingestion"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 5
+    labels:
+      - "dependencies"
+      - "python"
+    groups:
+      python-minor-patch:
+        update-types:
+          - "minor"
+          - "patch"
+    ignore:
+      # urllib3 is pinned <2.0 transitively via tableauserverclient==0.25.
+      # See ingestion/setup.py comment on the tableau pin.
+      - dependency-name: "urllib3"
+        versions: [">=2.0.0"]
+
+  - package-ecosystem: "maven"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 5
+    labels:
+      - "dependencies"
+      - "java"
+    groups:
+      maven-minor-patch:
+        update-types:
+          - "minor"
+          - "patch"
+
+  - package-ecosystem: "npm"
+    directory: "/openmetadata-ui/src/main/resources/ui"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 5
+    labels:
+      - "dependencies"
+      - "javascript"
+    groups:
+      npm-minor-patch:
+        update-types:
+          - "minor"
+          - "patch"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 3
+    labels:
+      - "dependencies"
+      - "github-actions"

.github/workflows/security-scan.yml

@@ -12,7 +12,7 @@
 name: security-scan
 on:
   schedule:
-    - cron:  '0 0 */2 * *'
+    - cron: "0 0 */2 * *"
   workflow_dispatch:
 
 jobs:
@@ -27,7 +27,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: 'openmetadata-ui/src/main/resources/ui/.nvmrc'
+          node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"
 
       - name: Enable yarn
         run: corepack enable
@@ -124,30 +124,6 @@ jobs:
                   print()
           EOF
 
-      - name: Slack on Failure
-        if: steps.retire-scan.outcome == 'failure'
-        uses: slackapi/slack-github-action@v1.23.0
-        with:
-          channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
-          payload: |
-            {
-              "text": "🚨 Vulnerability scan failed on branch `${{ github.ref_name }}`, please check it <https://github.com/open-metadata/OpenMetadata/actions/runs/${{ github.run_id }}|here>. 🚨"
-            }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-
-      - name: Slack on Success
-        if: steps.retire-scan.outcome == 'success'
-        uses: slackapi/slack-github-action@v1.23.0
-        with:
-          channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
-          payload: |
-            {
-              "text": "🟢 Vulnerability scan passed for OpenMetadata Repo on branch `${{ github.ref_name }}`, please check it <https://github.com/open-metadata/OpenMetadata/actions/runs/${{ github.run_id }}|here>."
-            }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-
       - name: Force failure on vulnerabilities found
         if: steps.retire-scan.outcome == 'failure'
         run: exit 1
@@ -163,25 +139,25 @@ jobs:
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@main
         with:
-            tool-cache: false
-            android: true
-            dotnet: true
-            haskell: true
-            large-packages: false
-            docker-images: true
-            swap-storage: true      
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: true
+          swap-storage: true
       - uses: actions/checkout@v4
 
       - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
       - name: Set up JDK 21
         uses: actions/setup-java@v4
         with:
-          java-version: '21'
-          distribution: 'temurin'
+          java-version: "21"
+          distribution: "temurin"
 
       - name: Install Ubuntu dependencies
         run: |
@@ -215,40 +191,111 @@ jobs:
         continue-on-error: true
         run: |
           source env/bin/activate
-          make snyk-report
-
-      - name: Slack on Failure
-        if: steps.security-report.outcome != 'success'
-        uses: slackapi/slack-github-action@v1.23.0
-        with:
-          channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
-          payload: |
-            {
-              "text": "🚨 Security report failed on branch `${{ github.ref_name }}`, please check it <https://github.com/open-metadata/OpenMetadata/actions/runs/${{ github.run_id }}|here>. 🚨"
-            }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-
-      - name: Slack on Success
-        if: steps.security-report.outcome == 'success'
-        uses: slackapi/slack-github-action@v1.23.0
-        with:
-          channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
-          payload: |
-            {
-              "text": "🟢 Security report generated for OpenMetadata Repo on branch `${{ github.ref_name }}`, please check it <https://github.com/open-metadata/OpenMetadata/actions/runs/${{ github.run_id }}|here>."
-            }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          rm -rf security-report
+          mkdir -p security-report
+          # Run snyk subtargets directly; skip `export-snyk-pdf-report` which deletes JSONs after PDF conversion.
+          make snyk-ingestion-report || true
+          make snyk-ingestion-base-slim-report || true
+          make snyk-airflow-apis-report || true
+          make snyk-server-report || true
+          make snyk-ui-report || true
+
+      - name: Publish Snyk Summary
+        id: snyk-summary
+        if: always() && steps.maven-build.outcome == 'success'
+        run: |
+          python3 scripts/snyk_summary.py security-report \
+            --counts-file security-report/_counts.json \
+            --slack-file security-report/_slack.txt \
+            >> $GITHUB_STEP_SUMMARY
+          # Expose counts as step output for downstream gating.
+          counts=$(cat security-report/_counts.json)
+          echo "counts=$counts" >> $GITHUB_OUTPUT
+          high=$(jq '.high + .critical' security-report/_counts.json)
+          echo "high_critical=$high" >> $GITHUB_OUTPUT
+
+      - name: Fail on high/critical Snyk findings
+        if: always() && steps.snyk-summary.outputs.high_critical != '' && steps.snyk-summary.outputs.high_critical != '0'
+        run: |
+          echo "::error::Snyk found ${{ steps.snyk-summary.outputs.high_critical }} high/critical vulnerabilities (see Job Summary)"
+          exit 1
 
-      - name: Upload Snyk Report HTML files
-        if: steps.security-report.outcome == 'success'
+      - name: Generate Snyk HTML/PDF
+        if: always() && steps.maven-build.outcome == 'success'
+        run: |
+          # Back up JSONs because html_to_pdf.py deletes them after PDF conversion.
+          mkdir -p /tmp/snyk-json-backup
+          cp security-report/*.json /tmp/snyk-json-backup/ 2>/dev/null || true
+          make export-snyk-pdf-report || true
+          # Restore JSONs alongside generated PDFs/HTMLs.
+          cp /tmp/snyk-json-backup/*.json security-report/ 2>/dev/null || true
+
+      - name: Upload Snyk Reports
+        if: always() && steps.maven-build.outcome == 'success'
         uses: actions/upload-artifact@v4
         with:
           name: security-report
           path: security-report
+          retention-days: 30
 
       - name: Force failure
         if: steps.maven-build.outcome != 'success' || steps.security-report.outcome != 'success'
         run: |
           exit 1
+
+  notify:
+    runs-on: ubuntu-latest
+    environment: security-scan
+    needs: [vulnerability-scan, security-scan]
+    if: always()
+    steps:
+      - name: Download Snyk artifact
+        if: needs.security-scan.result != 'skipped'
+        uses: actions/download-artifact@v4
+        with:
+          name: security-report
+          path: security-report
+        continue-on-error: true
+
+      - name: Build Slack payload
+        id: build
+        run: |
+          retire="${{ needs.vulnerability-scan.result }}"
+          snyk="${{ needs.security-scan.result }}"
+          status_icon() {
+            case "$1" in
+              success)   echo "✅" ;;
+              cancelled) echo "⚠️ (cancelled)" ;;
+              skipped)   echo "⚠️ (skipped)" ;;
+              *)         echo "❌" ;;
+            esac
+          }
+          retire_icon=$(status_icon "$retire")
+          snyk_icon=$(status_icon "$snyk")
+          if [ "$retire" = "success" ] && [ "$snyk" = "success" ]; then
+            icon="🟢"
+          elif [ "$retire" = "failure" ] || [ "$snyk" = "failure" ]; then
+            icon="🚨"
+          else
+            icon="⚠️"
+          fi
+          run_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          {
+            echo "$icon *Security scan* — *OpenMetadata Repo* on branch \`${{ github.ref_name }}\`"
+            echo "• Vulnerability scan (Retire.js): $retire_icon"
+            echo "• Security scan (Snyk): $snyk_icon"
+            echo "<$run_url|Open run details>"
+            if [ -f security-report/_slack.txt ]; then
+              echo
+              cat security-report/_slack.txt
+            fi
+          } > slack_body.txt
+          jq -Rs '{text: ., mrkdwn: true}' slack_body.txt > payload.json
+
+      - name: Send Slack Notification
+        uses: slackapi/slack-github-action@v1.27.1
+        with:
+          channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
+          payload-file-path: payload.json
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.gitignore

@@ -206,3 +206,5 @@ ingestion/.claude/agents
 .serena/
 
 test-results/
+
+docs/superpowers/*

bootstrap/sql/migrations/native/1.13.0/mysql/schemaChanges.sql

@@ -363,3 +363,38 @@ ALTER TABLE search_index_server_stats
 -- avoid INSERT failures on /mcp/authorize redirects.
 ALTER TABLE mcp_pending_auth_requests
     MODIFY COLUMN mcp_state TEXT;
+
+-- Allow multiple typed relations between the same pair of glossary terms.
+-- The previous PRIMARY KEY (fromId, toId, relation) caused INSERT ... ON DUPLICATE
+-- KEY UPDATE to overwrite the json discriminator when a second relationType
+-- ("synonym" + "seeAlso", etc.) was added between the same two terms, silently
+-- dropping the first relationship. Adding relationType to the PK lets the same
+-- (fromId, toId, RELATED_TO) pair carry one row per relation type.
+-- `IF NOT EXISTS` on `ADD COLUMN` only landed in MySQL 8.0.29; supported 8.0.x
+-- deployments may be older, so use plain ADD COLUMN. SERVER_CHANGE_LOG gates
+-- re-execution at the framework level — same reasoning as the PK swap below.
+ALTER TABLE entity_relationship
+    ADD COLUMN `relationType` varchar(64) NOT NULL DEFAULT '' AFTER `relation`;
+
+-- Backfill relationType for every glossary-term ↔ glossary-term RELATED_TO row.
+-- Pre-1.13 data has json = NULL (no discriminator existed yet) — those rows MUST
+-- collapse onto 'relatedTo' so that a subsequent insert of the same logical
+-- relation matches the existing row instead of creating a duplicate under a
+-- different PK. relation=15 is the ordinal of Relationship.RELATED_TO (see
+-- openmetadata-spec entityRelationship.json). 'relatedTo' is the default
+-- relation type that the application code uses when none is specified.
+UPDATE entity_relationship
+SET relationType =
+    COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(json, '$.relationType')), ''), 'relatedTo')
+WHERE fromEntity = 'glossaryTerm'
+  AND toEntity = 'glossaryTerm'
+  AND relation = 15;
+
+-- Swap the PK to include relationType. The native migration framework tracks
+-- completion in SERVER_CHANGE_LOG so this runs once per upgrade; we intentionally
+-- avoid information_schema gating because least-privilege migration users may
+-- not have SELECT on it. A manual replay of this step on an already-migrated
+-- table will rebuild the PK with the same columns — wasteful but not broken.
+ALTER TABLE entity_relationship
+    DROP PRIMARY KEY,
+    ADD PRIMARY KEY (`fromId`, `toId`, `relation`, `relationType`);

bootstrap/sql/migrations/native/1.13.0/postgres/schemaChanges.sql

@@ -511,3 +511,34 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_worksheet_entity_fqnhash_pattern
 -- avoid INSERT failures on /mcp/authorize redirects.
 ALTER TABLE mcp_pending_auth_requests
     ALTER COLUMN mcp_state TYPE TEXT;
+
+-- Allow multiple typed relations between the same pair of glossary terms.
+-- The previous PRIMARY KEY (fromId, toId, relation) caused INSERT ... ON CONFLICT
+-- DO UPDATE to overwrite the json discriminator when a second relationType
+-- ("synonym" + "seeAlso", etc.) was added between the same two terms, silently
+-- dropping the first relationship. Adding relationType to the PK lets the same
+-- (fromId, toId, RELATED_TO) pair carry one row per relation type.
+ALTER TABLE entity_relationship
+    ADD COLUMN IF NOT EXISTS relationType character varying(64) DEFAULT ''::character varying NOT NULL;
+
+-- Backfill relationType for every glossary-term ↔ glossary-term RELATED_TO row.
+-- Pre-1.13 data has json = NULL (no discriminator existed yet) — those rows MUST
+-- collapse onto 'relatedTo' so that a subsequent insert of the same logical
+-- relation matches the existing row instead of creating a duplicate under a
+-- different PK. relation=15 is the ordinal of Relationship.RELATED_TO (see
+-- openmetadata-spec entityRelationship.json). 'relatedTo' is the default
+-- relation type that the application code uses when none is specified.
+UPDATE entity_relationship
+SET relationType = COALESCE(NULLIF(json->>'relationType', ''), 'relatedTo')
+WHERE fromEntity = 'glossaryTerm'
+  AND toEntity = 'glossaryTerm'
+  AND relation = 15;
+
+-- Swap the PK to include relationType. The native migration framework tracks
+-- completion in SERVER_CHANGE_LOG so this runs once per upgrade; we intentionally
+-- avoid information_schema gating because least-privilege migration users may
+-- not have SELECT on it. DROP CONSTRAINT IF EXISTS keeps the statement safe to
+-- replay against a table that's already been migrated.
+ALTER TABLE entity_relationship DROP CONSTRAINT IF EXISTS entity_relationship_pkey;
+ALTER TABLE entity_relationship
+    ADD CONSTRAINT entity_relationship_pkey PRIMARY KEY (fromId, toId, relation, relationType);