Merge branch 'main' into FIX-Aggregation-incident-reindex

Author: TeddyCr

.github/workflows/integration-tests-mysql-elasticsearch.yml

@@ -50,11 +50,11 @@ jobs:
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@main
         with:
-          tool-cache: false
+          tool-cache: true
           android: true
           dotnet: true
           haskell: true
-          large-packages: false
+          large-packages: true
           docker-images: false
           swap-storage: true
 
@@ -110,6 +110,13 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: mvn -DskipTests clean install -pl :openmetadata-integration-tests -am
 
+      - name: Free build artifacts
+        run: |
+          rm -rf openmetadata-service/target/lib openmetadata-service/target/classes
+          rm -rf openmetadata-spec/target openmetadata-sdk/target common/target
+          rm -rf openmetadata-shaded-deps/*/target
+          df -h /
+
       - name: Run Integration Tests (MySQL + Elasticsearch)
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/integration-tests-postgres-opensearch.yml

@@ -50,11 +50,11 @@ jobs:
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@main
         with:
-          tool-cache: false
+          tool-cache: true
           android: true
           dotnet: true
           haskell: true
-          large-packages: false
+          large-packages: true
           docker-images: false
           swap-storage: true
 
@@ -110,6 +110,13 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: mvn -DskipTests clean install -pl :openmetadata-integration-tests -am
 
+      - name: Free build artifacts
+        run: |
+          rm -rf openmetadata-service/target/lib openmetadata-service/target/classes
+          rm -rf openmetadata-spec/target openmetadata-sdk/target common/target
+          rm -rf openmetadata-shaded-deps/*/target
+          df -h /
+
       - name: Run Integration Tests (PostgreSQL + OpenSearch)
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/maven-postgres-tests-build.yml

@@ -10,38 +10,11 @@
 #  limitations under the License.
 
 name: Maven Postgres Tests CI
+# Disabled: Integration tests (openmetadata-integration-tests) now cover the same ground
+# with full Testcontainers-based PostgreSQL testing.
 
 on:
-  push:
-    branches:
-      - main
-    paths:
-      - "openmetadata-service/**"
-      - "openmetadata-ui/**"
-      - "!openmetadata-ui/src/main/resources/ui/playwright/doc-generator/**"
-      - "!openmetadata-ui/src/main/resources/ui/playwright/docs/**"
-      - "openmetadata-spec/src/main/resources/json/schema/**"
-      - "openmetadata-dist/**"
-      - "openmetadata-clients/**"
-      - "openmetadata-sdk/**"
-      - "common/**"
-      - "pom.xml"
-      - "yarn.lock"
-      - "Makefile"
-      - "bootstrap/**"
-  pull_request_target:
-    types: [labeled, opened, synchronize, reopened, ready_for_review]
-    paths:
-      - "openmetadata-service/**"
-      - "openmetadata-spec/src/main/resources/json/schema/**"
-      - "openmetadata-dist/**"
-      - "openmetadata-clients/**"
-      - "openmetadata-sdk/**"
-      - "common/**"
-      - "pom.xml"
-      - "yarn.lock"
-      - "Makefile"
-      - "bootstrap/**"
+  workflow_dispatch:
 
 permissions:
   contents: read

bootstrap/sql/migrations/native/1.12.0/mysql/schemaChanges.sql

@@ -246,6 +246,11 @@ CREATE TABLE IF NOT EXISTS learning_resource_entity (
   UNIQUE KEY fqnHash (fqnHash)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
 
+-- Widen entityLink generated column from VARCHAR(255) to TEXT
+-- The entity link from workflow variables can exceed 255 characters for deeply nested entities
+ALTER TABLE workflow_instance_time_series
+MODIFY COLUMN entityLink TEXT GENERATED ALWAYS AS (json ->> '$.variables.global_relatedEntity');
+
 -- Add process and vector stage columns to search_index_server_stats table
 -- These columns support the 4-stage pipeline model (Reader, Process, Sink, Vector) for search indexing stats
 

bootstrap/sql/migrations/native/1.12.0/postgres/schemaChanges.sql

@@ -262,6 +262,10 @@ CREATE TABLE IF NOT EXISTS learning_resource_entity (
     UNIQUE (fqnhash)
 );
 
+-- Widen entityLink generated column from VARCHAR(255) to TEXT
+-- The entity link from workflow variables can exceed 255 characters for deeply nested entities
+ALTER TABLE workflow_instance_time_series ALTER COLUMN entityLink TYPE TEXT;
+
 -- Add process and vector stage columns to search_index_server_stats table
 -- These columns support the 4-stage pipeline model (Reader, Process, Sink, Vector) for search indexing stats
 

conf/openmetadata.yaml

@@ -225,12 +225,12 @@ database:
   # These work across both PostgreSQL and MySQL drivers
   properties:
     # HikariCP connection pool settings
-    connectionTimeout: ${DB_CONNECTION_TIMEOUT:-300000}  # 300 seconds
-    idleTimeout: ${DB_IDLE_TIMEOUT:-600000}  # 10 minutes
-    maxLifetime: ${DB_MAX_LIFETIME:-1800000}  # 30 minutes
-    leakDetectionThreshold: ${DB_LEAK_DETECTION_THRESHOLD:-600000}  # 10 minute
+    connectionTimeout: ${DB_CONNECTION_TIMEOUT:-30000}  # 30 seconds - fail fast if pool exhausted
+    idleTimeout: ${DB_IDLE_TIMEOUT:-120000}  # 2 minutes - reclaim idle connections faster
+    maxLifetime: ${DB_MAX_LIFETIME:-600000}  # 10 minutes - rotate connections (cloud DB friendly)
+    leakDetectionThreshold: ${DB_LEAK_DETECTION_THRESHOLD:-60000}  # 1 minute - detect leaks early
     keepaliveTime: ${DB_KEEPALIVE_TIME:-0}  # 0 = disabled (set to 30000 for Aurora)
-    validationTimeout: ${DB_VALIDATION_TIMEOUT:-300000}  # 300 seconds
+    validationTimeout: ${DB_VALIDATION_TIMEOUT:-5000}  # 5 seconds - HikariCP default
     # PostgreSQL specific - these are ignored by MySQL driver
     prepareThreshold: ${DB_PG_PREPARE_THRESHOLD:-1}  # Use prepared statements immediately
     preparedStatementCacheQueries: ${DB_PG_PREP_STMT_CACHE_QUERIES:-500}  # Cache more statements
@@ -580,10 +580,9 @@ limits:
   limitsConfigFile: ${LIMITS_CONFIG_FILE:-""}
 
 # Bulk Operation Configuration
-# Controls parallelism and resource usage for bulk API operations (e.g., bulk import/export)
-# Uses a bounded thread pool to prevent connection pool exhaustion
+# Controls parallelism and admission control for bulk API operations.
 bulkOperation:
-  # Max threads for bulk operations (recommendations: 2 vCore=5-8, 4 vCore=8-15, 8 vCore=15-25)
+  # Max threads for processing
   maxThreads: ${BULK_OPERATION_MAX_THREADS:-10}
   # Max queued operations before rejection (returns 503)
   queueSize: ${BULK_OPERATION_QUEUE_SIZE:-1000}

docker/development/distributed-test/docker-compose.yml

@@ -49,7 +49,7 @@ services:
 
   # Shared OpenSearch
   opensearch:
-    image: opensearchproject/opensearch:2.7.0
+    image: opensearchproject/opensearch:3.4.0
     container_name: distributed_test_opensearch
     restart: unless-stopped
     environment:

docker/development/distributed-test/local/docker-compose-deps.yml

@@ -49,7 +49,7 @@ services:
 
   # OpenSearch
   opensearch:
-    image: opensearchproject/opensearch:2.7.0
+    image: opensearchproject/opensearch:3.4.0
     container_name: distributed_test_opensearch
     restart: unless-stopped
     environment:

docker/development/docker-compose-gcp.yml

@@ -46,7 +46,7 @@ services:
      - ./docker-volume/db-data:/var/lib/mysql
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
     container_name: openmetadata_elasticsearch
     environment:
       - discovery.type=single-node

docker/development/docker-compose-postgres-fuseki.yml

@@ -44,7 +44,7 @@ services:
       retries: 10
 
   opensearch:
-    image: opensearchproject/opensearch:latest
+    image: opensearchproject/opensearch:3.4.0
     container_name: openmetadata_opensearch
     environment:
       - discovery.type=single-node