Merge remote-tracking branch 'origin/batch_pipeline_status' into batch_pipeline_status

Author: harshach

.github/workflows/playwright-postgresql-e2e.yml

@@ -95,7 +95,10 @@ jobs:
         uses: ./.github/actions/setup-openmetadata-test-environment
         with:
           python-version: "3.10"
-          args: "-d postgresql"
+          # Pass different args depending on the shardIndex:
+          # - shard 2: run with ingestion setup
+          # - other shards: skip ingestion setup
+          args: ${{ matrix.shardIndex == 2 && '-d postgresql' || '-d postgresql -i false' }}
           ingestion_dependency: "all"
 
       - name: Setup Node.js
@@ -118,27 +121,24 @@ jobs:
             npx playwright test \
               --project=setup \
               --project=DataAssetRulesEnabled \
-              --project=DataAssetRulesDisabled
+              --project=DataAssetRulesDisabled \
+              --project=Basic \
+          
+          elif [ "${{ matrix.shardIndex }}" -eq "2" ]; then
+            echo "🔹 Running ingestion tests on shard 2"
+            
+            # run ingestion framework tests
+            npx playwright test \
+              --project=ingestion \
 
           else
-            # Shards 2-6 handle chromium tests (5 workers total)
-            CHROMIUM_SHARD=$(( ${{ matrix.shardIndex }} - 1 ))
-            if [[ "${{ contains(github.event.pull_request.changed_files, 'ingestion/') }}" == "true" ]]; then
-              echo "🔹 Running ingestion tests on chromium shard ${CHROMIUM_SHARD}/5"
-              npx playwright test \
-                --project=chromium \
-                --grep @ingestion \
-                --grep-invert @dataAssetRules \
-                --shard=${CHROMIUM_SHARD}/5
-                --workers=50%
-
-            else
-              echo "🔹 Running all other tests (excluding DataAssetRules) on chromium shard ${CHROMIUM_SHARD}/5"
-              npx playwright test \
-                --project=chromium \
-                --grep-invert @dataAssetRules \
-                --shard=${CHROMIUM_SHARD}/5
-            fi
+            # Shards 3-6 handle chromium tests (4 workers total)
+            CHROMIUM_SHARD=$(( ${{ matrix.shardIndex }} - 2 ))
+            echo "🔹 Running all other tests (excluding DataAssetRules) on chromium shard ${CHROMIUM_SHARD}/4"
+            npx playwright test \
+              --project=chromium \
+              --grep-invert @dataAssetRules \
+              --shard=${CHROMIUM_SHARD}/4
           fi
 
         env:

docker/development/distributed-test/docker-compose.yml

@@ -49,7 +49,7 @@ services:
 
   # Shared OpenSearch
   opensearch:
-    image: opensearchproject/opensearch:2.7.0
+    image: opensearchproject/opensearch:3.4.0
     container_name: distributed_test_opensearch
     restart: unless-stopped
     environment:

docker/development/distributed-test/local/docker-compose-deps.yml

@@ -49,7 +49,7 @@ services:
 
   # OpenSearch
   opensearch:
-    image: opensearchproject/opensearch:2.7.0
+    image: opensearchproject/opensearch:3.4.0
     container_name: distributed_test_opensearch
     restart: unless-stopped
     environment:

docker/development/docker-compose-gcp.yml

@@ -46,7 +46,7 @@ services:
      - ./docker-volume/db-data:/var/lib/mysql
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
     container_name: openmetadata_elasticsearch
     environment:
       - discovery.type=single-node

docker/development/docker-compose-postgres-fuseki.yml

@@ -44,7 +44,7 @@ services:
       retries: 10
 
   opensearch:
-    image: opensearchproject/opensearch:latest
+    image: opensearchproject/opensearch:3.4.0
     container_name: openmetadata_opensearch
     environment:
       - discovery.type=single-node

docker/development/docker-compose-postgres.yml

@@ -43,7 +43,7 @@ services:
       retries: 10
 
   opensearch:
-    image: opensearchproject/opensearch:latest
+    image: opensearchproject/opensearch:3.4.0
     container_name: openmetadata_opensearch
     environment:
       - discovery.type=single-node

docker/development/docker-compose.yml

@@ -42,7 +42,7 @@ services:
      - ./docker-volume/db-data:/var/lib/mysql
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
     container_name: openmetadata_elasticsearch
     environment:
       - discovery.type=single-node

docker/development/helm/docker-compose-deps.yml

@@ -44,7 +44,7 @@ services:
   # OpenSearch for metadata search
   opensearch:
     container_name: openmetadata_opensearch_test
-    image: opensearchproject/opensearch:2.11.1
+    image: opensearchproject/opensearch:3.4.0
     restart: always
     environment:
       - discovery.type=single-node

docker/docker-compose-quickstart/docker-compose-opensearch-standalone.yml

@@ -1,7 +1,7 @@
-# Standalone OpenSearch 2.19 with security disabled for local testing
+# Standalone OpenSearch 3.0 with security disabled for local testing
 services:
   opensearch:
-    image: opensearchproject/opensearch:2.19.0
+    image: opensearchproject/opensearch:3.4.0
     platform: linux/arm64  # Native ARM64 support
     container_name: opensearch-standalone
     hostname: opensearch

docker/docker-compose-quickstart/docker-compose-postgres.yml

@@ -41,7 +41,7 @@ services:
 
   elasticsearch:
     container_name: openmetadata_elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
     environment:
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1024m -Xmx1024m