chore(devenv): upgrade of opensearch to 2.17 and stability improvements (#14895)

Author: alexsku

docker/elasticsearch-setup/create-indices.sh

@@ -89,7 +89,7 @@ function create_if_not_exists {
 
   else
     # when `USE_AWS_ELASTICSEARCH` was forgotten to be set to `true` when running against AWS ES OSS,
-    # this script will use wrong paths (e.g. `_ilm/policy/` instead of AWS-compatible `_opendistro/_ism/policies/`)
+    # this script will use wrong paths (e.g. `_ilm/policy/` instead of AWS-compatible `_plugins/_ism/policies/`)
     # and the ES endpoint will return `401 Unauthorized` or `405 Method Not Allowed`
     # let's use this as chance to point that wrong config might be used!
     if [ $RESOURCE_STATUS -eq 401 ] || [ $RESOURCE_STATUS -eq 405 ]; then
@@ -148,11 +148,11 @@ function create_datahub_usage_event_datastream() {
 function create_datahub_usage_event_aws_elasticsearch() {
   # AWS env requires creation of three resources for Datahub usage events:
   #   1. ISM policy
-  create_if_not_exists "_opendistro/_ism/policies/${PREFIX}datahub_usage_event_policy" aws_es_ism_policy.json
+  create_if_not_exists "_plugins/_ism/policies/${PREFIX}datahub_usage_event_policy" aws_es_ism_policy.json
 
   #   1.1 ISM policy update if it already existed
   if [ $RESOURCE_STATUS -eq 200 ]; then
-    update_ism_policy "_opendistro/_ism/policies/${PREFIX}datahub_usage_event_policy" aws_es_ism_policy.json
+    update_ism_policy "_plugins/_ism/policies/${PREFIX}datahub_usage_event_policy" aws_es_ism_policy.json
   fi
 
   #   2. index template

docker/elasticsearch/env/docker.env

@@ -1,2 +1,3 @@
 ES_JAVA_OPTS="-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true"
-OPENSEARCH_JAVA_OPTS="-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true"
+OPENSEARCH_JAVA_OPTS="-Xms768m -Xmx1024m -Dlog4j2.formatMsgNoLookups=true"
+DISABLE_SECURITY_PLUGIN="true"

docker/profiles/docker-compose.prerequisites.yml

@@ -397,25 +397,25 @@ services:
   opensearch:
     profiles: *opensearch-profiles
     hostname: search
-    image: ${DATAHUB_SEARCH_IMAGE:-opensearchproject/opensearch}:${DATAHUB_SEARCH_TAG:-2.11.0}
+    image: ${DATAHUB_SEARCH_IMAGE:-opensearchproject/opensearch}:${DATAHUB_SEARCH_TAG:-2.17.0}
     ports:
       - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
     env_file: elasticsearch/env/docker.env
     environment:
       - discovery.type=single-node
-      - ${XPACK_SECURITY_ENABLED:-plugins.security.disabled=true}
+      - DISABLE_SECURITY_PLUGIN=true
     deploy:
       resources:
         limits:
-          memory: 1G
+          memory: 2G
     healthcheck:
       test: curl -sS --fail http://search:$${DATAHUB_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s
-      start_period: 30s
-      interval: 1s
-      retries: 3
-      timeout: 5s
+      start_period: 60s
+      interval: 5s
+      retries: 10
+      timeout: 15s
     volumes:
-      - osdata:/usr/share/elasticsearch/data
+      - osdata:/usr/share/opensearch/data
   opensearch-setup: &opensearch-setup
     <<: *elasticsearch-setup-dev
     profiles: *opensearch-profiles-quickstart
@@ -427,6 +427,7 @@ services:
     depends_on:
       opensearch:
         condition: service_healthy
+    command: /bin/sh -c "sleep 10 && /create-indices.sh"
     labels:
       datahub_setup_job: true
   opensearch-setup-dev:
@@ -440,6 +441,7 @@ services:
     depends_on:
       opensearch:
         condition: service_healthy
+    command: /bin/sh -c "sleep 10 && /create-indices.sh"
   aws-localstack:
     profiles: *aws-profiles
     hostname: localstack

docker/quickstart/docker-compose.quickstart-profile.yml

@@ -315,22 +315,22 @@ services:
     deploy:
       resources:
         limits:
-          memory: '1073741824'
+          memory: '2147483648'
     environment:
+      DISABLE_SECURITY_PLUGIN: 'true'
       ES_JAVA_OPTS: -Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true
-      OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true
+      OPENSEARCH_JAVA_OPTS: -Xms768m -Xmx1024m -Dlog4j2.formatMsgNoLookups=true
       discovery.type: single-node
-      plugins.security.disabled: 'true'
     hostname: search
     healthcheck:
       test:
       - CMD-SHELL
       - curl -sS --fail http://search:$${DATAHUB_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s
-      timeout: 5s
-      interval: 1s
-      retries: 3
-      start_period: 30s
-    image: opensearchproject/opensearch:2.11.0
+      timeout: 15s
+      interval: 5s
+      retries: 10
+      start_period: 1m0s
+    image: opensearchproject/opensearch:2.17.0
     networks:
       default: null
     ports:
@@ -341,7 +341,7 @@ services:
     volumes:
     - type: volume
       source: osdata
-      target: /usr/share/elasticsearch/data
+      target: /usr/share/opensearch/data
       volume: {}
   opensearch-setup:
     profiles:
@@ -355,6 +355,10 @@ services:
     - quickstart-postgres-cdc
     - quickstart-consumers
     - quickstart-consumers-cdc
+    command:
+    - /bin/sh
+    - -c
+    - sleep 10 && /create-indices.sh
     depends_on:
       opensearch:
         condition: service_healthy

metadata-io/src/test/java/io/datahubproject/test/search/OpenSearchTestContainer.java

@@ -7,7 +7,7 @@
 import org.testcontainers.utility.DockerImageName;
 
 public class OpenSearchTestContainer implements SearchTestContainer {
-  private static final String OPENSEARCH_VERSION = "2.11.0";
+  private static final String OPENSEARCH_VERSION = "2.17.0";
   private static final String OPENSEARCH_IMAGE_NAME = "opensearchproject/opensearch";
   private static final String ENV_OPENSEARCH_IMAGE_FULL_NAME =
       System.getenv("OPENSEARCH_IMAGE_FULL_NAME");
@@ -31,6 +31,7 @@ public class OpenSearchTestContainer implements SearchTestContainer {
     checkContainerEngine(OS_CONTAINER.getDockerClient());
     OS_CONTAINER
         .withEnv("OPENSEARCH_JAVA_OPTS", SEARCH_JAVA_OPTS)
+        .withEnv("DISABLE_SECURITY_PLUGIN", "true")
         .withStartupTimeout(STARTUP_TIMEOUT);
   }