feat: Use Opensearch operator and OpenSearch Dashboards image in logging demo

docs/modules/demos/pages/logging.adoc

@@ -26,16 +26,6 @@ To run this demo, your system needs at least:
 
 If you use MacOS or Windows and use Docker to run Kubernetes, set the RAM to at least 4 GB in _Preferences > Resources_.
 
-==== Linux
-
-OpenSearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts
-are likely too low - usually 65530, which may result in out-of-memory exceptions. So, the Linux setting
-`vm.max_map_count` on the host machine where the containers are running must be set to at least 262144.
-
-This is automatically set by default in this demo (via the `setSysctlMaxMapCount` Stack parameter).
-
-OpenSearch has more information about this setting in their https://opensearch.org/docs/2.12/install-and-configure/install-opensearch/index/#important-settings[documentation].
-
 == Overview
 
 This demo will
@@ -63,15 +53,16 @@ To list the installed Stackable services run the following command:
 [source,console]
 ----
 $ stackablectl stacklet list
-┌───────────────────────┬───────────────────────┬───────────┬─────────────────────────────────────────────────┬─────────────────────────────────┐
-│ PRODUCT               ┆ NAME                  ┆ NAMESPACE ┆ ENDPOINTS                                       ┆ CONDITIONS                      │
-╞═══════════════════════╪═══════════════════════╪═══════════╪═════════════════════════════════════════════════╪═════════════════════════════════╡
-│ zookeeper             ┆ simple-zk             ┆ default   ┆ server-zk                                       ┆ Available, Reconciling, Running │
-│                       ┆                       ┆           ┆ simple-zk-server.default.svc.cluster.local:2282 ┆                                 │
-├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
-│ opensearch-dashboards ┆ opensearch-dashboards ┆ default   ┆ http       http://172.18.0.2:31734              ┆                                 │
-│                       ┆                       ┆           ┆ metrics    172.18.0.2:32120                     ┆                                 │
-└───────────────────────┴───────────────────────┴───────────┴─────────────────────────────────────────────────┴─────────────────────────────────┘
+┌───────────────────────┬───────────────────────┬───────────┬────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐
+│ PRODUCT               ┆ NAME                  ┆ NAMESPACE ┆ ENDPOINTS                                                                          ┆ CONDITIONS                      │
+╞═══════════════════════╪═══════════════════════╪═══════════╪════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡
+│ opensearch            ┆ opensearch            ┆ default   ┆ nodes-default-http  http://opensearch-nodes-default.default.svc.cluster.local:9200 ┆ Available, Reconciling, Running │
+├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
+│ zookeeper             ┆ simple-zk             ┆ default   ┆ server-zk           simple-zk-server.default.svc.cluster.local:2282                ┆ Available, Reconciling, Running │
+├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
+│ opensearch-dashboards ┆ opensearch-dashboards ┆ default   ┆ http                http://172.18.0.2:30595                                        ┆                                 │
+│                       ┆                       ┆           ┆ metrics             172.18.0.2:31767                                               ┆                                 │
+└───────────────────────┴───────────────────────┴───────────┴────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘
 ----
 
 include::partial$instance-hint.adoc[]

stacks/_templates/opensearch-dashboards.yaml

@@ -4,8 +4,12 @@ name: opensearch-dashboards
 repo:
   name: opensearch-dashboards
   url: https://opensearch-project.github.io/helm-charts
-version: 2.30.0 # 2.19.2
+version: {{ opensearchVersion }}
 options:
+  opensearchHosts: https://opensearch:9200
+  image:
+    repository: oci.stackable.tech/sdp/opensearch-dashboards
+    tag: "{{ opensearchVersion }}-stackable{{ stackableReleaseVersion }}"
   labels:
     stackable.tech/vendor: Stackable
   service:
@@ -19,9 +23,8 @@ options:
       stackable.tech/vendor: Stackable
   opensearchAccount:
     secret: opensearch-dashboard-user
-  extraEnvs:
-    - name: OPEN_SEARCH_ADMIN_PASSWORD
-      valueFrom:
-        secretKeyRef:
-          name: opensearch-user
-          key: password
+  serviceAccount:
+    create: false
+    # Use the ServiceAccount of OpenSearch because its permissions are already configured to work on
+    # OpenShift.
+    name: opensearch-serviceaccount

stacks/_templates/vector-aggregator.yaml

@@ -23,7 +23,7 @@ options:
         inputs:
           - vector
         endpoints:
-          - https://opensearch-cluster-master.default.svc.cluster.local:9200
+          - https://opensearch.default.svc.cluster.local:9200
         mode: bulk
         # The auto-detection of the API version does not work in Vector
         # 0.41.1 for OpenSearch, so the version must be set explicitly

stacks/argo-cd-git-ops/applicationsets/stackable-operators.yaml

@@ -20,8 +20,7 @@ spec:
                 - operator: kafka
                 - operator: nifi
                 - operator: opa
-                # TODO: enable Opensearch operator
-                # - opensearch
+                - operator: opensearch
                 - operator: spark-k8s
                 - operator: superset
                 - operator: trino

stacks/logging/opensearch.yaml

@@ -0,0 +1,179 @@
+apiVersion: opensearch.stackable.tech/v1alpha1
+kind: OpenSearchCluster
+metadata:
+  name: opensearch
+spec:
+  image:
+    productVersion: {{ opensearchVersion }}
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    vectorAggregatorConfigMapName: vector-aggregator-discovery
+  nodes:
+    config:
+      logging:
+        enableVectorAgent: true
+    roleGroups:
+      default:
+        config:
+          listenerClass: cluster-internal
+        replicas: 1
+    configOverrides:
+      opensearch.yml:
+        # Disable memory mapping in this stack; If memory mapping were activated, the kernel setting
+        # vm.max_map_count would have to be increased to 262144 on the node.
+        node.store.allow_mmap: "false"
+        # Disable the disk allocation decider in this stack; Otherwise depending on the disk
+        # usage of the node and if the relative watermark set in
+        # `cluster.routing.allocation.disk.watermark.high` is reached the security index can't
+        # be created even if enough disk space would be available.
+        cluster.routing.allocation.disk.threshold_enabled: "false"
+        plugins.security.allow_default_init_securityindex: "true"
+        plugins.security.ssl.transport.enabled: "true"
+        plugins.security.ssl.transport.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt
+        plugins.security.ssl.transport.pemkey_filepath: /stackable/opensearch/config/tls/tls.key
+        plugins.security.ssl.transport.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt
+        plugins.security.ssl.http.enabled: "true"
+        plugins.security.ssl.http.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt
+        plugins.security.ssl.http.pemkey_filepath: /stackable/opensearch/config/tls/tls.key
+        plugins.security.ssl.http.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt
+    podOverrides:
+      spec:
+        containers:
+          - name: opensearch
+            volumeMounts:
+              - name: security-config
+                mountPath: /stackable/opensearch/config/opensearch-security
+                readOnly: true
+              - name: tls
+                mountPath: /stackable/opensearch/config/tls
+                readOnly: true
+        volumes:
+          - name: security-config
+            secret:
+              secretName: opensearch-security-config
+              defaultMode: 0o660
+          - name: tls
+            ephemeral:
+              volumeClaimTemplate:
+                metadata:
+                  annotations:
+                    secrets.stackable.tech/class: tls
+                    secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-default-headless
+                spec:
+                  storageClassName: secrets.stackable.tech
+                  accessModes:
+                    - ReadWriteOnce
+                  resources:
+                    requests:
+                      storage: "1"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: opensearch-security-config
+stringData:
+  action_groups.yml: |
+    ---
+    _meta:
+      type: actiongroups
+      config_version: 2
+  allowlist.yml: |
+    ---
+    _meta:
+      type: allowlist
+      config_version: 2
+
+    config:
+      enabled: false
+  audit.yml: |
+    ---
+    _meta:
+      type: audit
+      config_version: 2
+
+    config:
+      enabled: false
+  config.yml: |
+    ---
+    _meta:
+      type: config
+      config_version: 2
+
+    config:
+      dynamic:
+        authc:
+          basic_internal_auth_domain:
+            description: Authenticate via HTTP Basic against internal users database
+            http_enabled: true
+            transport_enabled: true
+            order: 1
+            http_authenticator:
+              type: basic
+              challenge: true
+            authentication_backend:
+              type: intern
+        authz: {}
+  internal_users.yml: |
+    ---
+    _meta:
+      type: internalusers
+      config_version: 2
+
+    admin:
+      hash: {{ bcrypt(password=openSearchAdminPassword) }}
+      reserved: true
+      backend_roles:
+        - admin
+      description: OpenSearch admin user
+
+    kibanaserver:
+      hash: {{ bcrypt(password=openSearchDashboardPassword) }}
+      reserved: true
+      description: OpenSearch Dashboards user
+  nodes_dn.yml: |
+    ---
+    _meta:
+      type: nodesdn
+      config_version: 2
+  roles.yml: |
+    ---
+    _meta:
+      type: roles
+      config_version: 2
+  roles_mapping.yml: |
+    ---
+    _meta:
+      type: rolesmapping
+      config_version: 2
+
+    all_access:
+      reserved: false
+      backend_roles:
+        - admin
+
+    kibana_server:
+      reserved: true
+      users:
+        - kibanaserver
+  tenants.yml: |
+    ---
+    _meta:
+      type: tenants
+      config_version: 2
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: opensearch-user
+stringData:
+  username: admin
+  password: {{ openSearchAdminPassword }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: opensearch-dashboard-user
+stringData:
+  username: kibanaserver
+  password: {{ openSearchDashboardPassword }}
+  cookie: {{ random_password() }}

stacks/logging/setup-opensearch-dashboards.yaml

@@ -17,7 +17,6 @@ spec:
                   key: password
           command:
             - bash
-            - -x
             - -euo
             - pipefail
             - -c