Deploy: updated charts.

Author: vladd-bit

deploy/Makefile

@@ -11,11 +11,17 @@ REMOTE_SSH_KEY_ARG = $(if $(strip $(REMOTE_SSH_KEY)),-i $(REMOTE_SSH_KEY),)
 HELM_OPENSEARCH_RELEASE ?= cogstack-opensearch
 HELM_OPENSEARCH_NAMESPACE ?= cogstack
 HELM_OPENSEARCH_CHART ?= ./charts/opensearch
+HELM_OPENSEARCH_CONFIG_FILE ?= ../services/elasticsearch/config/opensearch.yml
+HELM_OPENSEARCH_LOG4J_FILE ?= ../services/elasticsearch/config/log4j2_opensearch.properties
+HELM_DASHBOARDS_CONFIG_FILE ?= ../services/kibana/config/opensearch.yml
 HELM_OPENSEARCH_ENV_FILE ?= ./elasticsearch.env
 HELM_OPENSEARCH_USERS_ENV_FILE ?= ../security/env/users_elasticsearch.env
 HELM_OPENSEARCH_CERTS_ENV_FILE ?= ../security/env/certificates_elasticsearch.env
 HELM_OPENSEARCH_SECURITY_DIR ?= ../security/es_roles/opensearch
-HELM_OPENSEARCH_SET_FILES = --set-file envFile.raw=$(HELM_OPENSEARCH_ENV_FILE) \
+HELM_OPENSEARCH_SET_FILES = --set-file configFiles.opensearchRaw=$(HELM_OPENSEARCH_CONFIG_FILE) \
+	--set-file configFiles.log4jRaw=$(HELM_OPENSEARCH_LOG4J_FILE) \
+	--set-file configFiles.dashboardsRaw=$(HELM_DASHBOARDS_CONFIG_FILE) \
+	--set-file envFile.raw=$(HELM_OPENSEARCH_ENV_FILE) \
 	--set-file usersEnvFile.raw=$(HELM_OPENSEARCH_USERS_ENV_FILE) \
 	--set-file certificatesEnvFile.raw=$(HELM_OPENSEARCH_CERTS_ENV_FILE) \
 	--set-file securityFiles.configRaw=$(HELM_OPENSEARCH_SECURITY_DIR)/config.yml \
@@ -83,10 +89,10 @@ load-env: ## Load variables from export_env_vars.sh in a subshell
 show-env: ## Print sorted environment variables after loading export_env_vars.sh
 	${WITH_ENV} >/dev/null 2>&1; printenv | sort
 
-helm-template-opensearch: ## Render OpenSearch chart using deploy/security env + opensearch role files
+helm-template-opensearch: ## Render OpenSearch chart using shared services/security config files
 	helm template $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_SET_FILES)
 
-helm-install-opensearch: ## Install/upgrade OpenSearch chart using deploy/security env + opensearch role files
+helm-install-opensearch: ## Install/upgrade OpenSearch chart using shared services/security config files
 	helm upgrade --install $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_SET_FILES) --namespace $(HELM_OPENSEARCH_NAMESPACE) --create-namespace
 
 

deploy/charts/README.md

@@ -16,6 +16,9 @@ This directory contains Helm charts owned by this repository's deployment layer.
 ```bash
 # Render manifests
 helm template cogstack-opensearch ./deploy/charts/opensearch \
+  --set-file configFiles.opensearchRaw=./services/elasticsearch/config/opensearch.yml \
+  --set-file configFiles.log4jRaw=./services/elasticsearch/config/log4j2_opensearch.properties \
+  --set-file configFiles.dashboardsRaw=./services/kibana/config/opensearch.yml \
   --set-file envFile.raw=./deploy/elasticsearch.env \
   --set-file usersEnvFile.raw=./security/env/users_elasticsearch.env \
   --set-file certificatesEnvFile.raw=./security/env/certificates_elasticsearch.env \
@@ -26,6 +29,9 @@ helm template cogstack-opensearch ./deploy/charts/opensearch \
 
 # Install/upgrade
 helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \
+  --set-file configFiles.opensearchRaw=./services/elasticsearch/config/opensearch.yml \
+  --set-file configFiles.log4jRaw=./services/elasticsearch/config/log4j2_opensearch.properties \
+  --set-file configFiles.dashboardsRaw=./services/kibana/config/opensearch.yml \
   --set-file envFile.raw=./deploy/elasticsearch.env \
   --set-file usersEnvFile.raw=./security/env/users_elasticsearch.env \
   --set-file certificatesEnvFile.raw=./security/env/certificates_elasticsearch.env \
@@ -36,4 +42,5 @@ helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \
   --namespace cogstack --create-namespace
 ```
 
+The OpenSearch and Dashboards config files should come from `services/`, and the security files from `security/`, so Docker and Kubernetes use the same source files.
 Only keys in `envFile.includeKeys` and `usersEnvFile.includeKeys` are imported.

deploy/charts/opensearch/README.md

@@ -38,6 +38,9 @@ Secret keys are mapped via:
 
 ```bash
 helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \
+  --set-file configFiles.opensearchRaw=./services/elasticsearch/config/opensearch.yml \
+  --set-file configFiles.log4jRaw=./services/elasticsearch/config/log4j2_opensearch.properties \
+  --set-file configFiles.dashboardsRaw=./services/kibana/config/opensearch.yml \
   --set-file envFile.raw=./deploy/elasticsearch.env \
   --set-file usersEnvFile.raw=./security/env/users_elasticsearch.env \
   --set-file certificatesEnvFile.raw=./security/env/certificates_elasticsearch.env \
@@ -64,6 +67,9 @@ helm upgrade --install cogstack-dashboards ./deploy/charts/opensearch \
 
 ```bash
 helm template cogstack-opensearch ./deploy/charts/opensearch \
+  --set-file configFiles.opensearchRaw=./services/elasticsearch/config/opensearch.yml \
+  --set-file configFiles.log4jRaw=./services/elasticsearch/config/log4j2_opensearch.properties \
+  --set-file configFiles.dashboardsRaw=./services/kibana/config/opensearch.yml \
   --set-file envFile.raw=./deploy/elasticsearch.env \
   --set-file usersEnvFile.raw=./security/env/users_elasticsearch.env \
   --set-file certificatesEnvFile.raw=./security/env/certificates_elasticsearch.env \
@@ -75,10 +81,16 @@ helm template cogstack-opensearch ./deploy/charts/opensearch \
 
 ## Notes
 
-- The chart packages current repository config files under `files/`.
+- Helm templates cannot read arbitrary `../../...` paths directly; `.Files.Get` only sees files packaged inside the chart.
+- In this repo, the chart `files/` entries are symlinked to the shared `services/` and `security/` sources so Docker and Kubernetes stay aligned.
+- The standard install/render commands still use `--set-file` explicitly to make the source-of-truth paths obvious at invocation time.
+- If you run Helm from `deploy/charts/opensearch`, the equivalent relative paths are `../../../services/...` and `../../../security/...`.
 - `envFile.raw` can be set from `deploy/elasticsearch.env` and is loaded via `envFrom` into OpenSearch and Dashboards.
 - `usersEnvFile.raw` can be set from `security/env/users_elasticsearch.env` and feeds the credentials Secret (`OPENSEARCH_INITIAL_ADMIN_PASSWORD`, `KIBANA_USER`, `KIBANA_PASSWORD`).
 - `certificatesEnvFile.raw` can be set from `security/env/certificates_elasticsearch.env`; currently `ES_CLIENT_CERT_NAME` is used to resolve Dashboards cert secret keys (`<name>.pem` / `<name>.key`).
+- `configFiles.opensearchRaw` can be set from `services/elasticsearch/config/opensearch.yml`.
+- `configFiles.log4jRaw` can be set from `services/elasticsearch/config/log4j2_opensearch.properties`.
+- `configFiles.dashboardsRaw` can be set from `services/kibana/config/opensearch.yml`.
 - `securityFiles.*Raw` can be set from `security/es_roles/opensearch/*.yml` and overrides the chart-bundled OpenSearch security files.
 - Only keys listed in `envFile.includeKeys` are imported (to avoid leaking secrets from env files into ConfigMaps).
 - Review security and certificate settings before production use.

deploy/charts/opensearch/files/log4j2.properties

@@ -0,0 +1 @@
+../../../../services/elasticsearch/config/log4j2_opensearch.properties

deploy/charts/opensearch/files/log4j2.properties

@@ -0,0 +1 @@
+../../../../../security/es_roles/opensearch/config.yml