diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml
index 9e30c71b7791..c0e3b12aa977 100644
--- a/.github/workflows/skywalking.yaml
+++ b/.github/workflows/skywalking.yaml
@@ -387,9 +387,6 @@ jobs:
- name: Storage ES 8.9.0
config: test/e2e-v2/cases/storage/es/e2e.yaml
env: ES_VERSION=8.18.1
- - name: Storage OpenSearch 1.1.0
- config: test/e2e-v2/cases/storage/opensearch/e2e.yaml
- env: OPENSEARCH_VERSION=1.1.0
- name: Storage OpenSearch 1.3.10
config: test/e2e-v2/cases/storage/opensearch/e2e.yaml
env: OPENSEARCH_VERSION=1.3.10
@@ -1121,4 +1118,4 @@ jobs:
[[ ${e2eJavaVersionResults} == 'success' ]] || [[ ${execute} != 'true' && ${e2eJavaVersionResults} == 'skipped' ]] || exit -7;
[[ ${timeConsumingITResults} == 'success' ]] || [[ ${execute} != 'true' && ${timeConsumingITResults} == 'skipped' ]] || exit -8;
- exit 0;
\ No newline at end of file
+ exit 0;
diff --git a/.licenserc.yaml b/.licenserc.yaml
index b77385e3fb54..15b5adf11402 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -109,10 +109,10 @@ dependency:
version: 2.13.4
license: Apache-2.0
- name: com.fasterxml.jackson.datatype:jackson-datatype-jsr310
- version: 2.18.2
+ version: 2.20.1
license: Apache-2.0
- name: com.fasterxml.jackson.datatype:jackson-datatype-jdk8
- version: 2.18.2
+ version: 2.20.1
license: Apache-2.0
- name: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml
version: 2.15.2
@@ -139,7 +139,7 @@ dependency:
version: 1.2.1
license: Apache-2.0
- name: com.aayushatharva.brotli4j:service
- version: 1.18.0
+ version: 1.20.0
license: Apache-2.0
- name: io.vertx:vertx-grpc
version: 4.5.9
diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE
index e0ffeb6cece7..6632dd2ac141 100644
--- a/dist-material/release-docs/LICENSE
+++ b/dist-material/release-docs/LICENSE
@@ -210,8 +210,8 @@ The following components are provided under the Apache-2.0 License. See project
The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/pgv-java-stub/1.2.1 Apache-2.0
https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/protoc-gen-validate/1.2.1 Apache-2.0
- https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/brotli4j/1.18.0 Apache-2.0
- https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/service/1.18.0 Apache-2.0
+ https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/brotli4j/1.20.0 Apache-2.0
+ https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/service/1.20.0 Apache-2.0
https://mvnrepository.com/artifact/com.alibaba.nacos/nacos-auth-plugin/2.3.2 Apache-2.0
https://mvnrepository.com/artifact/com.alibaba.nacos/nacos-client/2.3.2 Apache-2.0
https://mvnrepository.com/artifact/com.alibaba.nacos/nacos-encryption-plugin/2.3.2 Apache-2.0
@@ -222,8 +222,8 @@ The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind/2.16.0 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.15.2 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-guava/2.12.0 Apache-2.0
- https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.18.2 Apache-2.0
- https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.18.2 Apache-2.0
+ https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.20.1 Apache-2.0
+ https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.20.1 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-kotlin/2.13.4 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml/classmate/1.5.1 Apache-2.0
https://mvnrepository.com/artifact/com.google.api.grpc/proto-google-common-protos/2.48.0 Apache-2.0
@@ -238,12 +238,12 @@ The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/com.google.inject/guice/4.1.0 Apache-2.0
https://mvnrepository.com/artifact/com.google.j2objc/j2objc-annotations/2.8 Apache-2.0
https://mvnrepository.com/artifact/com.graphql-java/java-dataloader/3.2.1 Apache-2.0
- https://mvnrepository.com/artifact/com.linecorp.armeria/armeria/1.32.0 Apache-2.0
- https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql/1.32.0 Apache-2.0
- https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql-protocol/1.32.0 Apache-2.0
- https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-grpc/1.32.0 Apache-2.0
- https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-grpc-protocol/1.32.0 Apache-2.0
- https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-protobuf/1.32.0 Apache-2.0
+ https://mvnrepository.com/artifact/com.linecorp.armeria/armeria/1.34.2 Apache-2.0
+ https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql/1.34.2 Apache-2.0
+ https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql-protocol/1.34.2 Apache-2.0
+ https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-grpc/1.34.2 Apache-2.0
+ https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-grpc-protocol/1.34.2 Apache-2.0
+ https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-protobuf/1.34.2 Apache-2.0
https://mvnrepository.com/artifact/com.orbitz.consul/consul-client/1.5.3 Apache-2.0
https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp/3.14.9 Apache-2.0
https://mvnrepository.com/artifact/com.squareup.okio/okio/1.17.2 Apache-2.0
@@ -300,6 +300,7 @@ The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/io.grpc/grpc-services/1.70.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-stub/1.70.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-util/1.70.0 Apache-2.0
+ https://mvnrepository.com/artifact/io.micrometer/context-propagation/1.2.0 Apache-2.0
https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.14.4 Apache-2.0
https://mvnrepository.com/artifact/io.micrometer/micrometer-core/1.14.4 Apache-2.0
https://mvnrepository.com/artifact/io.micrometer/micrometer-observation/1.14.4 Apache-2.0
@@ -369,6 +370,7 @@ The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/org.jetbrains.kotlinx/kotlinx-coroutines-jdk8/1.6.4 Apache-2.0
https://mvnrepository.com/artifact/org.jetbrains.kotlinx/kotlinx-coroutines-reactive/1.6.4 Apache-2.0
https://mvnrepository.com/artifact/org.jetbrains/annotations/13.0 Apache-2.0
+ https://mvnrepository.com/artifact/org.jspecify/jspecify/1.0.0 Apache-2.0
https://mvnrepository.com/artifact/org.lz4/lz4-java/1.8.0 Apache-2.0
https://mvnrepository.com/artifact/org.slf4j/jcl-over-slf4j/1.7.30 Apache-2.0
https://mvnrepository.com/artifact/org.slf4j/log4j-over-slf4j/1.7.30 Apache-2.0
@@ -543,7 +545,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt.
https://npmjs.com/package/nanoid/v/3.3.8 3.3.8 MIT
https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.33.0 MIT
https://mvnrepository.com/artifact/org.codehaus.mojo/animal-sniffer-annotations/1.24 MIT
- https://mvnrepository.com/artifact/org.curioswitch.curiostack/protobuf-jackson/2.7.0 MIT
+ https://mvnrepository.com/artifact/org.curioswitch.curiostack/protobuf-jackson/2.8.1 MIT
https://mvnrepository.com/artifact/org.slf4j/slf4j-api/1.7.30 MIT
https://npmjs.com/package/pinia/v/2.0.28 2.0.28 MIT
https://npmjs.com/package/pinia/node_modules/vue-demi/v/0.13.11 0.13.11 MIT
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 66b7a3427c55..2803d047c05d 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -14,6 +14,7 @@
* Add `LatestLabeledFunction` for meter.
* MAL Labeled metrics support additional attributes.
* Bump up netty to 4.2.9.Final.
+* Add support for OpenSearch/ElasticSearch client certificate authentication.
#### UI
* Fix the missing icon in new native trace view.
diff --git a/docs/en/setup/backend/storages/elasticsearch.md b/docs/en/setup/backend/storages/elasticsearch.md
index a2d60a5a731f..2f6ec21ba58c 100644
--- a/docs/en/setup/backend/storages/elasticsearch.md
+++ b/docs/en/setup/backend/storages/elasticsearch.md
@@ -11,7 +11,7 @@ In order to activate OpenSearch as storage, set the storage provider to **elasti
We support and tested the following versions of OpenSearch:
-- 1.1.0, 1.3.10
+- 1.3.10
- 2.4.0, 2.8.0, 3.0.0
## Elasticsearch
@@ -51,6 +51,8 @@ storage:
protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"}
trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""}
trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""}
+ keyStorePath: ${SW_STORAGE_ES_SSL_KEY_STORE_PATH:""} # Path to client certificate keystore for mutual TLS (OpenSearch/Elasticsearch client cert auth). Supports PKCS12 (.p12, .pfx) and JKS (.jks) formats.
+ keyStorePass: ${SW_STORAGE_ES_SSL_KEY_STORE_PASS:""} # Password for the client certificate keystore. Can be managed via secretsManagementFile.
user: ${SW_ES_USER:""}
password: ${SW_ES_PASSWORD:""}
secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool.
@@ -83,7 +85,9 @@ storage:
enableCustomRouting: ${SW_STORAGE_ES_ENABLE_CUSTOM_ROUTING:false}
```
-### ElasticSearch With Https SSL Encrypting communications.
+### ElasticSearch/OpenSearch With HTTPS SSL Encrypting Communications
+
+#### Basic HTTPS with Server Certificate Verification
Example:
@@ -103,6 +107,32 @@ storage:
- File at `trustStorePath` is being monitored. Once it is changed, the ElasticSearch client will reconnect.
- `trustStorePass` could be changed in the runtime through [**Secrets Management File Of ElasticSearch Authentication**](#secrets-management-file-of-elasticsearch-authentication).
+#### Mutual TLS (mTLS) with Client Certificate Authentication
+
+For enhanced security, you can configure mutual TLS where the client presents a certificate to the server. This is commonly used with OpenSearch security plugin's client certificate authentication.
+
+Example:
+
+```yaml
+storage:
+ selector: ${SW_STORAGE:elasticsearch}
+ elasticsearch:
+ namespace: ${SW_NAMESPACE:""}
+ clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200}
+ protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"https"}
+ trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:"../truststore.jks"}
+ trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:"changeit"}
+ keyStorePath: ${SW_STORAGE_ES_SSL_KEY_STORE_PATH:"../client.p12"}
+ keyStorePass: ${SW_STORAGE_ES_SSL_KEY_STORE_PASS:"changeit"}
+ ...
+```
+
+- `keyStorePath` points to the client certificate keystore file. Supports both PKCS12 (`.p12`, `.pfx`) and JKS (`.jks`) formats.
+- `keyStorePass` is the password for the client keystore. Use empty string `""` for keystores without password.
+- Both `trustStorePath` and `keyStorePath` files are being monitored. Once they are changed, the ElasticSearch client will reconnect.
+- `trustStorePass` and `keyStorePass` could be changed in the runtime through [**Secrets Management File Of ElasticSearch Authentication**](#secrets-management-file-of-elasticsearch-authentication).
+- When `keyStorePath` is configured, `keyStorePass` must also be provided (can be empty string for no password).
+
### Daily Index Step
Daily index step(`storage/elasticsearch/dayStep`, default 1) represents the index creation period. In this period, metrics for several days (dayStep value) are saved.
@@ -121,17 +151,18 @@ NOTE: TTL deletion would be affected by these steps. You should set an extra day
### Secrets Management File Of ElasticSearch Authentication
The value of `secretsManagementFile` should point to the secrets management file absolute path.
-The file includes the username, password, and JKS password of the ElasticSearch server in the properties format.
+The file includes the username, password, JKS password, and keystore password of the ElasticSearch server in the properties format.
```properties
user=xxx
password=yyy
trustStorePass=zzz
+keyStorePass=aaa
```
-The major difference between using `user, password, trustStorePass` configs in the `application.yaml` file is that the **Secrets Management File** is being watched by the OAP server.
+The major difference between using `user, password, trustStorePass, keyStorePass` configs in the `application.yaml` file is that the **Secrets Management File** is being watched by the OAP server.
Once it is changed manually or through a 3rd party tool, such as [Vault](https://github.com/hashicorp/vault),
-the storage provider will use the new username, password, and JKS password to establish the connection and close the old one. If the information exists in the file,
-the `user/password` will be overridden.
+the storage provider will use the new username, password, JKS password, and keystore password to establish the connection and close the old one. If the information exists in the file,
+the `user/password/trustStorePass/keyStorePass` will be overridden.
### Index Settings
diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml
index f990e755c640..4a35af481e9c 100644
--- a/oap-server-bom/pom.xml
+++ b/oap-server-bom/pom.xml
@@ -67,7 +67,7 @@
42.4.4
0.6.1
1.17.6
- 1.32.0
+ 1.34.2
3.0.0
4.4.16
4.1.5
diff --git a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java
index 10330088ae48..5a6a7933f5be 100644
--- a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java
+++ b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java
@@ -76,6 +76,11 @@ public class ElasticSearchClient implements Client, HealthCheckable {
@Setter
private volatile String trustStorePass;
+ private final String keyStorePath;
+
+ @Setter
+ private volatile String keyStorePass;
+
@Setter
private volatile String user;
@@ -107,6 +112,8 @@ public ElasticSearchClient(ModuleManager moduleManager,
String protocol,
String trustStorePath,
String trustStorePass,
+ String keyStorePath,
+ String keyStorePass,
String user,
String password,
Function indexNameConverter,
@@ -119,6 +126,8 @@ public ElasticSearchClient(ModuleManager moduleManager,
this.protocol = protocol;
this.trustStorePath = trustStorePath;
this.trustStorePass = trustStorePass;
+ this.keyStorePath = keyStorePath;
+ this.keyStorePass = keyStorePass;
this.user = user;
this.password = password;
this.indexNameConverter = indexNameConverter;
@@ -152,9 +161,17 @@ public void connect() {
if (!Strings.isNullOrEmpty(trustStorePath)) {
cb.trustStorePath(trustStorePath);
+ // Always set trustStorePass if trustStorePath is set (even if empty string)
+ if (trustStorePass != null) {
+ cb.trustStorePass(trustStorePass);
+ }
}
- if (!Strings.isNullOrEmpty(trustStorePass)) {
- cb.trustStorePass(trustStorePass);
+ if (!Strings.isNullOrEmpty(keyStorePath)) {
+ cb.keyStorePath(keyStorePath);
+ // Always set keyStorePass if keyStorePath is set (even if empty string)
+ if (keyStorePass != null) {
+ cb.keyStorePass(keyStorePass);
+ }
}
if (!Strings.isNullOrEmpty(user)) {
cb.username(user);
diff --git a/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/ElasticSearchIT.java b/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/ElasticSearchIT.java
index b7eefc669c59..1cf99af3bf69 100644
--- a/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/ElasticSearchIT.java
+++ b/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/ElasticSearchIT.java
@@ -112,7 +112,7 @@ public void indexOperate(final ElasticsearchContainer server,
final ElasticSearchClient client = new ElasticSearchClient(
moduleManager,
server.getHttpHostAddress(),
- "http", "", "", "test", "test",
+ "http", "", "", "", "", "test", "test",
indexNameConverter(namespace), 500, 6000,
0, 15
);
@@ -165,7 +165,7 @@ public void documentOperate(final ElasticsearchContainer server,
final ElasticSearchClient client = new ElasticSearchClient(
moduleManager,
server.getHttpHostAddress(),
- "http", "", "", "test", "test",
+ "http", "", "", "", "", "test", "test",
indexNameConverter(namespace), 500, 6000,
0, 15
);
@@ -241,7 +241,7 @@ public void templateOperate(final ElasticsearchContainer server,
final ElasticSearchClient client = new ElasticSearchClient(
moduleManager,
server.getHttpHostAddress(),
- "http", "", "", "test", "test",
+ "http", "", "", "", "", "test", "test",
indexNameConverter(namespace), 500, 6000,
0, 15
);
@@ -297,7 +297,7 @@ public void bulk(final ElasticsearchContainer server,
final ElasticSearchClient client = new ElasticSearchClient(
moduleManager,
server.getHttpHostAddress(),
- "http", "", "", "test", "test",
+ "http", "", "", "", "", "test", "test",
indexNameConverter(namespace), 500, 6000,
0, 15
);
@@ -331,7 +331,7 @@ public void bulkPer_1KB(final ElasticsearchContainer server,
final ElasticSearchClient client = new ElasticSearchClient(
moduleManager,
server.getHttpHostAddress(),
- "http", "", "", "test", "test",
+ "http", "", "", "", "", "test", "test",
indexNameConverter(namespace), 500, 6000,
0, 15
);
@@ -361,7 +361,7 @@ public void timeSeriesOperate(final ElasticsearchContainer server,
final ElasticSearchClient client = new ElasticSearchClient(
moduleManager,
server.getHttpHostAddress(),
- "http", "", "", "test", "test",
+ "http", "", "", "", "", "test", "test",
indexNameConverter(namespace), 500, 6000,
0, 15
);
diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchBuilder.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchBuilder.java
index b85b749216ff..e4ab1243d057 100644
--- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchBuilder.java
+++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchBuilder.java
@@ -27,15 +27,19 @@
import com.linecorp.armeria.client.logging.LoggingClient;
import com.linecorp.armeria.common.SessionProtocol;
import com.linecorp.armeria.common.auth.AuthToken;
+
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.KeyStore;
+import java.security.cert.X509Certificate;
import java.time.Duration;
import java.util.Arrays;
+import java.util.Enumeration;
import java.util.List;
import java.util.function.Consumer;
import java.util.stream.Collectors;
+import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.TrustManagerFactory;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@@ -46,164 +50,284 @@
@Slf4j
public final class ElasticSearchBuilder {
- private static final int NUM_PROC = Runtime.getRuntime().availableProcessors();
-
- private SessionProtocol protocol = SessionProtocol.HTTP;
-
- private String username;
-
- private String password;
+ private static final int NUM_PROC = Runtime.getRuntime().availableProcessors();
- private Duration healthCheckRetryInterval = Duration.ofSeconds(30);
+ private SessionProtocol protocol = SessionProtocol.HTTP;
- private final ImmutableList.Builder endpoints = ImmutableList.builder();
+ private String username;
- private String trustStorePath;
+ private String password;
- private String trustStorePass;
+ private Duration healthCheckRetryInterval = Duration.ofSeconds(30);
- private Duration responseTimeout = Duration.ofSeconds(15);
+ private final ImmutableList.Builder endpoints = ImmutableList.builder();
- private Duration connectTimeout = Duration.ofMillis(500);
+ private String trustStorePath;
- private Duration socketTimeout = Duration.ofSeconds(30);
+ private String trustStorePass;
- private Consumer healthyListener;
+ private String keyStorePath;
- private int numHttpClientThread;
-
- public ElasticSearchBuilder protocol(String protocol) {
- checkArgument(StringUtil.isNotBlank(protocol), "protocol cannot be blank");
- this.protocol = SessionProtocol.of(protocol);
- return this;
- }
-
- public ElasticSearchBuilder username(String username) {
- this.username = requireNonNull(username, "username");
- return this;
- }
+ private String keyStorePass;
- public ElasticSearchBuilder password(String password) {
- this.password = requireNonNull(password, "password");
- return this;
- }
-
- public ElasticSearchBuilder endpoints(Iterable endpoints) {
- requireNonNull(endpoints, "endpoints");
- this.endpoints.addAll(endpoints);
- return this;
- }
-
- public ElasticSearchBuilder endpoints(String... endpoints) {
- return endpoints(Arrays.asList(endpoints));
- }
-
- public ElasticSearchBuilder healthCheckRetryInterval(Duration healthCheckRetryInterval) {
- requireNonNull(healthCheckRetryInterval, "healthCheckRetryInterval");
- this.healthCheckRetryInterval = healthCheckRetryInterval;
- return this;
- }
+ private Duration responseTimeout = Duration.ofSeconds(15);
- public ElasticSearchBuilder trustStorePath(String trustStorePath) {
- requireNonNull(trustStorePath, "trustStorePath");
- this.trustStorePath = trustStorePath;
- return this;
- }
+ private Duration connectTimeout = Duration.ofMillis(500);
+
+ private Duration socketTimeout = Duration.ofSeconds(30);
+
+ private Consumer healthyListener;
+
+ private int numHttpClientThread;
+
+ public ElasticSearchBuilder protocol(String protocol) {
+ checkArgument(StringUtil.isNotBlank(protocol), "protocol cannot be blank");
+ this.protocol = SessionProtocol.of(protocol);
+ return this;
+ }
+
+ public ElasticSearchBuilder username(String username) {
+ this.username = requireNonNull(username, "username");
+ return this;
+ }
+
+ public ElasticSearchBuilder password(String password) {
+ this.password = requireNonNull(password, "password");
+ return this;
+ }
+
+ public ElasticSearchBuilder endpoints(Iterable endpoints) {
+ requireNonNull(endpoints, "endpoints");
+ this.endpoints.addAll(endpoints);
+ return this;
+ }
+
+ public ElasticSearchBuilder endpoints(String... endpoints) {
+ return endpoints(Arrays.asList(endpoints));
+ }
+
+ public ElasticSearchBuilder healthCheckRetryInterval(Duration healthCheckRetryInterval) {
+ requireNonNull(healthCheckRetryInterval, "healthCheckRetryInterval");
+ this.healthCheckRetryInterval = healthCheckRetryInterval;
+ return this;
+ }
+
+ public ElasticSearchBuilder trustStorePath(String trustStorePath) {
+ requireNonNull(trustStorePath, "trustStorePath");
+ this.trustStorePath = trustStorePath;
+ return this;
+ }
+
+ public ElasticSearchBuilder trustStorePass(String trustStorePass) {
+ requireNonNull(trustStorePass, "trustStorePass");
+ this.trustStorePass = trustStorePass;
+ return this;
+ }
+
+ public ElasticSearchBuilder keyStorePath(String keyStorePath) {
+ requireNonNull(keyStorePath, "keyStorePath");
+ this.keyStorePath = keyStorePath;
+ return this;
+ }
+
+ public ElasticSearchBuilder keyStorePass(String keyStorePass) {
+ requireNonNull(keyStorePass, "keyStorePass");
+ this.keyStorePass = keyStorePass;
+ return this;
+ }
+
+ public ElasticSearchBuilder connectTimeout(int connectTimeout) {
+ checkArgument(connectTimeout > 0, "connectTimeout must be positive");
+ this.connectTimeout = Duration.ofMillis(connectTimeout);
+ return this;
+ }
+
+ public ElasticSearchBuilder responseTimeout(int responseTimeout) {
+ checkArgument(responseTimeout >= 0, "responseTimeout must be 0 or positive");
+ this.responseTimeout = Duration.ofMillis(responseTimeout);
+ return this;
+ }
+
+ public ElasticSearchBuilder socketTimeout(int socketTimeout) {
+ checkArgument(socketTimeout > 0, "socketTimeout must be positive");
+ this.socketTimeout = Duration.ofMillis(socketTimeout);
+ return this;
+ }
+
+ public ElasticSearchBuilder healthyListener(Consumer healthyListener) {
+ requireNonNull(healthyListener, "healthyListener");
+ this.healthyListener = healthyListener;
+ return this;
+ }
+
+ public ElasticSearchBuilder numHttpClientThread(int numHttpClientThread) {
+ this.numHttpClientThread = numHttpClientThread;
+ return this;
+ }
+
+ @SneakyThrows
+ public ElasticSearch build() {
+ final List endpoints = this.endpoints.build().stream()
+ .filter(StringUtil::isNotBlank)
+ .map(Endpoint::parse)
+ .collect(Collectors.toList());
+ final ClientFactoryBuilder factoryBuilder = ClientFactory.builder()
+ .connectTimeout(connectTimeout)
+ .idleTimeout(socketTimeout)
+ .useHttp2Preface(false)
+ .workerGroup(numHttpClientThread > 0 ? numHttpClientThread : NUM_PROC);
+
+ // Configure SSL/TLS with optional mutual TLS (client certificate
+ // authentication)
+ final boolean hasTrustStore = StringUtil.isNotBlank(trustStorePath);
+ final boolean hasKeyStore = StringUtil.isNotBlank(keyStorePath);
+
+ if (hasTrustStore || hasKeyStore) {
+ factoryBuilder.tlsCustomizer(sslContextBuilder -> {
+ try {
+ // Configure trust store for server certificate validation
+ if (hasTrustStore) {
+ log.info("Loading truststore from: {}", trustStorePath);
+ final var trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ final var truststoreType = detectKeystoreType(trustStorePath);
+ log.info("Detected truststore type: {} for file: {}", truststoreType, trustStorePath);
+ final var truststore = KeyStore.getInstance(truststoreType);
+ try (final InputStream is = Files.newInputStream(Paths.get(trustStorePath))) {
+ truststore.load(is, trustStorePass != null ? trustStorePass.toCharArray() : null);
+ log.info("Successfully loaded truststore from: {}", trustStorePath);
+ }
+ trustManagerFactory.init(truststore);
+ sslContextBuilder.trustManager(trustManagerFactory);
+ log.info("Truststore configured successfully");
+ }
+
+ // Configure key store for client certificate authentication (mutual TLS)
+ if (hasKeyStore) {
+ log.info("Loading keystore from: {}", keyStorePath);
+ final var keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+
+ // Detect keystore type from file extension or try both PKCS12 and JKS
+ final var keystoreType = detectKeystoreType(keyStorePath);
+ log.info("Detected keystore type: {} for file: {}", keystoreType, keyStorePath);
+ final var keystore = KeyStore.getInstance(keystoreType);
+
+ try (final var is = Files.newInputStream(Paths.get(keyStorePath))) {
+ keystore.load(is, keyStorePass != null ? keyStorePass.toCharArray() : null);
+ log.info("Successfully loaded keystore from: {}", keyStorePath);
+ }
- public ElasticSearchBuilder trustStorePass(String trustStorePass) {
- requireNonNull(trustStorePass, "trustStorePass");
- this.trustStorePass = trustStorePass;
- return this;
- }
+ // Log certificate details for troubleshooting
+ logCertificateDetails(keystore, keyStorePath);
- public ElasticSearchBuilder connectTimeout(int connectTimeout) {
- checkArgument(connectTimeout > 0, "connectTimeout must be positive");
- this.connectTimeout = Duration.ofMillis(connectTimeout);
- return this;
- }
+ keyManagerFactory.init(keystore, keyStorePass != null ? keyStorePass.toCharArray() : null);
+ sslContextBuilder.keyManager(keyManagerFactory);
- public ElasticSearchBuilder responseTimeout(int responseTimeout) {
- checkArgument(responseTimeout >= 0, "responseTimeout must be 0 or positive");
- this.responseTimeout = Duration.ofMillis(responseTimeout);
- return this;
+ log.info("Client certificate authentication enabled with keystore: {} (type: {})",
+ keyStorePath, keystoreType);
+ }
+ } catch (Exception e) {
+ log.error("Failed to configure SSL/TLS context", e);
+ throw new RuntimeException("Failed to configure SSL/TLS context: " + e.getMessage(), e);
+ }
+ });
}
- public ElasticSearchBuilder socketTimeout(int socketTimeout) {
- checkArgument(socketTimeout > 0, "socketTimeout must be positive");
- this.socketTimeout = Duration.ofMillis(socketTimeout);
- return this;
+ final ClientFactory clientFactory = factoryBuilder.build();
+
+ final HealthCheckedEndpointGroupBuilder endpointGroupBuilder = HealthCheckedEndpointGroup
+ .builder(EndpointGroup.of(endpoints), "_cluster/health")
+ .protocol(protocol)
+ .useGet(true)
+ .clientFactory(clientFactory)
+ .retryInterval(healthCheckRetryInterval)
+ .withClientOptions(options -> {
+ options.decorator(
+ LoggingClient.builder()
+ .logger(log)
+ .newDecorator());
+ options.decorator((delegate, ctx, req) -> {
+ ctx.logBuilder().name("health-check");
+ return delegate.execute(ctx, req);
+ });
+ return options;
+ });
+ if (StringUtil.isNotBlank(username) && StringUtil.isNotBlank(password)) {
+ endpointGroupBuilder.auth(AuthToken.ofBasic(username, password));
}
-
- public ElasticSearchBuilder healthyListener(Consumer healthyListener) {
- requireNonNull(healthyListener, "healthyListener");
- this.healthyListener = healthyListener;
- return this;
+ final HealthCheckedEndpointGroup endpointGroup = endpointGroupBuilder.build();
+
+ return new ElasticSearch(
+ protocol,
+ username,
+ password,
+ endpointGroup,
+ clientFactory,
+ healthyListener,
+ responseTimeout);
+ }
+
+ /**
+ * Detects keystore type from file extension.
+ * Defaults to PKCS12 as it's the modern standard and recommended format.
+ */
+ private static String detectKeystoreType(String keyStorePath) {
+ if (keyStorePath == null) {
+ return "PKCS12";
}
- public ElasticSearchBuilder numHttpClientThread(int numHttpClientThread) {
- this.numHttpClientThread = numHttpClientThread;
- return this;
+ String lowerPath = keyStorePath.toLowerCase();
+ if (lowerPath.endsWith(".jks")) {
+ return "JKS";
+ } else if (lowerPath.endsWith(".p12") || lowerPath.endsWith(".pfx")) {
+ return "PKCS12";
}
- @SneakyThrows
- public ElasticSearch build() {
- final List endpoints =
- this.endpoints.build().stream()
- .filter(StringUtil::isNotBlank)
- .map(Endpoint::parse)
- .collect(Collectors.toList());
- final ClientFactoryBuilder factoryBuilder =
- ClientFactory.builder()
- .connectTimeout(connectTimeout)
- .idleTimeout(socketTimeout)
- .useHttp2Preface(false)
- .workerGroup(numHttpClientThread > 0 ? numHttpClientThread : NUM_PROC);
-
- if (StringUtil.isNotBlank(trustStorePath)) {
- final TrustManagerFactory trustManagerFactory =
- TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
- final KeyStore truststore = KeyStore.getInstance("jks");
- try (final InputStream is = Files.newInputStream(Paths.get(trustStorePath))) {
- truststore.load(is, trustStorePass.toCharArray());
- }
- trustManagerFactory.init(truststore);
-
- factoryBuilder.tlsCustomizer(
- sslContextBuilder -> sslContextBuilder.trustManager(trustManagerFactory));
- }
-
- final ClientFactory clientFactory = factoryBuilder.build();
-
- final HealthCheckedEndpointGroupBuilder endpointGroupBuilder =
- HealthCheckedEndpointGroup.builder(EndpointGroup.of(endpoints), "_cluster/health")
- .protocol(protocol)
- .useGet(true)
- .clientFactory(clientFactory)
- .retryInterval(healthCheckRetryInterval)
- .withClientOptions(options -> {
- options.decorator(
- LoggingClient.builder()
- .logger(log)
- .newDecorator());
- options.decorator((delegate, ctx, req) -> {
- ctx.logBuilder().name("health-check");
- return delegate.execute(ctx, req);
- });
- return options;
- });
- if (StringUtil.isNotBlank(username) && StringUtil.isNotBlank(password)) {
- endpointGroupBuilder.auth(AuthToken.ofBasic(username, password));
+ // Default to PKCS12 for unknown extensions
+ log.info("Unknown keystore extension for {}, defaulting to PKCS12 format", keyStorePath);
+ return "PKCS12";
+ }
+
+ /**
+ * Logs certificate details from the keystore for troubleshooting purposes.
+ */
+ @SneakyThrows
+ private static void logCertificateDetails(KeyStore keystore, String keyStorePath) {
+ try {
+ final Enumeration aliases = keystore.aliases();
+ int certCount = 0;
+ while (aliases.hasMoreElements()) {
+ final String alias = aliases.nextElement();
+ certCount++;
+ log.info("Keystore entry [{}]: alias='{}', isKeyEntry={}, isCertificateEntry={}",
+ certCount, alias, keystore.isKeyEntry(alias), keystore.isCertificateEntry(alias));
+
+ if (keystore.isKeyEntry(alias)) {
+ final java.security.cert.Certificate cert = keystore.getCertificate(alias);
+ if (cert instanceof X509Certificate) {
+ final X509Certificate x509Cert = (X509Certificate) cert;
+ log.info(" Certificate subject: {}", x509Cert.getSubjectDN());
+ log.info(" Certificate issuer: {}", x509Cert.getIssuerDN());
+ log.info(" Certificate serial number: {}", x509Cert.getSerialNumber());
+ log.info(" Certificate valid from: {} to {}",
+ x509Cert.getNotBefore(), x509Cert.getNotAfter());
+ log.info(" Certificate algorithm: {}", x509Cert.getSigAlgName());
+ }
+ } else if (keystore.isCertificateEntry(alias)) {
+ final java.security.cert.Certificate cert = keystore.getCertificate(alias);
+ if (cert instanceof X509Certificate) {
+ final X509Certificate x509Cert = (X509Certificate) cert;
+ log.info(" Certificate subject: {}", x509Cert.getSubjectDN());
+ log.info(" Certificate issuer: {}", x509Cert.getIssuerDN());
+ }
}
- final HealthCheckedEndpointGroup endpointGroup = endpointGroupBuilder.build();
-
- return new ElasticSearch(
- protocol,
- username,
- password,
- endpointGroup,
- clientFactory,
- healthyListener,
- responseTimeout
- );
+ }
+ if (certCount == 0) {
+ log.warn("No certificates found in keystore: {}", keyStorePath);
+ } else {
+ log.info("Total entries in keystore: {}", certCount);
+ }
+ } catch (Exception e) {
+ log.warn("Failed to log certificate details from keystore: {}", keyStorePath, e);
}
+ }
}
diff --git a/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ElasticSearchIT.java b/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ElasticSearchIT.java
index bc1130832d1a..295a9eb81a0b 100644
--- a/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ElasticSearchIT.java
+++ b/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ElasticSearchIT.java
@@ -422,4 +422,25 @@ public void testDocDeleteById(final String ignored,
assertFalse(client.documents().exists(index, type, idWithSpace));
server.close();
}
+
+ @ParameterizedTest(name = "version: {0}")
+ @MethodSource("es")
+ public void testClientBuilder(final String ignored,
+ final ElasticsearchContainer server) {
+ server.start();
+
+ // Test basic builder functionality
+ final ElasticSearch client =
+ ElasticSearch.builder()
+ .endpoints(server.getHttpHostAddress())
+ .build();
+ client.connect();
+
+ final String index = "test-builder-index";
+ assertTrue(client.index().create(index, null, null));
+ assertTrue(client.index().exists(index));
+ assertTrue(client.index().delete(index));
+
+ server.close();
+ }
}
diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml
index fd6356f20374..f62259e19a82 100644
--- a/oap-server/server-starter/src/main/resources/application.yml
+++ b/oap-server/server-starter/src/main/resources/application.yml
@@ -158,8 +158,10 @@ storage:
numHttpClientThread: ${SW_STORAGE_ES_NUM_HTTP_CLIENT_THREAD:0}
user: ${SW_ES_USER:""}
password: ${SW_ES_PASSWORD:""}
- trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""}
- trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""}
+ trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""} # Path to the truststore that contains CA certificates used to verify the OpenSearch/Elasticsearch server certificate.
+ trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""} # Password for the truststore defined in trustStorePath.
+ keyStorePath: ${SW_STORAGE_ES_SSL_KEY_STORE_PATH:""} # Path to the client certificate keystore for mutual TLS (OpenSearch/Elasticsearch client certificate authentication).
+ keyStorePass: ${SW_STORAGE_ES_SSL_KEY_STORE_PASS:""} # Password for the client certificate keystore defined in keyStorePath. This can be externalized via secretsManagementFile.
secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool.
dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index.
indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java
index d8dc848be5c3..606e131ba707 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java
@@ -113,6 +113,19 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig {
* @since 7.0.0 This could be managed inside {@link #secretsManagementFile}
*/
private String trustStorePass;
+ /**
+ * Path to the keystore file for client certificate authentication (OpenSearch/Elasticsearch mTLS).
+ * This enables mutual TLS where the client presents a certificate to the server.
+ * Supports PKCS12 (.p12, .pfx) and JKS (.jks) formats.
+ * When set, {@link #keyStorePass} must also be provided.
+ */
+ private String keyStorePath;
+ /**
+ * Password for the keystore file specified in {@link #keyStorePath}.
+ * Required when {@link #keyStorePath} is set.
+ * This could be managed inside {@link #secretsManagementFile}
+ */
+ private String keyStorePass;
private int resultWindowMaxSize = 10000;
private int metadataQueryMaxSize = 5000;
/**
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java
index 3cfd5e014c6b..aad41665f6a3 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java
@@ -179,6 +179,7 @@ public void prepare() throws ServiceNotProvidedException {
config.setUser(secrets.getProperty("user", null));
config.setPassword(secrets.getProperty("password", null));
config.setTrustStorePass(secrets.getProperty("trustStorePass", null));
+ config.setKeyStorePass(secrets.getProperty("keyStorePass", null));
if (elasticSearchClient == null) {
// In the startup process, we just need to change the username/password
@@ -187,20 +188,27 @@ public void prepare() throws ServiceNotProvidedException {
elasticSearchClient.setUser(config.getUser());
elasticSearchClient.setPassword(config.getPassword());
elasticSearchClient.setTrustStorePass(config.getTrustStorePass());
+ elasticSearchClient.setKeyStorePass(config.getKeyStorePass());
elasticSearchClient.connect();
}
- }, config.getSecretsManagementFile(), config.getTrustStorePath());
+ }, config.getSecretsManagementFile(), config.getTrustStorePath(), config.getKeyStorePath());
/*
* By leveraging the sync update check feature when startup.
*/
monitor.start();
}
+ // Validate keystore configuration
+ validateKeystoreConfiguration(config);
+
elasticSearchClient = new ElasticSearchClient(
getManager(),
- config.getClusterNodes(), config.getProtocol(), config.getTrustStorePath(), config
- .getTrustStorePass(), config.getUser(), config.getPassword(),
- indexNameConverter(config.getNamespace()), config.getConnectTimeout(),
+ config.getClusterNodes(), config.getProtocol(),
+ config.getTrustStorePath(), config.getTrustStorePass(),
+ config.getKeyStorePath(), config.getKeyStorePass(),
+ config.getUser(), config.getPassword(),
+ indexNameConverter(config.getNamespace()),
+ config.getConnectTimeout(),
config.getSocketTimeout(), config.getResponseTimeout(),
config.getNumHttpClientThread()
);
@@ -346,4 +354,29 @@ public static Function indexNameConverter(String namespace) {
return indexName;
};
}
+
+ /**
+ * Validates keystore configuration to ensure mutual TLS setup is complete.
+ * If keyStorePath is provided, keyStorePass must also be provided (can be empty string for no password).
+ */
+ private void validateKeystoreConfiguration(StorageModuleElasticsearchConfig config) {
+ final boolean hasKeyStorePath = !StringUtil.isEmpty(config.getKeyStorePath());
+ final boolean hasKeyStorePass = config.getKeyStorePass() != null;
+
+ if (hasKeyStorePath && !hasKeyStorePass) {
+ throw new IllegalArgumentException(
+ "keyStorePass must be provided when keyStorePath is configured for client certificate authentication. " +
+ "Use empty string (\"\") for keystores without password."
+ );
+ }
+
+ if (!hasKeyStorePath && hasKeyStorePass) {
+ log.warn("keyStorePass is configured but keyStorePath is not set. The keyStorePass will be ignored.");
+ }
+
+ if (hasKeyStorePath) {
+ log.info("Client certificate authentication (mutual TLS) is enabled with keystore: {}",
+ config.getKeyStorePath());
+ }
+ }
}
diff --git a/test/e2e-v2/cases/storage/opensearch/certs/admin-key.pem b/test/e2e-v2/cases/storage/opensearch/certs/admin-key.pem
new file mode 100644
index 000000000000..c57777dedb95
--- /dev/null
+++ b/test/e2e-v2/cases/storage/opensearch/certs/admin-key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC+b8X+z9oYTUsT
+UfsKWr+dtGKHVEStllH9QmcTGZOkEsm3ERY+78WZgNhN/7mV02OeU9T7/ooQim1i
+dN0VgF7bm1iFDKilqRTlJqYZZF8xcgPr++x2vDlLANG+Qce58RoWdzDpKO1u96XT
+Z8nJpQWtSBfYBmxjNiQwiUueuXKT1grVEijtLNC/Fpa4I2SJUjI62/pi8XyQiPHL
+nW1PbJdkb7y+DcKomJAarsGd1ar+tVR6CZUpLztYLKW3oPwCNeC3TV63KV4f+/9D
+Hn6ENAbaVhiR/2YdrvU+XcZJ3apuSIBy4mKfSIH16I9iOp8Kuteo39M2f577JcoP
+bWaMi44DAgMBAAECggEALtKJknSlcXszncz21d1pJovO++oWtr1ybDwS3snXmKw7
+52RunUvTwEHDLS5WgYWHhUqkX87+QEHg0ifcoeg9qm4gDhqGLrELX6ooha69jwky
++KcoxSrTRWMursI6qreii+qDXph/BF0kav2mSgtmgWvr3OP7a0tJC5v+OUjsaHd3
+/4S1FFPXdza/0XAoBwMuMvG5arNooGttdgGknsPCJjEliPKYnLBZakFGCgZLZltX
+HGJpD9lvPp1XOCMpRrcF6spcW2/t70SR8zGj3VUckxaNKVt06i8Lqa4Up6dKhF8x
+rh3TMUol41B7lEHdmZQ3tBoXyuxJkBYr7M9egAFiXQKBgQDd0P/0MnYVVOUAL4cv
+2l2ywWxjlu421sZZ9Hf4sSrD65tJHJS4SqeUj2OVIbrCZ0G/lFBEva/3NYcZD4R2
+jyIQnWJOjemJAhTe6x4NKnJHqWXaZU61GGdTnLwHxcc9/sZSLd+wunOwzlCviPbR
+ReMgJhWJ9/XcFIY7slcB0KpkrQKBgQDbyMrWsycXIZ36hntPytuZZHtU/CdxSpR3
+XCEZtF/KdvVbzmZnW9Y+HDj4XqUOykuDrg3wTjmPI3SgjEVURbjraBJFSJUeF4w/
+XhG6K6Tx1Cmdqqih3q6nrkxOD9tiN6tT+R/wQfUOtenrCZhLcj9aZ//eIcHLyrSf
+WvpMwQdjbwKBgAxakYbGMLFrcv2ZqAvQO5uzDhhV1ZqUR6PG68+b/me+/X0K7HV/
+IuoxOjiaEk61dYH3/qh1cBFyl72bkaMQwbLvMQRy/ui0hvkLWzccgBThqFyLe+C2
+JTsQ5aABMeGQCPeWunibSco1E2VTWXu6SrYFqPlwJ+9D7V3xxsrBFlxZAoGAAh9x
+XhuC4CVR+k58OGwULOocitiYpO58ep6oLzBf0HvPqOBYet0XN6hcIIIBhCAOFKqE
+tfJ7edd00+wm60Z1H8j0jDjEP/MoRqBo+Wxcfn13HW+9izq0Yyg60nIyw0MYY4o/
+dbmdXVQCe2OvVeM3m27vuLyIu6gskHF3g3BF2v8CgYEAiNC7qcySASSWHDhSmGSv
+Xx/AXP1jGzrqb3BT0+0Dexxc+lumTqVCFWFcJmAgazgMThE4VLYZ1pK5KS9FwZHX
+HVSQluPNTy25DJ+pSXnNMtOiG8g+C61H7G1yQ+dS1QwH8f8/iVFEZ1dgVIxv5nV4
+Vxm2zOKeAF23sfOBkmYb15Y=
+-----END PRIVATE KEY-----
diff --git a/test/e2e-v2/cases/storage/opensearch/certs/admin.pem b/test/e2e-v2/cases/storage/opensearch/certs/admin.pem
new file mode 100644
index 000000000000..c438f30db908
--- /dev/null
+++ b/test/e2e-v2/cases/storage/opensearch/certs/admin.pem
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDTjCCAjYCFE5mcJsLUgt96WKdeQWD/6Pw52bAMA0GCSqGSIb3DQEBCwUAMGox
+CzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTENMAsGA1UEBwwEVGVzdDETMBEGA1UE
+CgwKU2t5V2Fsa2luZzENMAsGA1UECwwEVGVzdDEbMBkGA1UEAwwSU2t5V2Fsa2lu
+ZyBSb290IENBMB4XDTI2MDEwNDEyMjY0NloXDTI4MDEwNDEyMjY0NlowXTELMAkG
+A1UEBhMCVVMxCzAJBgNVBAgMAkNBMQ0wCwYDVQQHDARUZXN0MRMwEQYDVQQKDApT
+a3lXYWxraW5nMQ0wCwYDVQQLDARUZXN0MQ4wDAYDVQQDDAVhZG1pbjCCASIwDQYJ
+KoZIhvcNAQEBBQADggEPADCCAQoCggEBAL5vxf7P2hhNSxNR+wpav520YodURK2W
+Uf1CZxMZk6QSybcRFj7vxZmA2E3/uZXTY55T1Pv+ihCKbWJ03RWAXtubWIUMqKWp
+FOUmphlkXzFyA+v77Ha8OUsA0b5Bx7nxGhZ3MOko7W73pdNnycmlBa1IF9gGbGM2
+JDCJS565cpPWCtUSKO0s0L8WlrgjZIlSMjrb+mLxfJCI8cudbU9sl2RvvL4NwqiY
+kBquwZ3Vqv61VHoJlSkvO1gspbeg/AI14LdNXrcpXh/7/0MefoQ0BtpWGJH/Zh2u
+9T5dxkndqm5IgHLiYp9IgfXoj2I6nwq616jf0zZ/nvslyg9tZoyLjgMCAwEAATAN
+BgkqhkiG9w0BAQsFAAOCAQEAaukSpVdgSv5qtfcTH5spKH7TLClEshJukHXhnWAR
+ElMfua1/4+eP0KlW778N8uDsFHqjfCnXHnrzvgju0uXs+nl4eqUwloJ3WbwHnt+b
+6ArseyGF8ybn6Oc0IiYX3cCoxr0DxFh6Ft5I9H0oS5rohjp6aIpFd+ZUXRMFchgf
++NiZtPgaGvQk94WoHIoYeF44E8VQTdFu80mBWj0PhUGe4dXy1lnoUUBUcGOG0f3I
+QXHH2PP7nALBSEyluK23hUObmE8wqmLWl/M7se5kDgCndl/6zD8z9KFfUbWMaKzz
+n3JDGh+S92i4lHIxp4JJhru0zh/ZmOhbCyPiF9i5gyREyA==
+-----END CERTIFICATE-----
diff --git a/test/e2e-v2/cases/storage/opensearch/certs/client-key.pem b/test/e2e-v2/cases/storage/opensearch/certs/client-key.pem
new file mode 100644
index 000000000000..718d67cc39b7
--- /dev/null
+++ b/test/e2e-v2/cases/storage/opensearch/certs/client-key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC5sz1zVe77Wk+L
+hJyoN4RbCYdThbLIse//qyS9V+Kid1RavHlic5xlWBJRs0t9xmDWsI9pxybKz0Cw
+NuuKx/yk2zZv2dChMf6NY+K8AsHj8lSVrIqeQHJq9oq94IWwanwcibCkuJLad+pU
+q6bFHqp629eopm8iMAvbeqKTVlMVKcg6VEhiiCm5pnh5OdiDcYRd92weLTpNUnw+
+exnYhbcjcO6sLUqEbf7cEFga4rwtsGDE9toKoL9aMhvhOGKE179fX07Y0hqNPJjG
+i43KhOeU5ND5radsMoi0yctp8Mpkx32xFSFwh9zLKSmN8QqIPJsIKrl+Rn7b0P7K
+Z4QT6UTxAgMBAAECggEAAqHTKKQZuR4ibHl+6jA+rGQT58dMJDEELono1/LbtB2P
+NZqvU3q54awIp/gCX8IWtqAUEubT2Ke/tZVaaBMiKQqv/FQkpEqgojUnG2sVO/i0
+uIsJPEAlodrv5hABKGWA0eElGRszK6zVL/MUQthVxKdr+248JHew51Wxf9lWSXYz
+5I81Oc2iZPY0AyT0pmMqQ6PxUS7xWfUx/vkXujvNEzn48qij7T9nPUlrE6olZgy2
+APzBMI0ws+KrUCv4jAfAaedtMbco4TxOt4LMKaMgXmiNlrKicCnIo4XY6vQuJCop
+BPV3aPscj7zDVwCEtQKpitUNEEjoMr3QYH02TSOtJQKBgQDgN8x06nRle061GucV
+GYZj1Ax/IohNRa0tuwr7WJn1Mky691r+jaIUfjRugFEsaFJmkR9UbZbAC/69RrSn
+8i2nhhHCRi6TUFxKQ4CRm/0BlvwXuH7SowNvTWgHvPeax4ztqFG3xMmtWs2RV6Dm
+zLcH1jVulKvtfFEvsPvMwb5uVQKBgQDUBb65Fi1RP6hLXS5fp2zh+jT5GTCq0bLp
+j8tv6iPBI9k02XqE3PNLqlPSgz+aBORhUIyZPuT1Qx1/tbDmuBxPntNETjpl5GwF
+qTxyFQIOAdZjy0eWpOIWM+lo9xEtJv3oN7TIMQHueXS4edrNsmYIvAel2lIjRH8Z
+fCt6G9BgLQKBgBBP/vACnrVDY1aJvoYqdTyOENqvCHuWtiK9mO7wY0MThcGUfWpH
+o6MaC3Z+n2k7rcMIi974mh8ewEnE+x+83tVxS5l2way2DADbKF9vmdijw3N2WMO6
+WGWgnBD0Do+UNQyVUlysVH/oO0x3s50XB7nqO7jv2BJPGRj/J1KeRdyBAoGBAKWG
+BqvAoIh5xg1wJbAPqXWSPKDsBY6WP7MPy6cHh/pU3lHgJ0JqrJY5107VoGXBw/ol
+RF6vN1gymWkGk6DLw251dEIzQGwjtCGHSeVWeVAuJw2pua3l84uZ43NKz2IMutT8
+CGrxt6xRrcoHd8Z2rCnNgbr9gnp+Eyv2QIsIA9nRAoGBAIVAyGg2Q6VNZzErWB21
++Cue68YF46VEiTXKhzOBawj1oeu2YAEFUO4jQQk1Z/Lng8UJF7hd7v8ALEcEKmS5
+Stmyq6wLAHqkQdF7o4r8c5ILSRo6hjCFXJ7+Zl+WbBuKrpgQd11eUXilUymh/cRM
+IluIOndu6l2Fo5ZEaW5EvKJb
+-----END PRIVATE KEY-----
diff --git a/test/e2e-v2/cases/storage/opensearch/certs/client.p12 b/test/e2e-v2/cases/storage/opensearch/certs/client.p12
new file mode 100644
index 000000000000..0b1802049dcd
Binary files /dev/null and b/test/e2e-v2/cases/storage/opensearch/certs/client.p12 differ
diff --git a/test/e2e-v2/cases/storage/opensearch/certs/client.pem b/test/e2e-v2/cases/storage/opensearch/certs/client.pem
new file mode 100644
index 000000000000..08078f74e5db
--- /dev/null
+++ b/test/e2e-v2/cases/storage/opensearch/certs/client.pem
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDWzCCAkMCFE5mcJsLUgt96WKdeQWD/6Pw52bBMA0GCSqGSIb3DQEBCwUAMGox
+CzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTENMAsGA1UEBwwEVGVzdDETMBEGA1UE
+CgwKU2t5V2Fsa2luZzENMAsGA1UECwwEVGVzdDEbMBkGA1UEAwwSU2t5V2Fsa2lu
+ZyBSb290IENBMB4XDTI2MDEwNDEyMjY0NloXDTI4MDEwNDEyMjY0NlowajELMAkG
+A1UEBhMCVVMxCzAJBgNVBAgMAkNBMQ0wCwYDVQQHDARUZXN0MRMwEQYDVQQKDApT
+a3lXYWxraW5nMQ0wCwYDVQQLDARUZXN0MRswGQYDVQQDDBJub2RlLTAuZXhhbXBs
+ZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5sz1zVe77Wk+L
+hJyoN4RbCYdThbLIse//qyS9V+Kid1RavHlic5xlWBJRs0t9xmDWsI9pxybKz0Cw
+NuuKx/yk2zZv2dChMf6NY+K8AsHj8lSVrIqeQHJq9oq94IWwanwcibCkuJLad+pU
+q6bFHqp629eopm8iMAvbeqKTVlMVKcg6VEhiiCm5pnh5OdiDcYRd92weLTpNUnw+
+exnYhbcjcO6sLUqEbf7cEFga4rwtsGDE9toKoL9aMhvhOGKE179fX07Y0hqNPJjG
+i43KhOeU5ND5radsMoi0yctp8Mpkx32xFSFwh9zLKSmN8QqIPJsIKrl+Rn7b0P7K
+Z4QT6UTxAgMBAAEwDQYJKoZIhvcNAQELBQADggEBALlyJEE/xNkkCUzehlSYSs+o
++/BK5sC56sUE2y6Ar7bYhaeHpbh4SFUeow16+9cYgqLvz91PRQU/Iub+F31Bg9Vw
+lnaAC/pT+14CUAyXbtSklWBtlqGqWQjZxVgjOrop96Ygnte5d90ppD2+mxcju1Lc
+8iKBISR3mIOY9DVNj/G89FuRytX1vJ7kMvjVrbRcG6Du997a/NnMzolWWu4YERg9
+xHuSfAaXWVGmL7iMT3Czz/Jqv2nleb15OO0J2+XJkRZqA266xyG5T4vIew+YV7qM
+wqK2KasrciVttQKHhiJMpy0PmjrsxkyIe6nAR0T3mMB9OxPkIBNbqvNMRjsJy2I=
+-----END CERTIFICATE-----
diff --git a/test/e2e-v2/cases/storage/opensearch/certs/node-key.pem b/test/e2e-v2/cases/storage/opensearch/certs/node-key.pem
new file mode 100644
index 000000000000..575d28a62427
--- /dev/null
+++ b/test/e2e-v2/cases/storage/opensearch/certs/node-key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCVvg/acsfUtigU
+sxqUNBS/KV/4adtEYEB9V4mt4G+Q5N7PHAoyHakyspjDHZb+IHWYnRl4zOhIcMFZ
+9MOC/NQw//nYbRs5oB3YlZbZLfC4MCS6jKszkpA1LPuMvcCGz271SScu8/DXLiLV
+GbwXseJC4w9Qxbi7RMXRkCGhDcmm+Eg0JpaArYZUsPLiRRhP95LjDhtacqoPozO2
+/0y2ZvLu1vQyNpQQmjuL0BxHmQeXjFe4AEJ8ZLgbTPg2Cyu8XFJKf1bsJcH4xowc
+8nAugOTR3CV1aYvwUg+a+M9GOZPD+jdsac/kRfYJDBPf3ydKx7mPHAJh70GftG/y
+bMoD0NZRAgMBAAECggEACejPAcTQxpnbYytLGf2sVc42JKlUslywR9swKEqO94a9
+2/TeE9z0on9BLsBZiq25OoVC90SQJqMXmI1jFtsGa8u0zAbEY3beIsQbfHnW0UmW
+VKYUelA7rNkyOGkiQYmerSrPJgNMr+DEu1d9pA/IimaeT9kV1YbClJC2OQNBkgQI
+TfDV1FqO+6HyE//5WrG32wgEB/YDu80WSf13eIE9qrFqzzb6WXCYP4gT7WASvZRH
+lnCU08EFJKT6jVMmrizTO+tCqT2A9G0R/ASYylYS5W73+8rLJBd1g8uuHG9PGeB0
+NWvRxn6I3i/gHaJ2ltA5Fhy09zi8lEfwKiu5HxUUZQKBgQDKFAlrcv56XtNhIzup
+jah4tEdLOPwehIKJ+1qEQu0QXglWFivShj+WhrM6wy1VOgWgcCLsx9ZuQigUFFKM
+D+NsIaz277XSKbqE46KZJy+IzgSTzWUFlgDqzpfHqgW9XpiJzxOqd9h5QmXbW9Zc
+N75WTgFRfp1/Ig2lCD5uP7aBjwKBgQC9svb9xMM2zX7KAI3ADwovcnz3ca2TheKA
+li0MkeRcDAvrojkPE8WtzrrPXoZ892iLFgktEy9hEMyiW5XMKSqa6kh0JM1OUKsz
+In4/GNtZzRKwe4ZFIoBFP4d2OE3WNT3nHT8OFuEZkWwPsrDFVRlUtsHZzEHkqpl8
+ObJxjvd6HwKBgQCyrAiwIpry48kOSDLGdeQR5YRr9FSnPw6UpdOgwfQN1rd2kF/q
+4pxyoWLzgAMjKgwzkTKwHPlxv7jkGBvsj1fMEfJ22/ftfMvYF9V6iPU0hsPxU1gR
+GlJxSn1VIvW0PGGu55NB1HlordaVn5vnKbp3YL01qzfiYt+hnaplnJvn7QKBgFvj
+zyUKJQ3s6RfswL1iC6sEKGislko5tohXNqc6HIZCB5wyzrTw/Pa+h1tgDIGITwng
+uL0u5+p6+sVC3AMzhcHY7xPjp9fh16xDbygdYFPVtNHsZBQlLEFfDr1DdODolX3Y
+euzWRF/gQ5ovEtXj7QtOJATenqSnxwWX5UqA2Hw9AoGARRQ/RoxCrjtPOInC85Zg
+rl1qr0WmBwws8CarOI3glOnUHrZ6oH/fmWCMzNg+i5Z35BMqBY/w6LsiefYc+eGm
+sbj3lulzN+duR6K6HSHX6ipsokS5yVqPdYlfUfrqDHzTJpwHmypneZxJPubEm8z9
+Xy4Mfwseqj+aW5hmlzk9eFs=
+-----END PRIVATE KEY-----
diff --git a/test/e2e-v2/cases/storage/opensearch/certs/node.pem b/test/e2e-v2/cases/storage/opensearch/certs/node.pem
new file mode 100644
index 000000000000..d364ee00c239
--- /dev/null
+++ b/test/e2e-v2/cases/storage/opensearch/certs/node.pem
@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIIDxDCCAqygAwIBAgIUTmZwmwtSC33pYp15BYP/o/DnZr8wDQYJKoZIhvcNAQEL
+BQAwajELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQ0wCwYDVQQHDARUZXN0MRMw
+EQYDVQQKDApTa3lXYWxraW5nMQ0wCwYDVQQLDARUZXN0MRswGQYDVQQDDBJTa3lX
+YWxraW5nIFJvb3QgQ0EwHhcNMjYwMTA0MTIyNjQ2WhcNMjgwMTA0MTIyNjQ2WjBi
+MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExDTALBgNVBAcMBFRlc3QxEzARBgNV
+BAoMClNreVdhbGtpbmcxDTALBgNVBAsMBFRlc3QxEzARBgNVBAMMCm9wZW5zZWFy
+Y2gwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVvg/acsfUtigUsxqU
+NBS/KV/4adtEYEB9V4mt4G+Q5N7PHAoyHakyspjDHZb+IHWYnRl4zOhIcMFZ9MOC
+/NQw//nYbRs5oB3YlZbZLfC4MCS6jKszkpA1LPuMvcCGz271SScu8/DXLiLVGbwX
+seJC4w9Qxbi7RMXRkCGhDcmm+Eg0JpaArYZUsPLiRRhP95LjDhtacqoPozO2/0y2
+ZvLu1vQyNpQQmjuL0BxHmQeXjFe4AEJ8ZLgbTPg2Cyu8XFJKf1bsJcH4xowc8nAu
+gOTR3CV1aYvwUg+a+M9GOZPD+jdsac/kRfYJDBPf3ydKx7mPHAJh70GftG/ybMoD
+0NZRAgMBAAGjajBoMCYGA1UdEQQfMB2CCm9wZW5zZWFyY2iCCWxvY2FsaG9zdIcE
+fwAAATAdBgNVHQ4EFgQUWBLZsHgz29wzfO/jeRawe2mhx0EwHwYDVR0jBBgwFoAU
+gOjBnhpAdisZ/mzDyMgw5XCuNvcwDQYJKoZIhvcNAQELBQADggEBAMpdZPWisHxn
+3tMaLxH0kbiK+Kftws8EKRd8IYATK/NVvDRZyiKFgUNJEvADAzKTHqq5S8QTB6/U
+6s0Uu2kqkRUQdFuMkswEFqnP/dcYohJtT6Sa3cbOl32GDvoGQOGPd86Hj7SyeoIC
+MG1wmfOzv/COXuynecXUBDSMFRQ3kbClEHH2ELNMaYiWmU13tKA01FA7sWNOyiNR
+hJ72WSb2SkuF5GsEJjpo7EGtwerZHMDYDWoQfWgVaM1jSewOyy/Z5V/e8WnFqRgd
+p4MFyqGHuaFZRrR3+/LYQpBbV77dP+A+HltUieg/QvcarW5hBXlvlJDyvzMkPW0e
+7jv0Fh0jQhc=
+-----END CERTIFICATE-----
diff --git a/test/e2e-v2/cases/storage/opensearch/certs/root-ca-key.pem b/test/e2e-v2/cases/storage/opensearch/certs/root-ca-key.pem
new file mode 100644
index 000000000000..6d03332c4013
--- /dev/null
+++ b/test/e2e-v2/cases/storage/opensearch/certs/root-ca-key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDaV7mh/uKfwe9L
+sgvW834oBai9YNj7OGBM0inKWHQfO78OUwM9ri6EXgbxNnIbmvPPyX6ciUFuo9+Q
++Q+7mlyAvUyPGn7eomFa32ViOjvIHl9mtF4SvWcZ8yfSJdzFf/8Ul1OPxKM8MiCM
+mr6B6EqSUD+L/LNDyA+j9SCctwoPgpXljCkF2rfQepu51tw6RelokQ3dNoUQE0gL
+3FajOoZlbdGBX2h3Esik2h8bIIaZ8KM4at4ldr1nrsLY71nXqTGAsO/2/loF6Ccv
+wgnGqQYdkwR2u6y4U0B9K2m4c7lJ1+g/SiTBoDaSnXhwJxOujRACekPGKsT8p1+m
+saWK4OKBAgMBAAECggEADEbOprejCWvvmjhE5ycIt5P3ev9B8M1byMxjHvMrQ4Hp
+/UWyOUEc13XfLdsR8Z015HLsl6Ocb+3VFhzGHPySAyEU3jgvD8vEh1+a0gbHmFOk
+D8b7dR7i+BGoC9TAkMN6InyOhvDKltaT5iP7RM88high8pG/Ijc4/eiJMJaWgVM+
+2Fu02LeNf7YJS08lU5Kz+rl1XhN6VWGoufOd8hgA8um4OOMO8v8rBmsSfzGVm2L9
+BYeH7t3ktBHAtr5HguNpVnwqatx+q2qf2noKuo3OiGxRzxA0/vJpRNXwVaMcJadK
+ecAXjViZ+3Ksw5sRH6J3OfvdCaz6K43ED5kPWOp/GwKBgQDy0Y1tt6oY9NsDPx92
+g3fTwWqrNPqTYXKRAIncR/SL8cG1x+3mHFh5pPK+rIWykWig7l5k98jFn/86Y6Li
+7DxLPjmxVyBa4pkT1Jaz7rtZcsE8OOGFLl75zCvW1ygaAUOt9HmqeBRcYeqNuney
+dbjgO8W18nuL9619agygMdbJNwKBgQDmMggN+ClzKX4xWdUJ8j5rFqv+8MnOgsxH
+4xu9l2tdlpdquKqpcV/qh3WLasyK+lMOZ3BgskfBN/+RGAvCw+aZlTUiXHFsKujc
+zJOtD09Bu5fuF27CaSC7CoGmStspLrRpko+jfSkqEs7J1/Cem6QAyAo0liM2EI/I
+s4SWf7KuBwKBgA0WFhUvrM8jgxotsLhmZXLYHbOUa1y+B7qg2M0yY4+XO+VcHQol
+xO7pYNu92IbDJ2xk7FlssTlVYh/3drPcH3O+qsVP+MJtK+rRrj2MRDSR5rAkMKNI
+2H2F72bouZSNNOSPJG93gUFpVYbF4eWQSqJrFkC0DMyCUKtNp9iKVxUJAoGBAMcE
+2IvjU0raw9y9EwA/bRG/D0MiQQgHc8BvLOu0v0Gx0gWV1Q8cE1Y8eTbpRiCeHjLk
+4XbojDsURCPYy0o/ft6n7sFfdTyUuLE1OjQ0eUyWeNuDbOIua/rqMX9pVqP7WkWw
+TfmGW5GhoyFFTiaC379BM/mVGKpElVtrQaWwj/X/AoGBAO91lMkWQDIEYLMQ9qMX
+p8MLTNRdtd07RkBmP7aaBIScttSpm/wFBLADtJOClLS7dlip1FIliy4Ax6cKZiF0
+McTm7puNJJ6Ub/H16vDpsHzITHKL6YnxuYDTPdzmrnbWATfCwLg+djMGCPhVdzx8
+N16xDsaVNQD6rc7VbyPA9cG0
+-----END PRIVATE KEY-----
diff --git a/test/e2e-v2/cases/storage/opensearch/certs/root-ca.pem b/test/e2e-v2/cases/storage/opensearch/certs/root-ca.pem
new file mode 100644
index 000000000000..2b7c4e8f19f9
--- /dev/null
+++ b/test/e2e-v2/cases/storage/opensearch/certs/root-ca.pem
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDtTCCAp2gAwIBAgIUZGztPs0CHJ8E/LSMytSGotu5LgYwDQYJKoZIhvcNAQEL
+BQAwajELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQ0wCwYDVQQHDARUZXN0MRMw
+EQYDVQQKDApTa3lXYWxraW5nMQ0wCwYDVQQLDARUZXN0MRswGQYDVQQDDBJTa3lX
+YWxraW5nIFJvb3QgQ0EwHhcNMjYwMTA0MTIyNjQ2WhcNMjgwMTA0MTIyNjQ2WjBq
+MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExDTALBgNVBAcMBFRlc3QxEzARBgNV
+BAoMClNreVdhbGtpbmcxDTALBgNVBAsMBFRlc3QxGzAZBgNVBAMMElNreVdhbGtp
+bmcgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANpXuaH+
+4p/B70uyC9bzfigFqL1g2Ps4YEzSKcpYdB87vw5TAz2uLoReBvE2chua88/JfpyJ
+QW6j35D5D7uaXIC9TI8aft6iYVrfZWI6O8geX2a0XhK9ZxnzJ9Il3MV//xSXU4/E
+ozwyIIyavoHoSpJQP4v8s0PID6P1IJy3Cg+CleWMKQXat9B6m7nW3DpF6WiRDd02
+hRATSAvcVqM6hmVt0YFfaHcSyKTaHxsghpnwozhq3iV2vWeuwtjvWdepMYCw7/b+
+WgXoJy/CCcapBh2TBHa7rLhTQH0rabhzuUnX6D9KJMGgNpKdeHAnE66NEAJ6Q8Yq
+xPynX6axpYrg4oECAwEAAaNTMFEwHQYDVR0OBBYEFIDowZ4aQHYrGf5sw8jIMOVw
+rjb3MB8GA1UdIwQYMBaAFIDowZ4aQHYrGf5sw8jIMOVwrjb3MA8GA1UdEwEB/wQF
+MAMBAf8wDQYJKoZIhvcNAQELBQADggEBADnjc1eZgpGpQznQaRrp603wxyOVGTuE
+eH54FP/WhSp0asHDEYUTwymMYjSGswCYZttJKS9FjqvLGRyCG8s5q33MwI5uVXR+
+9noGKpnbpEEVQwziO5/V60vzuCN6qZcjfNHKU5WnaHRGGJ0zONb1K8aeBq+WIAE0
+CylXC958O4ZGhrzz9W+fo74Q0nqJ2UVVvuq+wDKStO1KjfIYDDP3bOLa1gJgBdLr
+nCjJDyok2GgofnNUfIvb2uJvTm7C4R91bT6oq4OKG6N2+KFE1AIuNLFuAM+3hgBK
+HJir+GJHztoz0AtrzAikgmkfmqYj03A5fLl0NuWAD5+E3ON0nC9Yu+Y=
+-----END CERTIFICATE-----
diff --git a/test/e2e-v2/cases/storage/opensearch/certs/truststore.jks b/test/e2e-v2/cases/storage/opensearch/certs/truststore.jks
new file mode 100644
index 000000000000..e646a2b78134
Binary files /dev/null and b/test/e2e-v2/cases/storage/opensearch/certs/truststore.jks differ
diff --git a/test/e2e-v2/cases/storage/opensearch/clientcert_config.yml b/test/e2e-v2/cases/storage/opensearch/clientcert_config.yml
new file mode 100644
index 000000000000..bb11046b4130
--- /dev/null
+++ b/test/e2e-v2/cases/storage/opensearch/clientcert_config.yml
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+_meta:
+ type: "config"
+ config_version: 2
+
+config:
+ dynamic:
+ http:
+ anonymous_auth_enabled: false
+ xff:
+ enabled: false
+ authc:
+ clientcert_auth_domain:
+ description: "Authenticate via SSL client certificates"
+ http_enabled: true
+ transport_enabled: true
+ order: 0
+ http_authenticator:
+ type: clientcert
+ config:
+ username_attribute: cn
+ challenge: false
+ authentication_backend:
+ type: noop
+ basic_internal_auth_domain:
+ description: "Authenticate via HTTP Basic against internal users database"
+ http_enabled: true
+ transport_enabled: true
+ order: 1
+ http_authenticator:
+ type: basic
+ challenge: true
+ authentication_backend:
+ type: intern
diff --git a/test/e2e-v2/cases/storage/opensearch/docker-compose.yml b/test/e2e-v2/cases/storage/opensearch/docker-compose.yml
index 41b072b9e07b..9a6876c63a1a 100644
--- a/test/e2e-v2/cases/storage/opensearch/docker-compose.yml
+++ b/test/e2e-v2/cases/storage/opensearch/docker-compose.yml
@@ -13,8 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-version: '2.1'
-
+version: "2.1"
services:
opensearch:
image: opensearchproject/opensearch:${OPENSEARCH_VERSION}
@@ -25,15 +24,81 @@ services:
environment:
- discovery.type=single-node
- cluster.routing.allocation.disk.threshold_enabled=false
- - plugins.security.ssl.http.enabled=false
+ # Security plugin settings
- DISABLE_INSTALL_DEMO_CONFIG=true
- - DISABLE_SECURITY_PLUGIN=true
+ - DISABLE_SECURITY_PLUGIN=false
+ - OPENSEARCH_INITIAL_ADMIN_PASSWORD=SecurePass@2024!
+ - NO_PROXY=opensearch,localhost,127.0.0.1
+ volumes:
+ - ./opensearch.yml:/usr/share/opensearch/config/opensearch.yml:ro
+ - ./certs:/usr/share/opensearch/config/certs:ro
+ - ./clientcert_config.yml:/tmp/clientcert_config.yml:ro
+ - ./internal_users.yml:/tmp/internal_users.yml:ro
healthcheck:
- test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"]
- interval: 5s
- timeout: 60s
- retries: 120
+ test:
+ - CMD
+ - bash
+ - -c
+ - |
+ curl --silent --fail \
+ --resolve opensearch:9200:127.0.0.1 \
+ --cacert /usr/share/opensearch/config/certs/root-ca.pem \
+ --cert /usr/share/opensearch/config/certs/admin.pem \
+ --key /usr/share/opensearch/config/certs/admin-key.pem \
+ https://opensearch:9200/_cluster/health || exit 1
+ interval: 10s
+ timeout: 10s
+ retries: 30
+ start_period: 60s
+ opensearch-init:
+ image: opensearchproject/opensearch:${OPENSEARCH_VERSION}
+ depends_on:
+ opensearch:
+ condition: service_healthy
+ networks:
+ - e2e
+ command: |
+ bash -c '
+ set -e
+ echo "Configuring client certificate authentication..."
+ /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \
+ -h opensearch \
+ -f /tmp/clientcert_config.yml \
+ -t config \
+ -icl \
+ -nhnv \
+ -cacert /usr/share/opensearch/config/certs/root-ca.pem \
+ -cert /usr/share/opensearch/config/certs/admin.pem \
+ -key /usr/share/opensearch/config/certs/admin-key.pem
+
+ echo "Configuring internal users..."
+ /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \
+ -h opensearch \
+ -f /tmp/internal_users.yml \
+ -t internalusers \
+ -icl \
+ -nhnv \
+ -cacert /usr/share/opensearch/config/certs/root-ca.pem \
+ -cert /usr/share/opensearch/config/certs/admin.pem \
+ -key /usr/share/opensearch/config/certs/admin-key.pem
+
+ sleep 2
+ echo "Adding certificate user to all_access role..."
+ export no_proxy=opensearch,localhost
+ curl --cacert /usr/share/opensearch/config/certs/root-ca.pem \
+ --cert /usr/share/opensearch/config/certs/admin.pem \
+ --key /usr/share/opensearch/config/certs/admin-key.pem \
+ -X PUT "https://opensearch:9200/_plugins/_security/api/rolesmapping/all_access" \
+ -H "Content-Type: application/json" \
+ -d "{\"users\":[\"node-0.example.com\"],\"backend_roles\":[\"admin\"],\"description\":\"Maps admin to all_access\"}"
+ echo "✓ Client certificate authentication configured!"
+ '
+ volumes:
+ - ./clientcert_config.yml:/tmp/clientcert_config.yml:ro
+ - ./internal_users.yml:/tmp/internal_users.yml:ro
+ - ./certs:/usr/share/opensearch/config/certs:ro
+ restart: "no"
oap:
extends:
file: ../../../script/docker-compose/base-compose.yml
@@ -42,15 +107,23 @@ services:
SW_STORAGE: elasticsearch
SW_STORAGE_ES_CLUSTER_NODES: opensearch:9200
SW_ES_USER: admin
- SW_ES_PASSWORD: admin
+ SW_ES_PASSWORD: SecurePass@2024!
+ SW_STORAGE_ES_HTTP_PROTOCOL: https
+ SW_STORAGE_ES_SSL_JKS_PATH: /etc/skywalking/certs/truststore.jks
+ SW_STORAGE_ES_SSL_JKS_PASS: changeit
+ SW_STORAGE_ES_SSL_KEY_STORE_PATH: /etc/skywalking/certs/client.p12
+ SW_STORAGE_ES_SSL_KEY_STORE_PASS: changeit
ports:
- 12800
+ volumes:
+ - ./certs:/etc/skywalking/certs:ro
depends_on:
opensearch:
condition: service_healthy
+ opensearch-init:
+ condition: service_completed_successfully
networks:
- e2e
-
provider:
extends:
file: ../../../script/docker-compose/base-compose.yml
@@ -62,7 +135,6 @@ services:
condition: service_healthy
networks:
- e2e
-
consumer:
extends:
file: ../../../script/docker-compose/base-compose.yml
@@ -74,6 +146,5 @@ services:
condition: service_healthy
provider:
condition: service_healthy
-
networks:
e2e:
diff --git a/test/e2e-v2/cases/storage/opensearch/generate-certs.sh b/test/e2e-v2/cases/storage/opensearch/generate-certs.sh
new file mode 100755
index 000000000000..a2344189a0c2
--- /dev/null
+++ b/test/e2e-v2/cases/storage/opensearch/generate-certs.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+CERTS_DIR="./certs"
+mkdir -p "$CERTS_DIR"
+
+# Generate Root CA
+openssl genrsa -out "$CERTS_DIR/root-ca-key.pem" 2048
+openssl req -new -x509 -sha256 -key "$CERTS_DIR/root-ca-key.pem" \
+ -subj "/C=US/ST=CA/L=Test/O=SkyWalking/OU=Test/CN=SkyWalking Root CA" \
+ -out "$CERTS_DIR/root-ca.pem" -days 730
+
+# Generate Node Certificate
+openssl genrsa -out "$CERTS_DIR/node-key.pem" 2048
+openssl req -new -key "$CERTS_DIR/node-key.pem" \
+ -subj "/C=US/ST=CA/L=Test/O=SkyWalking/OU=Test/CN=opensearch" \
+ -out "$CERTS_DIR/node.csr"
+
+# Create SAN config for node cert
+cat >"$CERTS_DIR/node-san.cnf" <30.1.1-jre
2.1.210
8.0.13
- 1.18.22
+ 1.18.30
2.4.1
2.22.0
- 3.8.0
+ 3.11.0
3.1.0