Enable os3 client by default, via feature flag

changelog/unreleased/pr-24626.toml

@@ -0,0 +1,5 @@
+type = "a"
+message = "Enable opensearch 3 client by default, via feature flag opensearch3_client"
+
+issues = []
+pulls = ["24626"]

distribution/pom.xml

@@ -82,5 +82,10 @@
             <artifactId>graylog-storage-opensearch2</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.graylog</groupId>
+            <artifactId>graylog-storage-opensearch3</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
 </project>

distribution/src/main/assembly/graylog.xml

@@ -66,5 +66,9 @@
             <source>${project.basedir}/../graylog-storage-opensearch2/target/graylog-storage-opensearch2-${project.version}.jar</source>
             <outputDirectory>plugin/</outputDirectory>
         </file>
+        <file>
+            <source>${project.basedir}/../graylog-storage-opensearch3/target/graylog-storage-opensearch3-${project.version}.jar</source>
+            <outputDirectory>plugin/</outputDirectory>
+        </file>
     </files>
 </assembly>

full-backend-tests/pom.xml

@@ -82,6 +82,19 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.graylog</groupId>
+            <artifactId>graylog-storage-opensearch3</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.graylog</groupId>
+            <artifactId>graylog-storage-opensearch3</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
         <!-- Logging dependencies -->
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>

graylog-storage-opensearch3/assembly.xml

@@ -16,6 +16,11 @@
             <useTransitiveDependencies>true</useTransitiveDependencies>
             <includes>
                 <include>org.graylog:graylog-storage-opensearch3</include>
+                <include>org.opensearch.client:opensearch-java</include>
+                <include>org.apache.httpcomponents.client5:httpclient5</include>
+                <include>org.apache.httpcomponents.core5:httpcore5</include>
+                <include>org.apache.httpcomponents.core5:httpcore5-h2</include>
+                <include>jakarta.json:jakarta.json-api</include>
             </includes>
         </dependencySet>
     </dependencySets>

graylog-storage-opensearch3/pom.xml

@@ -43,6 +43,7 @@
             <groupId>org.graylog2</groupId>
             <artifactId>graylog2-server</artifactId>
             <version>${project.parent.version}</version>
+            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.google.guava</groupId>
@@ -91,6 +92,15 @@
             <groupId>io.jsonwebtoken</groupId>
             <artifactId>jjwt-api</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>jakarta.annotation</groupId>
+            <artifactId>jakarta.annotation-api</artifactId>
+            <version>${jakarta.annotation-api.version}</version>
+        </dependency>
         <dependency>
             <groupId>io.jsonwebtoken</groupId>
             <artifactId>jjwt-impl</artifactId>
@@ -180,21 +190,35 @@
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-assembly-plugin</artifactId>
+                <artifactId>maven-shade-plugin</artifactId>
                 <configuration>
-                    <descriptors>assembly.xml</descriptors>
-                    <archive>
-                        <manifestEntries>
-                            <Graylog-Plugin-Properties-Path>${project.groupId}.${project.artifactId}</Graylog-Plugin-Properties-Path>
-                        </manifestEntries>
-                    </archive>
+                    <minimizeJar>false</minimizeJar>
+                    <createDependencyReducedPom>false</createDependencyReducedPom>
+                    <filters>
+                        <!-- Eclipse JARs are signed. When building a shaded JAR, either remove the signatures, or
+                             add dependency to the main JAR's manifest. Otherwise running the application will fail. -->
+                        <filter>
+                            <artifact>org.eclipse.osgi:*</artifact>
+                            <excludes>
+                                <exclude>META-INF/*.SF</exclude>
+                                <exclude>META-INF/*.DSA</exclude>
+                                <exclude>META-INF/*.RSA</exclude>
+                            </excludes>
+                        </filter>
+                    </filters>
                 </configuration>
                 <executions>
                     <execution>
                         <phase>package</phase>
                         <goals>
-                            <goal>single</goal>
+                            <goal>shade</goal>
                         </goals>
+                        <configuration>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                            </transformers>
+                        </configuration>
                     </execution>
                 </executions>
             </plugin>

graylog2-server/src/main/java/org/graylog2/bootstrap/CmdLineTool.java

@@ -650,9 +650,10 @@ protected void annotateInjectorExceptions(Collection<Message> messages) {
             } else if (rootCause instanceof AccessDeniedException) {
                 LOG.error(UI.wallString("Unable to access file " + rootCause.getMessage()));
                 System.exit(-2);
-            } else if (rootCause instanceof UnsupportedSearchException) {
-                final SearchVersion search = ((UnsupportedSearchException) rootCause).getSearchMajorVersion();
-                LOG.error(UI.wallString("Unsupported search version: " + search, DocsHelper.PAGE_ES_VERSIONS.toString()));
+            } else if (rootCause instanceof final UnsupportedSearchException searchException) {
+                final SearchVersion search = searchException.getSearchMajorVersion();
+                final String component = searchException.getMessage();
+                LOG.error(UI.wallString("Unsupported search version: " + search + "(component: " + component + ")", DocsHelper.PAGE_ES_VERSIONS.toString()));
                 System.exit(-3);
             } else if (rootCause instanceof ElasticsearchProbeException) {
                 LOG.error(UI.wallString(rootCause.getMessage(), DocsHelper.PAGE_ES_CONFIGURATION.toString()));

graylog2-server/src/main/java/org/graylog2/storage/VersionAwareProvider.java

@@ -23,10 +23,12 @@
 
 public class VersionAwareProvider<T> implements Provider<T> {
     private final SearchVersion elasticsearchMajorVersion;
+    private final SearchVersion actualVersion;
     private final Map<SearchVersion, Provider<T>> pluginBindings;
 
     @Inject
     public VersionAwareProvider(@DetectedSearchVersion SearchVersion indexerVersion, Map<SearchVersion, Provider<T>> pluginBindings) {
+        this.actualVersion = indexerVersion;
         this.elasticsearchMajorVersion = majorVersionFrom(indexerVersion);
         this.pluginBindings = pluginBindings;
     }
@@ -35,7 +37,7 @@ public VersionAwareProvider(@DetectedSearchVersion SearchVersion indexerVersion,
     public T get() {
         final Provider<T> provider = this.pluginBindings.get(elasticsearchMajorVersion);
         if (provider == null) {
-            throw new UnsupportedSearchException(elasticsearchMajorVersion, this.getClass().getName());
+            throw new UnsupportedSearchException(actualVersion, this.getClass().getName());
         }
         return provider.get();
     }

graylog2-server/src/main/resources/org/graylog2/featureflag/feature-flag.config

@@ -114,3 +114,6 @@ quick_jump=off
 
 # Enable message table favorite fields
 message_table_favorite_fields=on
+
+# Enable for OS3 client usage, disable for OS2 usage.
+opensearch3_client=on

graylog2-server/src/test/java/org/graylog/testing/completebackend/DefaultPluginJarsProvider.java

@@ -37,6 +37,8 @@ public List<Path> getJars() {
                         "graylog-storage-elasticsearch7-" + projectVersion + ".jar"),
                 Paths.get(reposDir, "graylog2-server/graylog-storage-opensearch2/target",
                         "graylog-storage-opensearch2-" + projectVersion + ".jar"),
+                Paths.get(reposDir, "graylog2-server/graylog-storage-opensearch3/target",
+                        "graylog-storage-opensearch3-" + projectVersion + ".jar"),
                 Paths.get(reposDir, "graylog-plugin-aws/target",
                         "graylog-plugin-aws-" + projectVersion + ".jar"),
                 Paths.get(reposDir, "graylog-plugin-threatintel/target",