Generate Java classes from OTLP proto files

build-tools-internal/src/main/resources/checkstyle_suppressions.xml

@@ -13,6 +13,7 @@
   <suppress files="plugin[/\\]sql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]sql[/\\]parser[/\\]SqlBase(Base(Listener|Visitor)|Lexer|Listener|Parser|Visitor).java" checks="." />
   <suppress files="plugin[/\\]eql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]eql[/\\]parser[/\\]EqlBase(Base(Listener|Visitor)|Lexer|Listener|Parser|Visitor).java" checks="." />
   <suppress files="plugin[/\\]esql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]esql[/\\]parser[/\\]EsqlBase(Parser|Lexer).*.java" checks="." />
+  <suppress files="x-pack[/\\]plugin[/\\]otel-data[/\\]build[/\\]generated[/\\]sources[/\\]" checks="." />
 
   <!-- JNA requires the no-argument constructor on JNAKernel32Library.SizeT to be public-->
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JNAKernel32Library.java" checks="RedundantModifier" />

gradle/verification-metadata.xml

@@ -707,6 +707,11 @@
             <sha256 value="9a4e0b9f4ec4d71a8a1d3641fd481118100fda2eeab712dbdfd4b2a06e9de4ce" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.gradle" name="osdetector-gradle-plugin" version="1.7.3">
+         <artifact name="osdetector-gradle-plugin-1.7.3.jar">
+            <sha256 value="6b4692f913a21b1fb603169ee78ba8f3e4ab2af9d762af9ca88b79126c1c0ad1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.guava" name="failureaccess" version="1.0">
          <artifact name="failureaccess-1.0.jar">
             <sha256 value="d084bef9cd07a8537a1753e4850a69b7e8bab1d1e22e9f3a1e4826309a7a2336" origin="Generated by Gradle"/>
@@ -867,6 +872,11 @@
             <sha256 value="193edf97aefa28b93c5892bdc598bac34fa4c396588030084f290b1440e8b98a" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.protobuf" name="protobuf-gradle-plugin" version="0.9.5">
+         <artifact name="protobuf-gradle-plugin-0.9.5.jar">
+            <sha256 value="d007dda81afeaead5dab05c27fe023ccbac97790e4d11787f4be68e6716557f8" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.protobuf" name="protobuf-java" version="2.5.0">
          <artifact name="protobuf-java-2.5.0.jar">
             <sha256 value="e0c1c64575c005601725e7c6a02cebf9e1285e888f756b2a1d73ffa8d725cc74" origin="Generated by Gradle"/>
@@ -877,11 +887,48 @@
             <sha256 value="8540247fad9e06baefa8fb45eb313802d019f485f14300e0f9d6b556ed88e753" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.protobuf" name="protobuf-java" version="4.32.0">
+         <artifact name="protobuf-java-4.32.0.jar">
+            <sha256 value="e902c91b454812b7d056b8d303a572733bf0587576ff157c1049116c9626241d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.protobuf" name="protobuf-java-util" version="3.25.5">
          <artifact name="protobuf-java-util-3.25.5.jar">
             <sha256 value="dacc58b2c3d2fa8d4bddc1acb881e78d6cf7c137dd78bc1d67f6aca732436a8d" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.protobuf" name="protoc" version="4.32.0">
+         <artifact name="protoc-4.32.0-linux-aarch_64.exe">
+            <sha256 value="bc95ab180ef088eed3f97c30b213ac6439ccac01989d18f9e908eb5233432136" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-linux-ppcle_64.exe">
+            <sha256 value="37d1c923270dcea474423994cae32e127d9aeb0cf97d40ccf652bd671358dc74" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-linux-s390_64.exe">
+            <sha256 value="0b97e27d36456e8a29855f10e6dfacd27043131d44bed460b921641c7201ec0a" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-linux-x86_32.exe">
+            <sha256 value="75abfd89ae91a12cfbd40fe9aca20a698cc6134fbc7bef999f5c8f3b8c604b13" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-linux-x86_64.exe">
+            <sha256 value="373c7da2c1cae98ac1a494a8a53e28100af70450ab55c064daa92ba48af5fe23" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-osx-aarch_64.exe">
+            <sha256 value="96c06aac6eb7e247a78f2ffe8b3aaf4419a6765e3d96d724cc3db0e296e7aa94" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-osx-universal_binary.exe">
+            <sha256 value="35f98ecc7c0efeb7d63427a6f3b9f7f2adc9f59d79275c3f0ae87030876c187c" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-osx-x86_64.exe">
+            <sha256 value="91b2b3b5ad84cfed2869c69830c926130d312e357b4f128796504b5448a6534c" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-windows-x86_32.exe">
+            <sha256 value="dc71390eb8a113f37a24275bc57cc59ddaceeac39b98af2181c94755f2f411f9" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-windows-x86_64.exe">
+            <sha256 value="a1e25d1b71e805233daefabba72a21566580354a192539cdab43bbf191dbf951" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+      </component>
       <component group="com.google.re2j" name="re2j" version="1.1">
          <artifact name="re2j-1.1.jar">
             <sha256 value="24ada84d1b5de584e3e84b06f0c7dd562cee6eafe8dea8083bd8eb123823bbe7" origin="Generated by Gradle"/>
@@ -1869,6 +1916,11 @@
             <sha256 value="8e495b634469d64fb8acfa3495a065cbacc8a0fff55ce1e31007be4c16dc57d3" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="kr.motd.maven" name="os-maven-plugin" version="1.7.1">
+         <artifact name="os-maven-plugin-1.7.1.jar">
+            <sha256 value="f47aeef86821e52b2b18758978bd045f03d722292e32e747082122c6228952e0" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="krallin" name="tini" version="0.19.0">
          <artifact name="tini-0.19.0-amd64.jar">
             <sha256 value="93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c" origin="Generated by Gradle"/>

x-pack/plugin/otel-data/build.gradle

@@ -4,9 +4,12 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-apply plugin: 'elasticsearch.internal-es-plugin'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.internal-cluster-test'
+plugins {
+  id 'elasticsearch.internal-es-plugin'
+  id 'elasticsearch.internal-yaml-rest-test'
+  id 'elasticsearch.internal-cluster-test'
+  id('com.google.protobuf') version '0.9.5'
+}
 
 esplugin {
   name = 'x-pack-otel-data'
@@ -15,6 +18,11 @@ esplugin {
   extendedPlugins = ['x-pack-core']
 }
 
+// Manually update verification-metadata.xml via dev-tools/protoc_exe_sha256.sh when updating the protobuf version.
+// That's because the protobuf gradle plugin is only using the protoc version for the current platform.
+// We need to add all platforms to verification-metadata.xml so that the appropriate metadata for the platform used by
+// CI and other collaborators is present.
+def protobufVersion = "4.32.0"
 dependencies {
   compileOnly project(path: xpackModule('core'))
   testImplementation project(path: ':x-pack:plugin:stack')
@@ -34,4 +42,100 @@ dependencies {
   clusterModules project(xpackModule('stack'))
   clusterModules project(xpackModule('wildcard'))
   clusterModules project(xpackModule('mapper-version'))
+
+  implementation "com.google.protobuf:protobuf-java:${protobufVersion}"
+}
+
+protobuf {
+  protoc {
+    // The artifact spec for the Protobuf Compiler
+    artifact = "com.google.protobuf:protoc:${protobufVersion}"
+  }
+}
+
+def otlpProtoVersion = "1.7.0"
+
+// To generate the checksum, download the file and run "shasum -a 256 ~/path/to/opentelemetry-proto-<version>.zip"
+def protoChecksum = "ddb80357ff146f5e3bda584907185b1f635412a4b31edf6f96b102a18b8e05dc"
+def protoArchivePath = layout.buildDirectory.file("archives/opentelemetry-proto-${otlpProtoVersion}.zip")
+
+def downloadProtoArchive = tasks.register("downloadProtoArchive") {
+  outputs.file(protoArchivePath)
+  onlyIf { protoArchivePath.get().asFile.exists() == false }
+  doLast {
+    protoArchivePath.get().asFile.parentFile.mkdirs()
+    "https://github.com/open-telemetry/opentelemetry-proto/archive/v${otlpProtoVersion}.zip".toURL()
+      .withInputStream { is ->
+        protoArchivePath.get().asFile.withOutputStream {os -> os << is }
+      }
+  }
+}
+
+def verifyProtoArchive = tasks.register("verifyProtoArchive") {
+  dependsOn(downloadProtoArchive)
+  doLast {
+    protoArchivePath.get().asFile.withInputStream { inputStream ->
+      def sha256 = inputStream.readAllBytes().digest("SHA-256")
+      if (sha256 != protoChecksum) {
+        throw new GradleException("Checksum verification failed for $protoArchivePath: expected $protoChecksum, got $sha256")
+      }
+    }
+  }
+}
+
+def unzipProtoArchive = tasks.register("unzipProtoArchive", Copy) {
+  dependsOn(verifyProtoArchive)
+  from zipTree(protoArchivePath)
+  into layout.buildDirectory.dir("protos")
+}
+
+// Avoid unnecessary String allocations in the generated AnyValue class
+// We always need the ByteString (UTF-8) representation of string attributes
+def adjustAnyValueBuilder = tasks.register("adjustAnyValueBuilder") {
+  doLast {
+    ant.replace(
+      file: 'build/generated/sources/proto/main/java/io/opentelemetry/proto/common/v1/AnyValue.java',
+      token: 'java.lang.String s = input.readStringRequireUtf8();',
+      value: 'com.google.protobuf.ByteString s = input.readBytes();',
+      encoding: 'UTF-8'
+    )
+  }
+}
+
+sourceSets {
+  main {
+    proto {
+      srcDir layout.buildDirectory.dir("protos/opentelemetry-proto-${otlpProtoVersion}")
+    }
+  }
+}
+
+afterEvaluate {
+  tasks.named("generateProto") {
+    dependsOn(unzipProtoArchive)
+    finalizedBy(adjustAnyValueBuilder)
+  }
+}
+
+idea {
+  module {
+    sourceDirs += layout.buildDirectory.dir("generated/sources/proto/main/java").get().asFile
+  }
+}
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /protobuf.*/, to: 'protobuf'
+}
+
+tasks.named("thirdPartyAudit").configure {
+  ignoreViolations(
+    // uses internal java api: sun.misc.Unsafe
+    'com.google.protobuf.MessageSchema',
+    'com.google.protobuf.UnsafeUtil',
+    'com.google.protobuf.UnsafeUtil$1',
+    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$MemoryAccessor'
+  )
 }

x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the "Elastic License
+# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+# Public License v 1"; you may not use this file except in compliance with, at
+# your election, the "Elastic License 2.0", the "GNU Affero General Public
+# License v3.0 only", or the "Server Side Public License, v 1".
+#
+
+# Script to download all .exe files from protobuf protoc repository and generate SHA256 checksums
+# URL to download from
+VERSION="4.32.0"
+URL="https://repo1.maven.org/maven2/com/google/protobuf/protoc/${VERSION}/"
+DOWNLOAD_DIR="protoc-${VERSION}-executables"
+
+# Create download directory if it doesn't exist
+mkdir -p "${DOWNLOAD_DIR}"
+cd "${DOWNLOAD_DIR}" || { echo "Failed to create/enter download directory"; exit 1; }
+
+# Get the HTML content, extract links to .exe files (but not .exe.md5 etc.)
+# Using grep with lookahead assertion to ensure we don't match .exe followed by something else
+curl -s "${URL}" | grep -o 'href="[^"]*\.exe"' | grep -v -E 'jsonl' | grep -v -E '\.exe\.[^"]+' | sed 's/href="//g' | sed 's/"//g' > exe_files.txt
+
+if [ ! -s exe_files.txt ]; then
+    echo "No .exe files found at ${URL}"
+    exit 1
+fi
+
+echo "Found $(wc -l < exe_files.txt | tr -d ' ') .exe files. Downloading..."
+
+# Download each file
+while IFS= read -r file; do
+    curl -s -O "${URL}${file}"
+done < exe_files.txt
+
+echo "Generating SHA256 checksums..."
+
+# Generate SHA256 checksums for all downloaded .exe files
+if command -v shasum &> /dev/null; then
+    # macOS/some Linux
+    shasum -a 256 *.exe > SHA256SUMS.txt
+elif command -v sha256sum &> /dev/null; then
+    # Most Linux distributions
+    sha256sum *.exe > SHA256SUMS.txt
+else
+    echo "Neither shasum nor sha256sum command found. Cannot generate checksums."
+    exit 1
+fi
+
+# Print the checksums
+cat SHA256SUMS.txt
+
+cd ..
+rm -rf "${DOWNLOAD_DIR}"

x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt

@@ -0,0 +1,32 @@
+Copyright 2008 Google Inc.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Code generated by the Protocol Buffer compiler is owned by the owner
+of the input file used when generating it.  This code is not
+standalone and requires a support library to be linked with it.  This
+support library is itself covered by the above license.

x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt

@@ -0,0 +1,32 @@
+Copyright 2008 Google Inc.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Code generated by the Protocol Buffer compiler is owned by the owner
+of the input file used when generating it.  This code is not
+standalone and requires a support library to be linked with it.  This
+support library is itself covered by the above license.