Generate Java classes from OTLP proto files

build-tools-internal/src/main/resources/checkstyle_suppressions.xml

@@ -13,6 +13,7 @@
   <suppress files="plugin[/\\]sql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]sql[/\\]parser[/\\]SqlBase(Base(Listener|Visitor)|Lexer|Listener|Parser|Visitor).java" checks="." />
   <suppress files="plugin[/\\]eql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]eql[/\\]parser[/\\]EqlBase(Base(Listener|Visitor)|Lexer|Listener|Parser|Visitor).java" checks="." />
   <suppress files="plugin[/\\]esql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]esql[/\\]parser[/\\]EsqlBase(Parser|Lexer).*.java" checks="." />
+  <suppress files="x-pack[/\\]plugin[/\\]otel-data[/\\]build[/\\]generated[/\\]sources[/\\]" checks="." />
 
   <!-- JNA requires the no-argument constructor on JNAKernel32Library.SizeT to be public-->
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JNAKernel32Library.java" checks="RedundantModifier" />

gradle/verification-metadata.xml

@@ -707,6 +707,11 @@
             <sha256 value="9a4e0b9f4ec4d71a8a1d3641fd481118100fda2eeab712dbdfd4b2a06e9de4ce" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.gradle" name="osdetector-gradle-plugin" version="1.7.3">
+         <artifact name="osdetector-gradle-plugin-1.7.3.jar">
+            <sha256 value="6b4692f913a21b1fb603169ee78ba8f3e4ab2af9d762af9ca88b79126c1c0ad1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.guava" name="failureaccess" version="1.0">
          <artifact name="failureaccess-1.0.jar">
             <sha256 value="d084bef9cd07a8537a1753e4850a69b7e8bab1d1e22e9f3a1e4826309a7a2336" origin="Generated by Gradle"/>
@@ -867,6 +872,11 @@
             <sha256 value="193edf97aefa28b93c5892bdc598bac34fa4c396588030084f290b1440e8b98a" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.protobuf" name="protobuf-gradle-plugin" version="0.9.5">
+         <artifact name="protobuf-gradle-plugin-0.9.5.jar">
+            <sha256 value="d007dda81afeaead5dab05c27fe023ccbac97790e4d11787f4be68e6716557f8" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.protobuf" name="protobuf-java" version="2.5.0">
          <artifact name="protobuf-java-2.5.0.jar">
             <sha256 value="e0c1c64575c005601725e7c6a02cebf9e1285e888f756b2a1d73ffa8d725cc74" origin="Generated by Gradle"/>
@@ -877,11 +887,48 @@
             <sha256 value="8540247fad9e06baefa8fb45eb313802d019f485f14300e0f9d6b556ed88e753" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.protobuf" name="protobuf-java" version="4.32.0">
+         <artifact name="protobuf-java-4.32.0.jar">
+            <sha256 value="e902c91b454812b7d056b8d303a572733bf0587576ff157c1049116c9626241d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.protobuf" name="protobuf-java-util" version="3.25.5">
          <artifact name="protobuf-java-util-3.25.5.jar">
             <sha256 value="dacc58b2c3d2fa8d4bddc1acb881e78d6cf7c137dd78bc1d67f6aca732436a8d" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.protobuf" name="protoc" version="4.32.0">
+         <artifact name="protoc-4.32.0-linux-aarch_64.exe">
+            <sha256 value="bc95ab180ef088eed3f97c30b213ac6439ccac01989d18f9e908eb5233432136" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-linux-ppcle_64.exe">
+            <sha256 value="37d1c923270dcea474423994cae32e127d9aeb0cf97d40ccf652bd671358dc74" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-linux-s390_64.exe">
+            <sha256 value="0b97e27d36456e8a29855f10e6dfacd27043131d44bed460b921641c7201ec0a" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-linux-x86_32.exe">
+            <sha256 value="75abfd89ae91a12cfbd40fe9aca20a698cc6134fbc7bef999f5c8f3b8c604b13" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-linux-x86_64.exe">
+            <sha256 value="373c7da2c1cae98ac1a494a8a53e28100af70450ab55c064daa92ba48af5fe23" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-osx-aarch_64.exe">
+            <sha256 value="96c06aac6eb7e247a78f2ffe8b3aaf4419a6765e3d96d724cc3db0e296e7aa94" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-osx-universal_binary.exe">
+            <sha256 value="35f98ecc7c0efeb7d63427a6f3b9f7f2adc9f59d79275c3f0ae87030876c187c" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-osx-x86_64.exe">
+            <sha256 value="91b2b3b5ad84cfed2869c69830c926130d312e357b4f128796504b5448a6534c" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-windows-x86_32.exe">
+            <sha256 value="dc71390eb8a113f37a24275bc57cc59ddaceeac39b98af2181c94755f2f411f9" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+         <artifact name="protoc-4.32.0-windows-x86_64.exe">
+            <sha256 value="a1e25d1b71e805233daefabba72a21566580354a192539cdab43bbf191dbf951" origin="x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh"/>
+         </artifact>
+      </component>
       <component group="com.google.re2j" name="re2j" version="1.1">
          <artifact name="re2j-1.1.jar">
             <sha256 value="24ada84d1b5de584e3e84b06f0c7dd562cee6eafe8dea8083bd8eb123823bbe7" origin="Generated by Gradle"/>
@@ -1869,6 +1916,11 @@
             <sha256 value="8e495b634469d64fb8acfa3495a065cbacc8a0fff55ce1e31007be4c16dc57d3" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="kr.motd.maven" name="os-maven-plugin" version="1.7.1">
+         <artifact name="os-maven-plugin-1.7.1.jar">
+            <sha256 value="f47aeef86821e52b2b18758978bd045f03d722292e32e747082122c6228952e0" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="krallin" name="tini" version="0.19.0">
          <artifact name="tini-0.19.0-amd64.jar">
             <sha256 value="93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c" origin="Generated by Gradle"/>
@@ -2002,6 +2054,11 @@
             <sha256 value="0a4aa0f2606bd6292a7ccd7d67a0db914bf5874dfb8a6184df3e6d63cdc93702" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="open-telemetry" name="opentelemetry-proto" version="1.7.0">
+         <artifact name="opentelemetry-proto-1.7.0.zip">
+            <sha256 value="ddb80357ff146f5e3bda584907185b1f635412a4b31edf6f96b102a18b8e05dc" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.abego.treelayout" name="org.abego.treelayout.core" version="1.0.3">
          <artifact name="org.abego.treelayout.core-1.0.3.jar">
             <sha256 value="fa5e31395c39c2e7d46aca0f81f72060931607b2fa41bd36038eb2cb6fb93326" origin="Generated by Gradle"/>

x-pack/plugin/otel-data/build.gradle

@@ -4,9 +4,14 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-apply plugin: 'elasticsearch.internal-es-plugin'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.internal-cluster-test'
+import org.elasticsearch.gradle.transform.UnzipTransform
+import com.google.protobuf.gradle.GenerateProtoTask
+plugins {
+  id 'elasticsearch.internal-es-plugin'
+  id 'elasticsearch.internal-yaml-rest-test'
+  id 'elasticsearch.internal-cluster-test'
+  id('com.google.protobuf') version '0.9.5'
+}
 
 esplugin {
   name = 'x-pack-otel-data'
@@ -15,7 +20,39 @@ esplugin {
   extendedPlugins = ['x-pack-core']
 }
 
+repositories {
+  ivy {
+    url = uri("https://github.com/open-telemetry/opentelemetry-proto/archive/")
+    patternLayout { artifact("v[revision].[ext]") }
+    metadataSources { artifact() }
+    content {
+      includeModule("open-telemetry", "opentelemetry-proto") // only this GAV comes from here
+    }
+  }
+}
+
+configurations {
+  opentelemetryProtobuf {
+    attributes.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE)
+    canBeConsumed = false
+  }
+}
+
+def otlpProtoVersion = "1.7.0"
+
+// Manually update verification-metadata.xml via dev-tools/protoc_exe_sha256.sh when updating the protobuf version.
+// That's because the protobuf gradle plugin is only using the protoc version for the current platform.
+// We need to add all platforms to verification-metadata.xml so that the appropriate metadata for the platform used by
+// CI and other collaborators is present.
+def protobufVersion = "4.32.0"
+
 dependencies {
+  registerTransform(UnzipTransform, transformSpec -> {
+    transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE);
+    transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
+  });
+  opentelemetryProtobuf "open-telemetry:opentelemetry-proto:${otlpProtoVersion}@zip"
+
   compileOnly project(path: xpackModule('core'))
   testImplementation project(path: ':x-pack:plugin:stack')
   testImplementation(testArtifact(project(xpackModule('core'))))
@@ -34,4 +71,71 @@ dependencies {
   clusterModules project(xpackModule('stack'))
   clusterModules project(xpackModule('wildcard'))
   clusterModules project(xpackModule('mapper-version'))
+
+  implementation "com.google.protobuf:protobuf-java:${protobufVersion}"
+}
+
+protobuf {
+  protoc {
+    // The artifact spec for the Protobuf Compiler
+    artifact = "com.google.protobuf:protoc:${protobufVersion}"
+  }
+}
+
+def extractOtelProtos = tasks.register("extractOtelProtos", Copy) {
+  from(configurations.opentelemetryProtobuf) {
+    include "**/*.proto"
+    eachFile { fileCopyDetails ->
+      // strip the leading directory (opentelemetry-proto-<version>)
+      def segments = fileCopyDetails.path.split('/')
+      fileCopyDetails.path = segments.length > 1 ? segments[1..-1].join('/') : fileCopyDetails.name
+    }
+    includeEmptyDirs = false
+  }
+  into(layout.buildDirectory.dir("protos/otel"))
+}
+
+
+sourceSets {
+  main {
+    proto {
+      srcDir(extractOtelProtos)
+    }
+  }
+}
+
+tasks.withType(GenerateProtoTask.class).matching { it.name == "generateProto" }.configureEach {
+  // Avoid unnecessary String allocations in the generated AnyValue class
+  // We always need the ByteString (UTF-8) representation of string attributes
+  doLast {
+      ant.replace(
+        file: 'build/generated/sources/proto/main/java/io/opentelemetry/proto/common/v1/AnyValue.java',
+        token: 'java.lang.String s = input.readStringRequireUtf8();',
+        value: 'com.google.protobuf.ByteString s = input.readBytes();',
+        encoding: 'UTF-8'
+      )
+  }
+}
+
+idea {
+  module {
+    sourceDirs += layout.buildDirectory.dir("generated/sources/proto/main/java").get().asFile
+  }
+}
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /protobuf.*/, to: 'protobuf'
+}
+
+tasks.named("thirdPartyAudit").configure {
+  ignoreViolations(
+    // uses internal java api: sun.misc.Unsafe
+    'com.google.protobuf.MessageSchema',
+    'com.google.protobuf.UnsafeUtil',
+    'com.google.protobuf.UnsafeUtil$1',
+    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$MemoryAccessor'
+  )
 }

x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the "Elastic License
+# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+# Public License v 1"; you may not use this file except in compliance with, at
+# your election, the "Elastic License 2.0", the "GNU Affero General Public
+# License v3.0 only", or the "Server Side Public License, v 1".
+#
+
+# Script to download all .exe files from protobuf protoc repository and generate SHA256 checksums
+# URL to download from
+VERSION="4.32.0"
+URL="https://repo1.maven.org/maven2/com/google/protobuf/protoc/${VERSION}/"
+DOWNLOAD_DIR="protoc-${VERSION}-executables"
+
+# Create download directory if it doesn't exist
+mkdir -p "${DOWNLOAD_DIR}"
+cd "${DOWNLOAD_DIR}" || { echo "Failed to create/enter download directory"; exit 1; }
+
+# Get the HTML content, extract links to .exe files (but not .exe.md5 etc.)
+# Using grep with lookahead assertion to ensure we don't match .exe followed by something else
+curl -s "${URL}" | grep -o 'href="[^"]*\.exe"' | grep -v -E 'jsonl' | grep -v -E '\.exe\.[^"]+' | sed 's/href="//g' | sed 's/"//g' > exe_files.txt
+
+if [ ! -s exe_files.txt ]; then
+    echo "No .exe files found at ${URL}"
+    exit 1
+fi
+
+echo "Found $(wc -l < exe_files.txt | tr -d ' ') .exe files. Downloading..."
+
+# Download each file
+while IFS= read -r file; do
+    curl -s -O "${URL}${file}"
+done < exe_files.txt
+
+echo "Generating SHA256 checksums..."
+
+# Generate SHA256 checksums for all downloaded .exe files
+if command -v shasum &> /dev/null; then
+    # macOS/some Linux
+    shasum -a 256 *.exe > SHA256SUMS.txt
+elif command -v sha256sum &> /dev/null; then
+    # Most Linux distributions
+    sha256sum *.exe > SHA256SUMS.txt
+else
+    echo "Neither shasum nor sha256sum command found. Cannot generate checksums."
+    exit 1
+fi
+
+# Print the checksums
+cat SHA256SUMS.txt
+
+cd ..
+rm -rf "${DOWNLOAD_DIR}"

x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt

@@ -0,0 +1,32 @@
+Copyright 2008 Google Inc.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Code generated by the Protocol Buffer compiler is owned by the owner
+of the input file used when generating it.  This code is not
+standalone and requires a support library to be linked with it.  This
+support library is itself covered by the above license.

x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt

@@ -0,0 +1,32 @@
+Copyright 2008 Google Inc.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Code generated by the Protocol Buffer compiler is owned by the owner
+of the input file used when generating it.  This code is not
+standalone and requires a support library to be linked with it.  This
+support library is itself covered by the above license.