From 6b5a588c034ae12a9c0c33bade40b1a1a625f02e Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Sun, 24 Aug 2025 18:47:41 +0200 Subject: [PATCH 1/7] Generate Java classes from OTLP proto files --- .../resources/checkstyle_suppressions.xml | 1 + gradle/verification-metadata.xml | 52 ++++++++++ x-pack/plugin/otel-data/build.gradle | 97 ++++++++++++++++++- .../otel-data/dev-tools/protoc_exe_sha256.sh | 56 +++++++++++ .../otel-data/licenses/protobuf-LICENSE.txt | 32 ++++++ .../otel-data/licenses/protobuf-NOTICE.txt | 32 ++++++ 6 files changed, 267 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh create mode 100644 x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt create mode 100644 x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt diff --git a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml index 98ded638773ce..cb4a58ee453a7 100644 --- a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml +++ b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml @@ -13,6 +13,7 @@ + diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index d161da5d92236..2af1a7f06b667 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -707,6 +707,11 @@ + + + + + @@ -867,6 +872,11 @@ + + + + + @@ -877,11 +887,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1869,6 +1916,11 @@ + + + + + diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle index 0ae8022bff617..13d943ac2e465 100644 --- a/x-pack/plugin/otel-data/build.gradle +++ b/x-pack/plugin/otel-data/build.gradle @@ -4,9 +4,12 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -apply plugin: 'elasticsearch.internal-es-plugin' -apply plugin: 'elasticsearch.internal-yaml-rest-test' -apply plugin: 'elasticsearch.internal-cluster-test' +plugins { + id 'elasticsearch.internal-es-plugin' + id 'elasticsearch.internal-yaml-rest-test' + id 'elasticsearch.internal-cluster-test' + id('com.google.protobuf') version '0.9.5' +} esplugin { name = 'x-pack-otel-data' @@ -15,6 +18,11 @@ esplugin { extendedPlugins = ['x-pack-core'] } +// Manually update verification-metadata.xml via dev-tools/protoc_exe_sha256.sh when updating the protobuf version. +// That's because the protobuf gradle plugin is only using the protoc version for the current platform. +// We need to add all platforms to verification-metadata.xml so that the appropriate metadata for the platform used by +// CI and other collaborators is present. +def protobufVersion = "4.32.0" dependencies { compileOnly project(path: xpackModule('core')) testImplementation project(path: ':x-pack:plugin:stack') @@ -34,4 +42,87 @@ dependencies { clusterModules project(xpackModule('stack')) clusterModules project(xpackModule('wildcard')) clusterModules project(xpackModule('mapper-version')) + + implementation "com.google.protobuf:protobuf-java:${protobufVersion}" +} + +protobuf { + protoc { + // The artifact spec for the Protobuf Compiler + artifact = "com.google.protobuf:protoc:${protobufVersion}" + } +} + +def otlpProtoVersion = "1.7.0" + +// To generate the checksum, download the file and run "shasum -a 256 ~/path/to/opentelemetry-proto-.zip" +def protoChecksum = "ddb80357ff146f5e3bda584907185b1f635412a4b31edf6f96b102a18b8e05dc" +def protoArchivePath = layout.buildDirectory.file("archives/opentelemetry-proto-${otlpProtoVersion}.zip") + +def downloadProtoArchive = tasks.register("downloadProtoArchive") { + outputs.file(protoArchivePath) + onlyIf { protoArchivePath.get().asFile.exists() == false } + doLast { + protoArchivePath.get().asFile.parentFile.mkdirs() + "https://github.com/open-telemetry/opentelemetry-proto/archive/v${otlpProtoVersion}.zip".toURL() + .withInputStream { is -> + protoArchivePath.get().asFile.withOutputStream {os -> os << is } + } + } +} + +def verifyProtoArchive = tasks.register("verifyProtoArchive") { + dependsOn(downloadProtoArchive) + doLast { + protoArchivePath.get().asFile.withInputStream { inputStream -> + def sha256 = inputStream.readAllBytes().digest("SHA-256") + if (sha256 != protoChecksum) { + throw new GradleException("Checksum verification failed for $protoArchivePath: expected $protoChecksum, got $sha256") + } + } + } +} + +def unzipProtoArchive = tasks.register("unzipProtoArchive", Copy) { + dependsOn(verifyProtoArchive) + from zipTree(protoArchivePath) + into layout.buildDirectory.dir("protos") +} + +// Avoid unnecessary String allocations in the generated AnyValue class +// We always need the ByteString (UTF-8) representation of string attributes +def adjustAnyValueBuilder = tasks.register("adjustAnyValueBuilder") { + doLast { + ant.replace( + file: 'build/generated/sources/proto/main/java/io/opentelemetry/proto/common/v1/AnyValue.java', + token: 'java.lang.String s = input.readStringRequireUtf8();', + value: 'com.google.protobuf.ByteString s = input.readBytes();', + encoding: 'UTF-8' + ) + } +} + +sourceSets { + main { + proto { + srcDir layout.buildDirectory.dir("protos/opentelemetry-proto-${otlpProtoVersion}") + } + } +} + +afterEvaluate { + tasks.named("generateProto") { + dependsOn(unzipProtoArchive) + finalizedBy(adjustAnyValueBuilder) + } +} + +idea { + module { + sourceDirs += layout.buildDirectory.dir("generated/sources/proto/main/java").get().asFile + } +} + +tasks.named("dependencyLicenses").configure { + mapping from: /protobuf.*/, to: 'protobuf' } diff --git a/x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh b/x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh new file mode 100644 index 0000000000000..58e31f4cdc50d --- /dev/null +++ b/x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". +# + +# Script to download all .exe files from protobuf protoc repository and generate SHA256 checksums +# URL to download from +VERSION="4.32.0" +URL="https://repo1.maven.org/maven2/com/google/protobuf/protoc/${VERSION}/" +DOWNLOAD_DIR="protoc-${VERSION}-executables" + +# Create download directory if it doesn't exist +mkdir -p "${DOWNLOAD_DIR}" +cd "${DOWNLOAD_DIR}" || { echo "Failed to create/enter download directory"; exit 1; } + +# Get the HTML content, extract links to .exe files (but not .exe.md5 etc.) +# Using grep with lookahead assertion to ensure we don't match .exe followed by something else +curl -s "${URL}" | grep -o 'href="[^"]*\.exe"' | grep -v -E 'jsonl' | grep -v -E '\.exe\.[^"]+' | sed 's/href="//g' | sed 's/"//g' > exe_files.txt + +if [ ! -s exe_files.txt ]; then + echo "No .exe files found at ${URL}" + exit 1 +fi + +echo "Found $(wc -l < exe_files.txt | tr -d ' ') .exe files. Downloading..." + +# Download each file +while IFS= read -r file; do + curl -s -O "${URL}${file}" +done < exe_files.txt + +echo "Generating SHA256 checksums..." + +# Generate SHA256 checksums for all downloaded .exe files +if command -v shasum &> /dev/null; then + # macOS/some Linux + shasum -a 256 *.exe > SHA256SUMS.txt +elif command -v sha256sum &> /dev/null; then + # Most Linux distributions + sha256sum *.exe > SHA256SUMS.txt +else + echo "Neither shasum nor sha256sum command found. Cannot generate checksums." + exit 1 +fi + +# Print the checksums +cat SHA256SUMS.txt + +cd .. +rm -rf "${DOWNLOAD_DIR}" diff --git a/x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt b/x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt b/x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. From 77604a427a597603ddf7687b8a5af78353b3d01a Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 25 Aug 2025 08:57:30 +0200 Subject: [PATCH 2/7] Ignore protobuf thirdPartyAudit violations There's precent for this in plugins/repository-hdfs/build.gradle --- x-pack/plugin/otel-data/build.gradle | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle index 13d943ac2e465..7dd3823795c46 100644 --- a/x-pack/plugin/otel-data/build.gradle +++ b/x-pack/plugin/otel-data/build.gradle @@ -126,3 +126,16 @@ idea { tasks.named("dependencyLicenses").configure { mapping from: /protobuf.*/, to: 'protobuf' } + +tasks.named("thirdPartyAudit").configure { + ignoreViolations( + // uses internal java api: sun.misc.Unsafe + 'com.google.protobuf.MessageSchema', + 'com.google.protobuf.UnsafeUtil', + 'com.google.protobuf.UnsafeUtil$1', + 'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor', + 'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor', + 'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor', + 'com.google.protobuf.UnsafeUtil$MemoryAccessor' + ) +} From 71f86a728755bcfa355635e2851236cac7594311 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Mon, 25 Aug 2025 12:36:33 +0200 Subject: [PATCH 3/7] Use Gradle dependency mgmt for resolving otel proto files --- gradle/verification-metadata.xml | 5 ++ x-pack/plugin/otel-data/build.gradle | 98 ++++++++++++++-------------- 2 files changed, 54 insertions(+), 49 deletions(-) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 2af1a7f06b667..873004687bb10 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2054,6 +2054,11 @@ + + + + + diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle index 7dd3823795c46..317e204e2032c 100644 --- a/x-pack/plugin/otel-data/build.gradle +++ b/x-pack/plugin/otel-data/build.gradle @@ -4,6 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import org.elasticsearch.gradle.transform.UnzipTransform +import com.google.protobuf.gradle.GenerateProtoTask plugins { id 'elasticsearch.internal-es-plugin' id 'elasticsearch.internal-yaml-rest-test' @@ -18,12 +20,39 @@ esplugin { extendedPlugins = ['x-pack-core'] } +repositories { + ivy { + url = uri("https://github.com/open-telemetry/opentelemetry-proto/archive/") + patternLayout { artifact("v[revision].[ext]") } + metadataSources { artifact() } + content { + includeModule("open-telemetry", "opentelemetry-proto") // only this GAV comes from here + } + } +} + +configurations { + opentelemetryProtobuf { + attributes.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE) + canBeConsumed = false + } +} + +def otlpProtoVersion = "1.7.0" + // Manually update verification-metadata.xml via dev-tools/protoc_exe_sha256.sh when updating the protobuf version. // That's because the protobuf gradle plugin is only using the protoc version for the current platform. // We need to add all platforms to verification-metadata.xml so that the appropriate metadata for the platform used by // CI and other collaborators is present. def protobufVersion = "4.32.0" + dependencies { + registerTransform(UnzipTransform, transformSpec -> { + transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE); + transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); + }); + opentelemetryProtobuf "open-telemetry:opentelemetry-proto:${otlpProtoVersion}@zip" + compileOnly project(path: xpackModule('core')) testImplementation project(path: ':x-pack:plugin:stack') testImplementation(testArtifact(project(xpackModule('core')))) @@ -53,67 +82,38 @@ protobuf { } } -def otlpProtoVersion = "1.7.0" - -// To generate the checksum, download the file and run "shasum -a 256 ~/path/to/opentelemetry-proto-.zip" -def protoChecksum = "ddb80357ff146f5e3bda584907185b1f635412a4b31edf6f96b102a18b8e05dc" -def protoArchivePath = layout.buildDirectory.file("archives/opentelemetry-proto-${otlpProtoVersion}.zip") - -def downloadProtoArchive = tasks.register("downloadProtoArchive") { - outputs.file(protoArchivePath) - onlyIf { protoArchivePath.get().asFile.exists() == false } - doLast { - protoArchivePath.get().asFile.parentFile.mkdirs() - "https://github.com/open-telemetry/opentelemetry-proto/archive/v${otlpProtoVersion}.zip".toURL() - .withInputStream { is -> - protoArchivePath.get().asFile.withOutputStream {os -> os << is } - } - } -} - -def verifyProtoArchive = tasks.register("verifyProtoArchive") { - dependsOn(downloadProtoArchive) - doLast { - protoArchivePath.get().asFile.withInputStream { inputStream -> - def sha256 = inputStream.readAllBytes().digest("SHA-256") - if (sha256 != protoChecksum) { - throw new GradleException("Checksum verification failed for $protoArchivePath: expected $protoChecksum, got $sha256") - } +def extractOtelProtos = tasks.register("extractOtelProtos", Copy) { + from(configurations.opentelemetryProtobuf) { + include "**/*.proto" + eachFile { fileCopyDetails -> + // strip the leading directory (opentelemetry-proto-) + def segments = fileCopyDetails.path.split('/') + fileCopyDetails.path = segments.length > 1 ? segments[1..-1].join('/') : fileCopyDetails.name } + includeEmptyDirs = false } + into(layout.buildDirectory.dir("protos/otel")) } -def unzipProtoArchive = tasks.register("unzipProtoArchive", Copy) { - dependsOn(verifyProtoArchive) - from zipTree(protoArchivePath) - into layout.buildDirectory.dir("protos") -} - -// Avoid unnecessary String allocations in the generated AnyValue class -// We always need the ByteString (UTF-8) representation of string attributes -def adjustAnyValueBuilder = tasks.register("adjustAnyValueBuilder") { - doLast { - ant.replace( - file: 'build/generated/sources/proto/main/java/io/opentelemetry/proto/common/v1/AnyValue.java', - token: 'java.lang.String s = input.readStringRequireUtf8();', - value: 'com.google.protobuf.ByteString s = input.readBytes();', - encoding: 'UTF-8' - ) - } -} sourceSets { main { proto { - srcDir layout.buildDirectory.dir("protos/opentelemetry-proto-${otlpProtoVersion}") + srcDir(extractOtelProtos) } } } -afterEvaluate { - tasks.named("generateProto") { - dependsOn(unzipProtoArchive) - finalizedBy(adjustAnyValueBuilder) +tasks.withType(GenerateProtoTask.class).matching { it.name == "generateProto" }.configureEach { + // Avoid unnecessary String allocations in the generated AnyValue class + // We always need the ByteString (UTF-8) representation of string attributes + doLast { + ant.replace( + file: 'build/generated/sources/proto/main/java/io/opentelemetry/proto/common/v1/AnyValue.java', + token: 'java.lang.String s = input.readStringRequireUtf8();', + value: 'com.google.protobuf.ByteString s = input.readBytes();', + encoding: 'UTF-8' + ) } } From b6065dc80a9b861955c8c8c31cf8661100d68b77 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 25 Aug 2025 16:03:57 +0200 Subject: [PATCH 4/7] Improve generation of verification metadata for protoc --- gradle/verification-metadata.xml | 20 +++---- x-pack/plugin/otel-data/build.gradle | 21 ++++--- .../otel-data/dev-tools/protoc_exe_sha256.sh | 56 ------------------- 3 files changed, 24 insertions(+), 73 deletions(-) delete mode 100644 x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 873004687bb10..374db938143cb 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -899,34 +899,34 @@ - + - + - + - + - + - + - + - + - + - + diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle index 317e204e2032c..c82990209b605 100644 --- a/x-pack/plugin/otel-data/build.gradle +++ b/x-pack/plugin/otel-data/build.gradle @@ -38,12 +38,6 @@ configurations { } } -def otlpProtoVersion = "1.7.0" - -// Manually update verification-metadata.xml via dev-tools/protoc_exe_sha256.sh when updating the protobuf version. -// That's because the protobuf gradle plugin is only using the protoc version for the current platform. -// We need to add all platforms to verification-metadata.xml so that the appropriate metadata for the platform used by -// CI and other collaborators is present. def protobufVersion = "4.32.0" dependencies { @@ -51,7 +45,7 @@ dependencies { transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE); transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); }); - opentelemetryProtobuf "open-telemetry:opentelemetry-proto:${otlpProtoVersion}@zip" + opentelemetryProtobuf "open-telemetry:opentelemetry-proto:1.7.0@zip" compileOnly project(path: xpackModule('core')) testImplementation project(path: ':x-pack:plugin:stack') @@ -73,6 +67,19 @@ dependencies { clusterModules project(xpackModule('mapper-version')) implementation "com.google.protobuf:protobuf-java:${protobufVersion}" + // The protobuf plugin only adds a dependency for the variant relevant for the current platform. + // Without explicitly adding all classifiers, the --write-verification-metadata task would only add the one for the current platform. + // Therefore, the verification check would fail on other platforms, like on CI or for contributors that work on another platform. + compileOnly "com.google.protobuf:protoc:${protobufVersion}:linux-aarch_64@exe" + compileOnly "com.google.protobuf:protoc:${protobufVersion}:linux-ppcle_64@exe" + compileOnly "com.google.protobuf:protoc:${protobufVersion}:linux-s390_64@exe" + compileOnly "com.google.protobuf:protoc:${protobufVersion}:linux-x86_32@exe" + compileOnly "com.google.protobuf:protoc:${protobufVersion}:linux-x86_64@exe" + compileOnly "com.google.protobuf:protoc:${protobufVersion}:osx-aarch_64@exe" + compileOnly "com.google.protobuf:protoc:${protobufVersion}:osx-universal_binary@exe" + compileOnly "com.google.protobuf:protoc:${protobufVersion}:osx-x86_64@exe" + compileOnly "com.google.protobuf:protoc:${protobufVersion}:windows-x86_32@exe" + compileOnly "com.google.protobuf:protoc:${protobufVersion}:windows-x86_64@exe" } protobuf { diff --git a/x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh b/x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh deleted file mode 100644 index 58e31f4cdc50d..0000000000000 --- a/x-pack/plugin/otel-data/dev-tools/protoc_exe_sha256.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the "Elastic License -# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side -# Public License v 1"; you may not use this file except in compliance with, at -# your election, the "Elastic License 2.0", the "GNU Affero General Public -# License v3.0 only", or the "Server Side Public License, v 1". -# - -# Script to download all .exe files from protobuf protoc repository and generate SHA256 checksums -# URL to download from -VERSION="4.32.0" -URL="https://repo1.maven.org/maven2/com/google/protobuf/protoc/${VERSION}/" -DOWNLOAD_DIR="protoc-${VERSION}-executables" - -# Create download directory if it doesn't exist -mkdir -p "${DOWNLOAD_DIR}" -cd "${DOWNLOAD_DIR}" || { echo "Failed to create/enter download directory"; exit 1; } - -# Get the HTML content, extract links to .exe files (but not .exe.md5 etc.) -# Using grep with lookahead assertion to ensure we don't match .exe followed by something else -curl -s "${URL}" | grep -o 'href="[^"]*\.exe"' | grep -v -E 'jsonl' | grep -v -E '\.exe\.[^"]+' | sed 's/href="//g' | sed 's/"//g' > exe_files.txt - -if [ ! -s exe_files.txt ]; then - echo "No .exe files found at ${URL}" - exit 1 -fi - -echo "Found $(wc -l < exe_files.txt | tr -d ' ') .exe files. Downloading..." - -# Download each file -while IFS= read -r file; do - curl -s -O "${URL}${file}" -done < exe_files.txt - -echo "Generating SHA256 checksums..." - -# Generate SHA256 checksums for all downloaded .exe files -if command -v shasum &> /dev/null; then - # macOS/some Linux - shasum -a 256 *.exe > SHA256SUMS.txt -elif command -v sha256sum &> /dev/null; then - # Most Linux distributions - sha256sum *.exe > SHA256SUMS.txt -else - echo "Neither shasum nor sha256sum command found. Cannot generate checksums." - exit 1 -fi - -# Print the checksums -cat SHA256SUMS.txt - -cd .. -rm -rf "${DOWNLOAD_DIR}" From edac103a4f2e3e50a59d12b30e49d138873e6b4b Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 25 Aug 2025 17:14:03 +0200 Subject: [PATCH 5/7] Ignore license headers for generated proto files --- x-pack/plugin/otel-data/build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle index c82990209b605..32a3cf5ed233f 100644 --- a/x-pack/plugin/otel-data/build.gradle +++ b/x-pack/plugin/otel-data/build.gradle @@ -146,3 +146,7 @@ tasks.named("thirdPartyAudit").configure { 'com.google.protobuf.UnsafeUtil$MemoryAccessor' ) } + +tasks.named("licenseHeaders").configure { + excludes << 'io/opentelemetry/proto/**/*' +} From adfe475946077fbf62a2225d9a66dcd73730ae68 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 25 Aug 2025 17:55:08 +0200 Subject: [PATCH 6/7] Use runtimeOnly scope to fix error in splitPackagesAudit --- x-pack/plugin/otel-data/build.gradle | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle index 32a3cf5ed233f..e9ffad87ccb41 100644 --- a/x-pack/plugin/otel-data/build.gradle +++ b/x-pack/plugin/otel-data/build.gradle @@ -70,16 +70,16 @@ dependencies { // The protobuf plugin only adds a dependency for the variant relevant for the current platform. // Without explicitly adding all classifiers, the --write-verification-metadata task would only add the one for the current platform. // Therefore, the verification check would fail on other platforms, like on CI or for contributors that work on another platform. - compileOnly "com.google.protobuf:protoc:${protobufVersion}:linux-aarch_64@exe" - compileOnly "com.google.protobuf:protoc:${protobufVersion}:linux-ppcle_64@exe" - compileOnly "com.google.protobuf:protoc:${protobufVersion}:linux-s390_64@exe" - compileOnly "com.google.protobuf:protoc:${protobufVersion}:linux-x86_32@exe" - compileOnly "com.google.protobuf:protoc:${protobufVersion}:linux-x86_64@exe" - compileOnly "com.google.protobuf:protoc:${protobufVersion}:osx-aarch_64@exe" - compileOnly "com.google.protobuf:protoc:${protobufVersion}:osx-universal_binary@exe" - compileOnly "com.google.protobuf:protoc:${protobufVersion}:osx-x86_64@exe" - compileOnly "com.google.protobuf:protoc:${protobufVersion}:windows-x86_32@exe" - compileOnly "com.google.protobuf:protoc:${protobufVersion}:windows-x86_64@exe" + runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-aarch_64@exe" + runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-ppcle_64@exe" + runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-s390_64@exe" + runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-x86_32@exe" + runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-x86_64@exe" + runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:osx-aarch_64@exe" + runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:osx-universal_binary@exe" + runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:osx-x86_64@exe" + runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:windows-x86_32@exe" + runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:windows-x86_64@exe" } protobuf { From 0567486d065fabd14c30cc78251c0797975eb330 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Tue, 26 Aug 2025 08:02:35 +0200 Subject: [PATCH 7/7] Use testRuntimeOnly scope so that generateTestBuildInfo does not fail --- x-pack/plugin/otel-data/build.gradle | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle index e9ffad87ccb41..af0592bca1703 100644 --- a/x-pack/plugin/otel-data/build.gradle +++ b/x-pack/plugin/otel-data/build.gradle @@ -70,16 +70,16 @@ dependencies { // The protobuf plugin only adds a dependency for the variant relevant for the current platform. // Without explicitly adding all classifiers, the --write-verification-metadata task would only add the one for the current platform. // Therefore, the verification check would fail on other platforms, like on CI or for contributors that work on another platform. - runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-aarch_64@exe" - runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-ppcle_64@exe" - runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-s390_64@exe" - runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-x86_32@exe" - runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-x86_64@exe" - runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:osx-aarch_64@exe" - runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:osx-universal_binary@exe" - runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:osx-x86_64@exe" - runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:windows-x86_32@exe" - runtimeOnly "com.google.protobuf:protoc:${protobufVersion}:windows-x86_64@exe" + testRuntimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-aarch_64@exe" + testRuntimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-ppcle_64@exe" + testRuntimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-s390_64@exe" + testRuntimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-x86_32@exe" + testRuntimeOnly "com.google.protobuf:protoc:${protobufVersion}:linux-x86_64@exe" + testRuntimeOnly "com.google.protobuf:protoc:${protobufVersion}:osx-aarch_64@exe" + testRuntimeOnly "com.google.protobuf:protoc:${protobufVersion}:osx-universal_binary@exe" + testRuntimeOnly "com.google.protobuf:protoc:${protobufVersion}:osx-x86_64@exe" + testRuntimeOnly "com.google.protobuf:protoc:${protobufVersion}:windows-x86_32@exe" + testRuntimeOnly "com.google.protobuf:protoc:${protobufVersion}:windows-x86_64@exe" } protobuf {