diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java new file mode 100644 index 0000000000000..adb39368d8d24 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.dependencies.patches.hdfs; + +import org.gradle.api.artifacts.transform.CacheableTransform; +import org.gradle.api.artifacts.transform.InputArtifact; +import org.gradle.api.artifacts.transform.TransformAction; +import org.gradle.api.artifacts.transform.TransformOutputs; +import org.gradle.api.artifacts.transform.TransformParameters; +import org.gradle.api.file.FileSystemLocation; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.Classpath; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Optional; +import org.jetbrains.annotations.NotNull; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Function; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.util.regex.Pattern; + +import static java.util.Map.entry; +import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES; +import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS; + +@CacheableTransform +public abstract class HdfsClassPatcher implements TransformAction { + + record JarPatchers(String artifactTag, Pattern artifactPattern, Map> jarPatchers) {} + + static final List allPatchers = List.of( + new JarPatchers( + "hadoop-common", + Pattern.compile("hadoop-common-(?!.*tests)"), + Map.ofEntries( + entry("org/apache/hadoop/util/ShutdownHookManager.class", ShutdownHookManagerPatcher::new), + entry("org/apache/hadoop/util/Shell.class", ShellPatcher::new), + entry("org/apache/hadoop/security/UserGroupInformation.class", SubjectGetSubjectPatcher::new) + ) + ), + new JarPatchers( + "hadoop-client-api", + Pattern.compile("hadoop-client-api.*"), + Map.ofEntries( + entry("org/apache/hadoop/util/ShutdownHookManager.class", ShutdownHookManagerPatcher::new), + entry("org/apache/hadoop/util/Shell.class", ShellPatcher::new), + entry("org/apache/hadoop/security/UserGroupInformation.class", SubjectGetSubjectPatcher::new), + entry("org/apache/hadoop/security/authentication/client/KerberosAuthenticator.class", SubjectGetSubjectPatcher::new) + ) + ) + ); + + interface Parameters extends TransformParameters { + @Input + @Optional + List getMatchingArtifacts(); + + void setMatchingArtifacts(List matchingArtifacts); + } + + @Classpath + @InputArtifact + public abstract Provider getInputArtifact(); + + @Override + public void transform(@NotNull TransformOutputs outputs) { + File inputFile = getInputArtifact().get().getAsFile(); + + List matchingArtifacts = getParameters().getMatchingArtifacts(); + List patchersToApply = allPatchers.stream() + .filter(jp -> matchingArtifacts.contains(jp.artifactTag()) && jp.artifactPattern().matcher(inputFile.getName()).find()) + .toList(); + if (patchersToApply.isEmpty()) { + outputs.file(getInputArtifact()); + } else { + patchersToApply.forEach(patchers -> { + System.out.println("Patching " + inputFile.getName()); + + Map> jarPatchers = new HashMap<>(patchers.jarPatchers()); + File outputFile = outputs.file(inputFile.getName().replace(".jar", "-patched.jar")); + + patchJar(inputFile, outputFile, jarPatchers); + + if (jarPatchers.isEmpty() == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "error patching [%s] with [%s]: the jar does not contain [%s]", + inputFile.getName(), + patchers.artifactPattern().toString(), + String.join(", ", jarPatchers.keySet()) + ) + ); + } + }); + } + } + + private static void patchJar(File inputFile, File outputFile, Map> jarPatchers) { + try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) { + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + String entryName = entry.getName(); + // Add the entry to the new JAR file + jos.putNextEntry(new JarEntry(entryName)); + + Function classPatcher = jarPatchers.remove(entryName); + if (classPatcher != null) { + byte[] classToPatch = jarFile.getInputStream(entry).readAllBytes(); + + ClassReader classReader = new ClassReader(classToPatch); + ClassWriter classWriter = new ClassWriter(classReader, COMPUTE_FRAMES | COMPUTE_MAXS); + classReader.accept(classPatcher.apply(classWriter), 0); + + jos.write(classWriter.toByteArray()); + } else { + // Read the entry's data and write it to the new JAR + try (InputStream is = jarFile.getInputStream(entry)) { + is.transferTo(jos); + } + } + jos.closeEntry(); + } + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } +} diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/MethodReplacement.java similarity index 94% rename from plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/MethodReplacement.java index e07a32cc294a5..7bc6a6c0d530f 100644 --- a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/MethodReplacement.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.hdfs.patch; +package org.elasticsearch.gradle.internal.dependencies.patches.hdfs; import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShellPatcher.java similarity index 94% rename from plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShellPatcher.java index 397b63e434ba2..ab63249f5c8e8 100644 --- a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShellPatcher.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.hdfs.patch; +package org.elasticsearch.gradle.internal.dependencies.patches.hdfs; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShutdownHookManagerPatcher.java similarity index 97% rename from plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShutdownHookManagerPatcher.java index 1235b5af9002f..4efe48a3bf72d 100644 --- a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShutdownHookManagerPatcher.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.hdfs.patch; +package org.elasticsearch.gradle.internal.dependencies.patches.hdfs; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java similarity index 97% rename from plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java index 3fb8a23be794d..3b241f7001fb4 100644 --- a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.hdfs.patch; +package org.elasticsearch.gradle.internal.dependencies.patches.hdfs; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 7df46699b79e2..eac0109bc499d 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -22,20 +22,35 @@ versions << [ 'hadoop': '3.4.1' ] +def patched = Attribute.of('patched', Boolean) + configurations { hdfsFixture2 hdfsFixture3 + compileClasspath { + attributes { + attribute(patched, true) + } + } + runtimeClasspath { + attributes { + attribute(patched, true) + } + } + testCompileClasspath { + attributes { + attribute(patched, true) + } + } + testRuntimeClasspath { + attributes { + attribute(patched, true) + } + } } dependencies { - api project(path: 'hadoop-client-api', configuration: 'default') - if (isEclipse) { - /* - * Eclipse can't pick up the shadow dependency so we point it at *something* - * so it can compile things. - */ - api project(path: 'hadoop-client-api') - } + api("org.apache.hadoop:hadoop-client-api:${versions.hadoop}") runtimeOnly "org.apache.hadoop:hadoop-client-runtime:${versions.hadoop}" implementation "org.apache.hadoop:hadoop-hdfs:${versions.hadoop}" api "com.google.protobuf:protobuf-java:${versions.protobuf}" @@ -69,6 +84,20 @@ dependencies { hdfsFixture2 project(path: ':test:fixtures:hdfs-fixture', configuration: 'shadowedHdfs2') hdfsFixture3 project(path: ':test:fixtures:hdfs-fixture', configuration: 'shadow') + + attributesSchema { + attribute(patched) + } + artifactTypes.getByName("jar") { + attributes.attribute(patched, false) + } + registerTransform(org.elasticsearch.gradle.internal.dependencies.patches.hdfs.HdfsClassPatcher) { + from.attribute(patched, false) + to.attribute(patched, true) + parameters { + matchingArtifacts = ["hadoop-client-api"] + } + } } restResources { @@ -190,6 +219,15 @@ tasks.named("thirdPartyAudit").configure { 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor', 'org.apache.hadoop.thirdparty.protobuf.MessageSchema', 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android32MemoryAccessor', - 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor' + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor', + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor', + 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm', + 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm$Slot', + 'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer', + 'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer$1', + 'org.apache.hadoop.io.nativeio.NativeIO', + 'org.apache.hadoop.service.launcher.InterruptEscalator', + 'org.apache.hadoop.service.launcher.IrqHandler', + 'org.apache.hadoop.util.SignalLogger$Handler' ) } diff --git a/plugins/repository-hdfs/hadoop-client-api/build.gradle b/plugins/repository-hdfs/hadoop-client-api/build.gradle deleted file mode 100644 index 46b0d949cdee2..0000000000000 --- a/plugins/repository-hdfs/hadoop-client-api/build.gradle +++ /dev/null @@ -1,54 +0,0 @@ -import org.gradle.api.file.ArchiveOperations - -apply plugin: 'elasticsearch.java' - -sourceSets { - patcher -} - -configurations { - thejar { - canBeResolved = true - } -} - -dependencies { - thejar("org.apache.hadoop:hadoop-client-api:${project.parent.versions.hadoop}") { - transitive = false - } - - patcherImplementation 'org.ow2.asm:asm:9.7.1' - patcherImplementation 'org.ow2.asm:asm-tree:9.7.1' -} - -def outputDir = layout.buildDirectory.dir("patched-classes") - -def patchTask = tasks.register("patchClasses", JavaExec) { - inputs.files(configurations.thejar).withPathSensitivity(PathSensitivity.RELATIVE) - inputs.files(sourceSets.patcher.output).withPathSensitivity(PathSensitivity.RELATIVE) - outputs.dir(outputDir) - classpath = sourceSets.patcher.runtimeClasspath - mainClass = 'org.elasticsearch.hdfs.patch.HdfsClassPatcher' - def thejar = configurations.thejar - doFirst { - args(thejar.singleFile, outputDir.get().asFile) - } -} - -interface InjectedArchiveOps { - @Inject ArchiveOperations getArchiveOperations() -} - -tasks.named('jar').configure { - dependsOn(configurations.thejar) - def injected = project.objects.newInstance(InjectedArchiveOps) - def thejar = configurations.thejar - from(patchTask) - from({ injected.getArchiveOperations().zipTree(thejar.singleFile) }) { - eachFile { - if (outputDir.get().file(it.relativePath.pathString).asFile.exists()) { - it.exclude() - } - } - } -} diff --git a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-LICENSE.txt b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-NOTICE.txt b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-NOTICE.txt deleted file mode 100644 index 62fc5816c996b..0000000000000 --- a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-NOTICE.txt +++ /dev/null @@ -1,2 +0,0 @@ -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java deleted file mode 100644 index 732c55929454e..0000000000000 --- a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.hdfs.patch; - -import org.objectweb.asm.ClassReader; -import org.objectweb.asm.ClassVisitor; -import org.objectweb.asm.ClassWriter; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; -import java.util.function.Function; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; - -public class HdfsClassPatcher { - static final Map> patchers = Map.of( - "org/apache/hadoop/util/ShutdownHookManager.class", - ShutdownHookManagerPatcher::new, - "org/apache/hadoop/util/Shell.class", - ShellPatcher::new, - "org/apache/hadoop/security/UserGroupInformation.class", - SubjectGetSubjectPatcher::new, - "org/apache/hadoop/security/authentication/client/KerberosAuthenticator.class", - SubjectGetSubjectPatcher::new - ); - - public static void main(String[] args) throws Exception { - String jarPath = args[0]; - Path outputDir = Paths.get(args[1]); - - try (JarFile jarFile = new JarFile(new File(jarPath))) { - for (var patcher : patchers.entrySet()) { - JarEntry jarEntry = jarFile.getJarEntry(patcher.getKey()); - if (jarEntry == null) { - throw new IllegalArgumentException("path [" + patcher.getKey() + "] not found in [" + jarPath + "]"); - } - byte[] classToPatch = jarFile.getInputStream(jarEntry).readAllBytes(); - - ClassReader classReader = new ClassReader(classToPatch); - ClassWriter classWriter = new ClassWriter(classReader, 0); - classReader.accept(patcher.getValue().apply(classWriter), 0); - - Path outputFile = outputDir.resolve(patcher.getKey()); - Files.createDirectories(outputFile.getParent()); - Files.write(outputFile, classWriter.toByteArray()); - } - } - } -} diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 857f497b29084..6360ead8126fa 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -13,13 +13,36 @@ apply plugin: 'com.gradleup.shadow' import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar +def patched = Attribute.of('patched', Boolean) configurations { - hdfs2 - hdfs3 + hdfs2 { + attributes { + attribute(patched, true) + } + } + hdfs3 { + attributes { + attribute(patched, true) + } + } consumable("shadowedHdfs2") } dependencies { + attributesSchema { + attribute(patched) + } + artifactTypes.getByName("jar") { + attributes.attribute(patched, false) + } + registerTransform(org.elasticsearch.gradle.internal.dependencies.patches.hdfs.HdfsClassPatcher) { + from.attribute(patched, false) + to.attribute(patched, true) + parameters { + matchingArtifacts = ["hadoop-common"] + } + } + compileOnly("org.apache.hadoop:hadoop-minicluster:2.8.5") api("com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}") { transitive = false