diff --git a/LICENSE-binary b/LICENSE-binary index c9d53de77981a..904a1219ced98 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -203,7 +203,10 @@ -------------------------------------------------------------------------------- This project bundles some components that are also licensed under the Apache -License Version 2.0: +License Version 2.0. +Note: some of the listed artifacts may not be included in a given build of the binary +distribution; it depends on the build options. This list intends +to be inclusive of all which may be included: hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.* (css and js files) @@ -215,6 +218,7 @@ com.aliyun:aliyun-java-sdk-core:4.5.10 com.aliyun:aliyun-java-sdk-kms:2.11.0 com.aliyun:aliyun-java-sdk-ram:3.1.0 com.aliyun:aliyun-java-sdk-sts:3.0.0 +com.aliyun:java-trace-api:0.2.11-beta.jar com.aliyun.oss:aliyun-sdk-oss:3.13.2 com.cedarsoftware:java-util:1.9.0 com.cedarsoftware:json-io:2.5.1 @@ -268,8 +272,13 @@ com.google.http-client:google-http-client-jackson2:1.46.3 com.google.http-client:google-http-client:1.46.3 com.google.j2objc:j2objc-annotations:3.0.0 com.google.oauth-client:google-oauth-client:1.37.0 +com.huaweicloud:esdk-obs-java:3.20.4.2 +com.jamesmurty.utils:java-xmlbuilder-1.2.jar com.microsoft.azure:azure-storage:7.0.0 com.nimbusds:nimbus-jose-jwt:10.4 +com.squareup.okhttp3:okhttp:jar:3.14.2 +com.squareup.okio:okio:jar:1.17.2 +com.volcengine:ve-tos-java-sdk-hadoop:2.8.9.jar com.zaxxer:HikariCP:4.0.3 commons-beanutils:commons-beanutils:1.9.4 commons-cli:commons-cli:1.9.0 @@ -346,6 +355,9 @@ io.opentelemetry:opentelemetry-sdk-logs:1.47.0 io.opentelemetry:opentelemetry-sdk-metrics:1.47.0 io.opentelemetry:opentelemetry-sdk-trace:1.47.0 io.opentelemetry.semconv:opentelemetry-semconv:1.29.0-alpha +io.opentracing:opentracing-api:0.33.0.jar +io.opentracing:opentracing-noop:0.33.0.jar +io.opentracing:opentracing-util:0.33.0.jar io.reactivex:rxjava:1.3.8 io.reactivex:rxjava-string:1.1.1 io.reactivex:rxnetty:0.4.20 @@ -373,6 +385,8 @@ org.apache.htrace:htrace-core:3.1.0-incubating org.apache.htrace:htrace-core4:4.1.0-incubating org.apache.httpcomponents:httpclient:4.5.13 org.apache.httpcomponents:httpcore:4.4.13 +org.apache.httpcomponents.client5:httpclient5:5.5 +org.apache.httpcomponents.core5:httpcore5:5.5 org.apache.kafka:kafka-clients:3.9.0 org.apache.kerby:kerb-admin:2.0.3 org.apache.kerby:kerb-client:2.0.3 @@ -496,6 +510,7 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage bootstrap v3.3.6 broccoli-asset-rev v2.4.2 broccoli-funnel v1.0.1 +cos_api-bundle-5.6.19.jar datatables v1.11.5 em-helpers v0.5.13 em-table v0.1.6 @@ -552,6 +567,7 @@ org.codehaus.mojo:animal-sniffer-annotations:1.24 org.jruby.jcodings:jcodings:1.0.13 org.jruby.joni:joni:2.1.2 org.ojalgo:ojalgo:43.0 +org.reactivestreams:reactive-streams:1.0.3.jar org.slf4j:jul-to-slf4j:1.7.36 org.slf4j:slf4j-api:1.7.36 org.slf4j:slf4j-reload4j:1.7.36 @@ -622,3 +638,8 @@ Public Domain ------------- aopalliance:aopalliance:1.0 + +Dom4J license +------------- + +org.dom4j:dom4j:2.1.4.jar \ No newline at end of file diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching index d4bfd8aaada3b..fee6b92219d4f 100755 --- a/dev-support/bin/dist-layout-stitching +++ b/dev-support/bin/dist-layout-stitching @@ -130,6 +130,10 @@ run cp -p "${ROOT}/README.txt" . run copy "${ROOT}/hadoop-common-project/hadoop-common/target/hadoop-common-${VERSION}" . run copy "${ROOT}/hadoop-common-project/hadoop-nfs/target/hadoop-nfs-${VERSION}" . run copy "${ROOT}/hadoop-common-project/hadoop-registry/target/hadoop-registry-${VERSION}" . + +# cloud connectors go into common +run copy "${ROOT}/hadoop-cloud-storage-project/hadoop-cloud-storage/target/hadoop-cloud-storage-${VERSION}" . + run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}" . run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${VERSION}" . run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-${VERSION}" . diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-cloud-storage.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-cloud-storage.xml new file mode 100644 index 0000000000000..de668d1516a4a --- /dev/null +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-cloud-storage.xml @@ -0,0 +1,55 @@ + + + hadoop-cloud-storage + + dir + + false + + + + ../../hadoop-tools/hadoop-aws/src/main/bin + /bin + 0755 + + + ./../hadoop-tools/hadoop-aws/src/main/shellprofile.d + + * + + /libexec/shellprofile.d + 0755 + + + + + + /share/hadoop/common/lib + false + runtime + false + + + org.apache.hadoop:hadoop-annotations + org.apache.hadoop.thirdparty:hadoop-shaded-guava + + + + diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml index 871694209393f..7895f4e57142f 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml @@ -57,6 +57,7 @@ **/file:/** **/SecurityAuth.audit* patchprocess/** + **/auth-keys.xml diff --git a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml index 2df2cd3d9242d..1cd932d2dd879 100644 --- a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml +++ b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml @@ -95,11 +95,7 @@ - - org.apache.hadoop - hadoop-aliyun - compile - + org.apache.hadoop hadoop-aws @@ -119,16 +115,34 @@ org.apache.hadoop hadoop-cos compile + + + * + * + + org.apache.hadoop hadoop-huaweicloud compile + + + * + * + + org.apache.hadoop hadoop-tos compile + + + * + * + + org.apache.hadoop @@ -146,4 +160,143 @@ + + + + + maven-deploy-plugin + + true + + + + org.apache.rat + apache-rat-plugin + + + + + + + + + + + dist + + false + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.hadoop + hadoop-assemblies + ${project.version} + + + + + dist + prepare-package + + single + + + false + false + ${project.artifactId}-${project.version} + + hadoop-cloud-storage + + + + + + + + + + + + huaweicloud + + false + + + + org.apache.hadoop + hadoop-huaweicloud + compile + + + com.fasterxml.jackson + * + + + + + + + + + cos + + false + + + + org.apache.hadoop + hadoop-cos + compile + + + + + + aliyun + + false + + + + org.apache.hadoop + hadoop-aliyun + compile + + + + + + adls + + false + + + + org.apache.hadoop + hadoop-azure-datalake + compile + + + + + + tos + + false + + + + org.apache.hadoop + hadoop-tos + compile + + + + + diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml index 641ad3afb1b0e..d00dc1f22c704 100755 --- a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml +++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml @@ -169,6 +169,10 @@ log4j-api org.apache.logging.log4j + + com.fasterxml.jackson.core + * + diff --git a/hadoop-cloud-storage-project/hadoop-tos/pom.xml b/hadoop-cloud-storage-project/hadoop-tos/pom.xml index 4bbaf74e0f892..1f44982bc9cc3 100644 --- a/hadoop-cloud-storage-project/hadoop-tos/pom.xml +++ b/hadoop-cloud-storage-project/hadoop-tos/pom.xml @@ -48,11 +48,27 @@ hadoop-mapreduce-client-core provided + + org.apache.httpcomponents.client5 + httpclient5 + + + org.apache.httpcomponents.core5 + httpcore5 + com.volcengine ve-tos-java-sdk-hadoop ${ve-tos-java-sdk.version} + + org.apache.httpcomponents.client5 + httpclient5 + + + org.apache.httpcomponents.core5 + httpcore5 + org.slf4j slf4j-api @@ -152,32 +168,6 @@ - - org.apache.maven.plugins - maven-shade-plugin - - - package - - shade - - - true - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - com.github.spotbugs spotbugs-maven-plugin @@ -190,4 +180,45 @@ + + + + + shade-tos + + false + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + true + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + mozilla/public-suffix-list.txt + + + + + + + + + + + + diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index ba16ed9b34e3b..2f94a34efb940 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -78,6 +78,8 @@ 4.5.13 4.4.13 + 5.5 + 5.3.6 1.7.36 @@ -865,6 +867,28 @@ httpcore ${httpcore.version} + + org.apache.httpcomponents.client5 + httpclient5 + ${httpclient5.version} + + + org.slf4j + * + + + + + org.apache.httpcomponents.core5 + httpcore5 + ${httpcore5.version} + + + org.apache.logging.log4j + * + + + commons-codec commons-codec diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml index 9fa02a0ac0027..803caf8fa0648 100644 --- a/hadoop-tools/hadoop-tools-dist/pom.xml +++ b/hadoop-tools/hadoop-tools-dist/pom.xml @@ -92,30 +92,12 @@ pom ${project.version} - - org.apache.hadoop - hadoop-aws - compile - ${project.version} - org.apache.hadoop hadoop-kafka compile ${project.version} - - org.apache.hadoop - hadoop-azure - compile - ${project.version} - - - org.apache.hadoop - hadoop-aliyun - compile - ${project.version} - org.apache.hadoop hadoop-sls @@ -127,34 +109,12 @@ ${project.version} compile - - org.apache.hadoop - hadoop-azure-datalake - compile - ${project.version} - org.apache.hadoop hadoop-fs2img compile ${project.version} - - org.apache.hadoop - hadoop-gcp - compile - ${project.version} - - - - * - * - - - diff --git a/licenses-binary/LICENSE-dom4j.txt b/licenses-binary/LICENSE-dom4j.txt new file mode 100644 index 0000000000000..1a02acb149474 --- /dev/null +++ b/licenses-binary/LICENSE-dom4j.txt @@ -0,0 +1,39 @@ +Copyright 2001-2023 © MetaStuff, Ltd. and DOM4J contributors. All Rights Reserved. + +Redistribution and use of this software and associated documentation +("Software"), with or without modification, are permitted provided +that the following conditions are met: + +1. Redistributions of source code must retain copyright + statements and notices. Redistributions must also contain a + copy of this document. + +2. Redistributions in binary form must reproduce the + above copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +3. The name "DOM4J" must not be used to endorse or promote + products derived from this Software without prior written + permission of MetaStuff, Ltd. For written permission, + please contact dom4j-info@metastuff.com. + +4. Products derived from this Software may not be called "DOM4J" + nor may "DOM4J" appear in their names without prior written + permission of MetaStuff, Ltd. DOM4J is a registered + trademark of MetaStuff, Ltd. + +5. Due credit should be given to the DOM4J Project - https://dom4j.github.io/ + +THIS SOFTWARE IS PROVIDED BY METASTUFF, LTD. AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT +NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +METASTUFF, LTD. OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE.