Merge remote-tracking branch 'upstream/master' into bigtop3

Author: Akanksha-kedia

.asf.yaml

@@ -16,6 +16,20 @@
 # limitations under the License.
 # 
 
+github:
+  description: "Bigtop is an Apache Foundation project for Infrastructure Engineers and Data Scientists looking for comprehensive packaging, testing, and configuration of the leading open source big data components."
+  homepage: https://bigtop.apache.org
+  labels:
+    - java
+    - big-data
+    - bigtop
+  enabled_merge_buttons:
+    squash: true
+    merge: false
+    rebase: false
+notifications:
+  commits: commits@bigtop.apache.org
+
 staging:
   whoami: asf-staging
 

CHANGES.txt

@@ -17,6 +17,43 @@ Bigtop Changelog
 
 * Release 3.3.0-SNAPSHOT (unreleased)
 
+
+* Release 3.2.1 (2023-07-05)
+
+Release Notes - Bigtop - Version 3.2.1
+
+** Sub-task
+    * [BIGTOP-3920] - Upgrade Hadoop to 3.3.5
+
+** Bug
+    * [BIGTOP-3897] - Failed to deploy gpdb on Fedora-36 for arm64/ppc64le
+    * [BIGTOP-3901] - Fix failure of Kerberos deployment on Rocky Linux 8 and Ubuntu 22.04
+    * [BIGTOP-3905] - Update Bigtop-stack-provisioner repo for the latest 3.2.0 release
+    * [BIGTOP-3926] - Fix build failure of Hive against Hadoop 3.3.5
+    * [BIGTOP-3927] - Fix build failure of HBase against Hadoop 3.3.5
+    * [BIGTOP-3929] - Fix build failure of Flink due to transitive dependency on pentaho-aggdesigner-algorithm
+    * [BIGTOP-3930] - Fix deployment failure on Ubuntu 22.04 due to unavailable dependency on python
+    * [BIGTOP-3945] - Fix YCSB build issues for deprecated mapr repo url
+    * [BIGTOP-3946] - Fix build failure of Ambari due to expired certificate of bower repo
+    * [BIGTOP-3947] - Fix build failure of Oozie due to insufficient test scope dependency on hive-service
+    * [BIGTOP-3948] - Fix build failure of Tez due to expired certificate of bower repo
+    * [BIGTOP-3949] - Fix dependency convergence error of Zeppelin
+    * [BIGTOP-3952] - Fix toolchain to download Maven from archive.apache.org instead of dlcdn.apache.org
+    * [BIGTOP-3954] - Fix the version definition in bigtop.bom so that `./gradlew setversion` works
+    * [BIGTOP-3956] - hadoop-deb FAILED on project hadoop-yarn-applications-catalog-webapp
+    * [BIGTOP-3961] - Fix version mismatch of hbase-client used by hadoop-yarn-server-timelineservice-hbase
+    * [BIGTOP-3962] - Fix setup failure of Ant due to unresolved gpg public key
+    * [BIGTOP-3966] - Make hive  3.1.3  compatible with hadoop 3.3.5
+    * [BIGTOP-3970] - Fix build failure of Zeppelin due to missing grpc-java-1.26.0 on ppc64le
+    * [BIGTOP-3972] - Fix startup failure of Oozie due to permission error on reading ssl-client.xml
+
+
+** Improvement
+    * [BIGTOP-3900] - Migrate secure DataNode settings from jsvc to SASL
+    * [BIGTOP-3915] - soft link error about jar of hadoop in {hbase_home}/lib
+    * [BIGTOP-3951] - Fix toolchain to work with the recent version of facter
+
+
 * Release 3.2.0 (2023-01-17)
 
 Release Notes - Bigtop - Version 3.2.0

README.md

@@ -185,7 +185,13 @@ __On all systems, Building Apache Bigtop requires certain set of tools__
   By default, the installation path of components will follow [Filesystem Hierarchy Standard](https://refspecs.linuxfoundation.org/FHS_3.0/fhs/index.html), but you can use `-PparentDir=/path` to add prefix to the path, in this example, prefix `/path/${bigtop_version}` will be added
 
   To avoid download wrong version of components when you have multiple repositories configured on your machine, you can use `-PpkgSuffix` to add bigtop version to package suffix, for example, package name `zookeeper` will be changed to `zookeeper_3_2_0` when using `Bigtop-3.2.0`
+
+* __Enabling Parallel Build for packages(BIGTOP-4044)__
+
+  Apache Bigtop defaults to non-parallel builds. Use -PbuildThreads with a number or a combination of numbers followed by 'C', such as '-PbuildThreads=2' or '-PbuildThreads=2C', to specify the number of CPU cores for Maven's parallel build feature, which can speed up the compilation process for compatible components.
 
+  Consult the bigtop.bom file to verify component compatibility with parallel builds; those marked with maven_parallel_build = true support this option.
+
 * __Building local YUM/APT repositories__ : `gradle [yum|apt]`
 
 * __Recommended build environments__
@@ -195,10 +201,10 @@ __On all systems, Building Apache Bigtop requires certain set of tools__
   environment configured and cached. All currently supported OSes could be pulled
   from official Bigtop repository at https://hub.docker.com/r/bigtop/slaves/tags/
 
-  To build a component (bigtop-groovy) for a particular OS (ubuntu-20.04) you can
+  To build a component (bigtop-groovy) for a particular OS (ubuntu-22.04) you can
   run the following from a clone of Bigtop workspace (assuming your system has
   Docker engine setup and working)
-  ```docker run --rm -u jenkins:jenkins -v `pwd`:/ws --workdir /ws bigtop/slaves:trunk-ubuntu-20.04
+  ```docker run --rm -u jenkins:jenkins -v `pwd`:/ws --workdir /ws bigtop/slaves:trunk-ubuntu-22.04
   bash -l -c './gradlew allclean ; ./gradlew bigtop-groovy-pkg'```
 
 For Developers: Building and modifying the web site

bigtop-bigpetstore/bigpetstore-mapreduce/pom.xml

@@ -33,7 +33,7 @@
 		<datanucleus.version>3.2.2</datanucleus.version>
 		<datanucleus.jpa.version>3.2.1</datanucleus.jpa.version>
 		<bonecp.version>1.0.0-SNAPSHOT.RELEASE</bonecp.version>
-		<derby.version>10.10.1.1</derby.version>
+		<derby.version>10.17.1.0</derby.version>
 		<plugin.surefire.version>2.17</plugin.surefire.version>
 	</properties>
 

bigtop-ci/build.sh

@@ -17,7 +17,7 @@
 BIGTOP_HOME=`cd $(dirname $0)/.. && pwd`
 
 usage() {
-    echo "usage build.sh --prefix trunk|1.4.0|1.3.0|... --os debian-10|centos-7|... --target hadoop|tez|..."
+    echo "usage build.sh --prefix trunk|1.4.0|1.3.0|... --os debian-11|centos-7|... --target hadoop|tez|..."
     echo "   [--nexus] [--preferred-java-version 8|11] [--mvn-cache-volume true|false] [--docker-run-option ...]"
     exit 1 # unknown option
 }

bigtop-ci/jenkins/jobsCreator.groovy

@@ -18,7 +18,7 @@
 // FIXME: it would be nice to extract the following from bigtop.mk on the fly
 def bigtopComponents = ["bigtop-groovy", "bigtop-jsvc", "bigtop-utils",
                         "zookeeper", "hadoop", "hbase", "hive",
-                        "oozie", "phoenix",
+                        "phoenix",
                         "solr", "spark", "alluxio"]
 // FIXME: it would be nice to extract the following from some static configuration file
 def targetOS = ["fedora-20",  "opensuse-12.3",  "ubuntu-14.04"]

bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml

@@ -16,8 +16,8 @@
 # "$components" list. If $components isn't set then everything in the stack will
 # be installed as usual. Otherwise only a specified list will be set
 # Possible elements:
-# hadoop,yarn,hbase,alluxio,flink,solrcloud,spark,oozie,hcat,httpfs,
-# pig,hive,zookeeper,ycsb
+# hadoop,yarn,hbase,alluxio,flink,solr,spark,hcat,httpfs,
+# pig,hive,zookeeper
 # Example (to deploy only HDFS and YARN server and gateway parts)
 # This can be a comma-separated list or an array.
 #hadoop_cluster_node::cluster_components:
@@ -146,8 +146,6 @@ solr::server::kerberos_realm: "%{hiera('kerberos::site::realm')}"
 # Default but needed here to make sure, hue uses the same port
 solr::server::port: "8983"
 
-hadoop_oozie::server::kerberos_realm: "%{hiera('kerberos::site::realm')}"
-
 hcatalog::server::kerberos_realm: "%{hiera('kerberos::site::realm')}"
 hcatalog::webhcat::server::kerberos_realm: "%{hiera('kerberos::site::realm')}"
 
@@ -173,7 +171,6 @@ hadoop_zookeeper::server::kerberos_realm: "%{hiera('kerberos::site::realm')}"
 bigtop::hadoop_rm_http_port: "8088"
 bigtop::hadoop_rm_proxy_port: "8088"
 bigtop::hbase_thrift_port: "9090"
-bigtop::hadoop_oozie_port: "11000"
 
 hadoop_hive::common_config::hbase_zookeeper_quorum: "%{hiera('hadoop_hbase::common_config::zookeeper_quorum')}"
 hadoop_hive::common_config::kerberos_realm: "%{hiera('kerberos::site::realm')}"
@@ -221,8 +218,5 @@ gpdb::common::db_base_dir: "/data_gp"
 gpdb::common::master_db_port: "5432"
 gpdb::common::segment_db_port_prefix: "4000"
 
-# Ambari
-ambari::agent::server_host: "%{hiera('bigtop::hadoop_head_node')}"
-
 # Ranger
 ranger::admin::admin_password: "Admin01234"

bigtop-deploy/puppet/hieradata/bigtop/repo.yaml

@@ -1,5 +1,5 @@
 bigtop::bigtop_repo_gpg_check: true
-bigtop::bigtop_repo_apt_key: "B60B04798ACB6352CA7CC41B48FE45F9CB1784BB"
+bigtop::bigtop_repo_apt_key: "36243EECE206BB0D"
 bigtop::bigtop_repo_yum_key_url: "https://dlcdn.apache.org/bigtop/KEYS"
-bigtop::bigtop_repo_default_version: "3.2.0"
+bigtop::bigtop_repo_default_version: "3.2.1"
 

bigtop-deploy/puppet/hieradata/site.yaml

@@ -13,21 +13,18 @@ hadoop::hadoop_storage_dirs:
 
 #hadoop_cluster_node::cluster_components:
 #  - alluxio
-#  - ambari
 #  - flink
 #  - gpdb
 #  - hbase
 #  - hcat
 #  - hive
 #  - httpfs
 #  - mapred-app
-#  - oozie
-#  - solrcloud
+#  - solr
 #  - spark
 #  - spark-standalone
 #  - tez
 #  - yarn
-#  - ycsb
 #  - zookeeper
 
 #bigtop::roles_enabled: false

bigtop-deploy/puppet/manifests/bigtop_repo.pp

@@ -70,8 +70,8 @@
       # BIGTOP-3343. This is a JDK-related stuff, so it should be in jdk.pp ordinarily.
       # But it looks like that this definition must be here to avoid cyclic resource dependencies.
       if ($operatingsystem == 'Debian' and 0 <= versioncmp($operatingsystemrelease, "10")) {
-        apt::source { 'adoptopenjdk':
-          location => 'https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/',
+        apt::source { 'adoptium':
+          location => 'https://packages.adoptium.net/artifactory/deb/',
         }
       }
 
@@ -86,9 +86,9 @@
 
         # BIGTOP-3343. This is a JDK-related stuff, but it's here for the same reason described above.
         if ($operatingsystem == 'Debian' and 0 <= versioncmp($operatingsystemrelease, "10")) {
-          apt::key { "add_adoptopenjdk_key":
-            id => "8ED17AF5D7E675EB3EE3BCE98AC3B29174885C03",
-            source => "https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public",
+          apt::key { "add_adoptium_key":
+            id => "3B04D753C9050D9A5D343F39843C48A565F8F04B",
+            source => "https://packages.adoptium.net/artifactory/api/gpg/key/public",
           }
         }
       } else {