BIGTOP-3964: Add isal installation in bigtop toolchain and use isal library when compiling hadoop (#1142)

Author: lvkaihua

bigtop-packages/src/common/hadoop/do-component-build

@@ -119,7 +119,12 @@ if [ -z "$BUNDLE_SNAPPY" ] ; then
   [ -f /usr/lib64/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib64"
   [ -f /usr/lib/${HOSTTYPE}-linux-gnu/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib/${HOSTTYPE}-linux-gnu"
 fi
-
+if [ -z "$BUNDLE_ISAL" ] ; then
+  [ -f /usr/lib/libisal.so ] && BUNDLE_ISAL="-Dbundle.isal=true -Disal.lib=/usr/lib"
+  [ -f /usr/lib64/libisal.so ] && BUNDLE_ISAL="-Dbundle.isal=true -Disal.lib=/usr/lib64"
+  [ -f /usr/local/lib/libisal.so ] && BUNDLE_ISAL="-Dbundle.isal=true -Disal.lib=/usr/local/lib/"
+  [ -f /usr/lib/${HOSTTYPE}-linux-gnu/libisal.so ] && BUNDLE_ISAL="-Dbundle.isal=true -Disal.lib=/usr/lib/${HOSTTYPE}-linux-gnu"
+fi
 mkdir build
 mkdir build/src
 
@@ -133,7 +138,7 @@ MAVEN_OPTS+="-DskipTests -DskipITs "
 . $(dirname ${0})/maven_deploy.sh
 
 # Build artifacts
-mvn $BUNDLE_SNAPPY -Pdist -Pnative -Psrc -Pyarn-ui -Dtar ${MAVEN_OPTS} install ${EXTRA_GOALS} "$@"
+mvn $BUNDLE_SNAPPY $BUNDLE_ISAL -Pdist -Pnative -Psrc -Pyarn-ui -Dtar ${MAVEN_OPTS} install ${EXTRA_GOALS} "$@"
 mvn site site:stage ${MAVEN_OPTS} $@
 
 (cd build ; tar --strip-components=1 -xzvf  ../hadoop-dist/target/hadoop-${HADOOP_VERSION}.tar.gz)

bigtop-packages/src/common/hadoop/install_hadoop.sh

@@ -275,7 +275,7 @@ cp ${BUILD_DIR}/include/hdfs.h $PREFIX/$SYSTEM_INCLUDE_DIR/
 cp -r ${BUILD_DIR}/include/hdfspp $PREFIX/$SYSTEM_INCLUDE_DIR/
 
 cp ${BUILD_DIR}/lib/native/*.a $PREFIX/$HADOOP_NATIVE_LIB_DIR/
-for library in `cd ${BUILD_DIR}/lib/native ; ls libsnappy.so.1.* 2>/dev/null` libhadoop.so.1.0.0 libnativetask.so.1.0.0; do
+for library in `cd ${BUILD_DIR}/lib/native ; ls libsnappy.so.1.* 2>/dev/null; ls libisal.so.2.* 2>/dev/null` libhadoop.so.1.0.0 libnativetask.so.1.0.0; do
   cp ${BUILD_DIR}/lib/native/${library} $PREFIX/$HADOOP_NATIVE_LIB_DIR/
   ldconfig -vlN $PREFIX/$HADOOP_NATIVE_LIB_DIR/${library}
   ln -s ${library} $PREFIX/$HADOOP_NATIVE_LIB_DIR/${library/.so.*/}.so

bigtop_toolchain/manifests/installer.pp

@@ -25,6 +25,7 @@
   include bigtop_toolchain::user
   include bigtop_toolchain::renv
   include bigtop_toolchain::grpc
+  include bigtop_toolchain::isal
   Class['bigtop_toolchain::jdk11']->Class['bigtop_toolchain::jdk']
 
   case $::operatingsystem {

bigtop_toolchain/manifests/isal.pp

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+class bigtop_toolchain::isal {
+  require bigtop_toolchain::packages
+  $url = "https://github.com/intel/isa-l/archive/refs/tags/v2.29.0.tar.gz"
+  $isal = "v2.29.0.tar.gz"
+  $isaldir = "isa-l-2.29.0"
+  exec { "download isal":
+    cwd     => "/usr/src",
+    command => "/usr/bin/wget $url && mkdir -p $isaldir && /bin/tar -xvzf $isal -C $isaldir --strip-components=1",
+    creates => "/usr/src/$isaldir",
+  }
+  exec { "install isal":
+    cwd     => "/usr/src/$isaldir",
+    command => "/usr/src/$isaldir/autogen.sh && /usr/src/$isaldir/configure  --prefix=/usr/local && /usr/bin/make && /usr/bin/make install",
+    creates => "/usr/local/bin/$isaldir",
+    require => EXEC["download isal"],
+    timeout => 3000
+  }
+}

bigtop_toolchain/manifests/packages.pp

@@ -63,7 +63,9 @@
         "libffi-devel",
         "krb5-devel",
         "net-tools",
-        "perl-Digest-SHA"
+        "perl-Digest-SHA",
+        "nasm",
+        "yasm"
       ]
 
       if ($operatingsystem == 'Fedora' or $operatingsystemmajrelease !~ /^[0-7]$/) {
@@ -118,7 +120,9 @@
         "libffi48-devel",
         "texlive-latex-bin-bin",
         "libapr1",
-        "libapr1-devel"
+        "libapr1-devel",
+        "nasm",
+        "yasm"
       ]
       # fix package dependencies: BIGTOP-2120 and BIGTOP-2152 and BIGTOP-2471
       exec { '/usr/bin/zypper -n install  --force-resolution krb5 libopenssl-devel libxml2-devel libxslt-devel boost-devel':
@@ -149,7 +153,9 @@
       "snappy-devel",
       "libzstd-devel",
       "bzip2-devel",
-      "libffi-devel"
+      "libffi-devel",
+      "nasm",
+      "yasm"
     ] }
     /(Ubuntu|Debian)/: {
       $_pkgs = [
@@ -209,7 +215,9 @@
         "python-setuptools",
         "libffi-dev",
         "python3-dev",
-        "python2.7-dev"
+        "python2.7-dev",
+        "nasm",
+        "yasm"
       ]
       if (($operatingsystem == 'Ubuntu' and 0 <= versioncmp($operatingsystemmajrelease, '22.04'))) {
         file { '/usr/bin/python':