Update the nightly tarball scripts to support the new web site

Author: Ralph Castain

contrib/build-server/hwloc-nightly-tarball.sh

@@ -7,30 +7,36 @@
 #####
 
 # e-mail address to send results to
-[email protected]
+#[email protected]
+[email protected]
 
-# svn repository uri
-code_uri=http://svn.open-mpi.org/svn/hwloc
+# git repository URL
+code_uri=https://github.com/open-mpi/hwloc.git
+raw_uri=https://raw.github.com/open-mpi/hwloc
 
 # where to put built tarballs
-outputroot=/l/osl/www/www.open-mpi.org/software/hwloc/nightly
+outputroot=$HOME/hwloc/nightly
 
 # where to find the build script
-script_uri=${code_uri}/trunk/contrib/nightly/create_tarball.sh
-script_uri=contrib/nightly/create_tarball.sh
+script_uri=contrib/nightly/make_snapshot_tarball
 
 # The tarballs to make
 if [ $# -eq 0 ] ; then
-  dirs="/trunk /branches/v1.5 /branches/v1.4 /branches/v1.3 /branches/v1.2 /branches/v1.1 /branches/v1.0"
+    # Branches v1.6 and earlier were not updated to build nightly
+    # snapshots from git, so only check v1.7 and later
+    branches="master v1.11"
 else
-  dirs=$@
+    branches=$@
 fi
 
 # Build root - scratch space
-build_root=/home/mpiteam/hwloc/nightly-tarball-build-root
+build_root=$HOME/hwloc/nightly-tarball-build-root
 
-export PATH=$HOME/local/bin:$PATH
-export LD_LIBRARY_PATH=$HOME/local/lib:$LD_LIBRARY_PATH
+# Coverity stuff
+coverity_token=`cat $HOME/coverity/hwloc-token.txt`
+
+export PATH=$HOME_PREFIX/bin:$PATH
+export LD_LIBRARY_PATH=$HOME_PREFIX/lib:$LD_LIBRARY_PATH
 
 #####
 #
@@ -39,23 +45,27 @@ export LD_LIBRARY_PATH=$HOME/local/lib:$LD_LIBRARY_PATH
 #####
 
 # load the modules configuration
-. /etc/profile.d/modules.sh
-module use ~/modules
+. $MODULE_INIT
+module use $AUTOTOOL_MODULE
 
 # get our nightly build script
 mkdir -p $build_root
 cd $build_root
 
+pending_coverity=$build_root/tarballs-to-run-through-coverity.txt
+rm -f $pending_coverity
+touch $pending_coverity
+debug=100
+
 # Loop making them
-for dir in $dirs; do
-    # Remove leading /
-    safe_dirname=`echo $dir | sed -e 's/^\///g'`
-    # Convert remaining /'s to -'s
-    safe_dirname=`echo $safe_dirname | sed -e 's/\//-/g'`
-    # Now form a URL-specific script name
-    script=$safe_dirname-`basename $script_uri`
-
-    wget --quiet --no-check-certificate --tries=10 $code_uri/$dir/$script_uri -O $script
+for branch in $branches; do
+    # Get the last tarball version that was made
+    prev_snapshot=`cat $outputroot/$branch/latest_snapshot.txt`
+
+    # Form a URL-specific script name
+    script=$branch-`basename $script_uri`
+
+    wget --quiet --no-check-certificate --tries=10 $raw_uri/$branch/$script_uri -O $script
     if test ! $? -eq 0 ; then
         echo "wget of hwloc nightly tarball create script failed."
         if test -f $script ; then
@@ -67,16 +77,46 @@ for dir in $dirs; do
     fi
     chmod +x $script
 
-    ver=`basename $dir`
-
-    module load "autotools/hwloc-$ver"
-    module load "tex-live/hwloc-$ver"
-
-	./$script \
-		$build_root/$ver \
-		$results_addr \
-		$code_uri/$dir \
-		$outputroot/$ver >/dev/null 2>&1
+    module load "autotools/hwloc-$branch"
+   # module load "tex-live/hwloc-$branch"
+
+    ./$script \
+        $build_root/$branch \
+        $results_addr \
+        $outputroot/$branch \
+        $code_uri \
+        $branch \
+        >/dev/null 2>&1
+
+    # Did the script generate a new tarball?  If so, save it so that we can
+    # spawn the coverity checker on it afterwards.  Only for this for the
+    # master (for now).
+    latest_snapshot=`cat $outputroot/$branch/latest_snapshot.txt`
+    if test "$prev_snapshot" != "$latest_snapshot" && \
+        test "$branch" = "master"; then
+        echo "$outputroot/$branch/hwloc-$latest_snapshot.tar.bz2" >> $pending_coverity
+        echo "=== Posting tarball to open-mpi.org"
+        # tell the web server to cleanup old nightly tarballs
+        ssh -p 2222 [email protected] "git/ompi/contrib/build-server/remove-old.pl 7 public_html/software/hwloc/nightly/$branch"
+        # upload the new ones
+        scp -P 2222 $outputroot/$branch/hwloc-$latest_snapshot.tar.* [email protected]:public_html/software/hwloc/nightly/$branch/
+        scp -P 2222 $outputroot/$branch/$latest_snapshot.txt [email protected]:public_html/software/hwloc/nightly/$branch/
+        # direct the web server to regenerate the checksums
+        ssh -p 2222 [email protected] "cd public_html/software/hwloc/nightly/$branch && md5sum hwloc* > md5sums.txt"
+        ssh -p 2222 [email protected] "cd public_html/software/hwloc/nightly/$branch && sha1sum hwloc* > sha1sums.txt"
+    fi
 
-    module unload autotools tex-live
+    module unload autotools
 done
+
+# If we had any new snapshots to send to coverity, process them now
+
+#for tarball in `cat $pending_coverity`; do
+#    /home/common/mpiteam/scripts/hwloc-nightly-coverity.pl \
+#        --filename=$tarball \
+#        --coverity-token=$coverity_token \
+#        --verbose \
+#        --logfile-dir=$HOME/coverity \
+#        --make-args="-j8"
+#done
+rm -f $pending_coverity

contrib/build-server/openmpi-nightly-tarball.sh

@@ -15,32 +15,33 @@ master_raw_uri=https://raw.github.com/open-mpi/ompi
 release_code_uri=https://github.com/open-mpi/ompi-release.git
 release_raw_uri=https://raw.github.com/open-mpi/ompi-release
 
-# where to put built tarballs
-outputroot=/l/osl/www/www.open-mpi.org/nightly
+# where to put built tarballs - needs to be
+# adjusted to match your site!
+outputroot=$HOME/openmpi/nightly
 
 # where to find the build script
 script_uri=contrib/nightly/create_tarball.sh
 
 # helper scripts dir
-script_dir=/u/mpiteam/scripts
+script_dir=$HOME/scripts
 
 # The tarballs to make
 if [ $# -eq 0 ] ; then
-    # We're no longer ever checking the 1.0 - 1.6 branches anymore
-    branches="master v1.8 v1.10 v2.x"
+    # We're no longer ever checking the 1.0 - 1.8 branches anymore
+    branches="master v1.10 v2.x"
 else
     branches=$@
 fi
 
 # Build root - scratch space
-build_root=/home/mpiteam/openmpi/nightly-tarball-build-root
+build_root=$HOME/openmpi/nightly-tarball-build-root
 
 # Coverity stuff
 coverity_token=`cat $HOME/coverity/openmpi-token.txt`
 coverity_configure_args="--enable-debug --enable-mpi-fortran --enable-mpi-java --enable-oshmem --enable-oshmem-fortran --enable-oshmem-java --with-mxm=/opt/mellanox/mxm --with-psm --with-usnic --with-libfabric=/u/mpiteam/libfabric-current/install"
 
-export PATH=$HOME/local/bin:$PATH
-export LD_LIBRARY_PATH=$HOME/local/lib:$LD_LIBRARY_PATH
+export PATH=$HOME_PREFIX/bin:$PATH
+export LD_LIBRARY_PATH=$HOME_PREFIX/lib:$LD_LIBRARY_PATH
 
 #####
 #
@@ -49,8 +50,8 @@ export LD_LIBRARY_PATH=$HOME/local/lib:$LD_LIBRARY_PATH
 #####
 
 # load the modules configuration
-. /etc/profile.d/modules.sh
-module use ~/modules
+. $MODULE_INIT
+module use $AUTOTOOL_MODULE
 
 # get our nightly build script
 mkdir -p $build_root
@@ -115,17 +116,27 @@ for branch in $branches; do
         test "$branch" = "master"; then
         echo "=== Saving output for a Coverity run"
         echo "$outputroot/$branch/openmpi-$latest_snapshot.tar.bz2" >> $pending_coverity
+        # tell the web server to cleanup old nightly tarballs
+        ssh -p 2222 [email protected] "git/ompi/contrib/build-server/remove-old.pl 7 public_html/nightly/$branch"
+        # upload the new ones
+        scp -P 2222 $outputroot/$branch/openmpi-$latest_snapshot.tar.* [email protected]:public_html/nightly/$branch/
+        scp -P 2222 $outputroot/$branch/latest_snapshot.txt [email protected]:public_html/nightly/$branch/
+        # direct the web server to regenerate the checksums
+        ssh -p 2222 [email protected] "cd public_html/nightly/$branch && md5sum openmpi* > md5sums.txt"
+        ssh -p 2222 [email protected] "cd public_html/nightly/$branch && sha1sum openmpi* > sha1sums.txt"
     else
         echo "=== NOT saving output for a Coverity run"
     fi
 
     # Failed builds are not removed.  But if a human forgets to come
     # in here and clean up the old failed builds, we can accumulate
-    # many over time.  So remove any old failed bbuilds that are over
+    # many over time.  So remove any old failed builds that are over
     # 4 weeks old.
     ${script_dir}/remove-old.pl 7 $build_root/$branch
+
 done
 
+
 # If we had any new snapshots to send to coverity, process them now
 
 for tarball in `cat $pending_coverity`; do

contrib/build-server/pmix-nightly-coverity.pl

@@ -0,0 +1,160 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use Getopt::Long;
+use File::Temp qw/ tempfile tempdir /;
+use File::Basename;
+
+my $coverity_project = "open-mpi%2Fpmix";
+
+my $filename_arg;
+my $coverity_token_arg;
+my $dry_run_arg = 0;
+my $verbose_arg = 0;
+my $debug_arg = 0;
+my $logfile_dir_arg = "/tmp";
+my $configure_args = "";
+my $make_args = "-j 32";
+my $help_arg = 0;
+
+&Getopt::Long::Configure("bundling");
+my $ok = Getopt::Long::GetOptions("filename=s" => \$filename_arg,
+                                  "coverity-token=s" => \$coverity_token_arg,
+                                  "logfile-dir=s" => \$logfile_dir_arg,
+                                  "configure-args=s" => \$configure_args,
+                                  "make-args=s" => \$make_args,
+                                  "dry-run!" => \$dry_run_arg,
+                                  "verbose!" => \$verbose_arg,
+                                  "debug!" => \$debug_arg,
+                                  "help|h" => \$help_arg);
+
+$ok = 0
+    if (!defined($filename_arg));
+$ok = 0
+    if (!defined($coverity_token_arg));
+if (!$ok || $help_arg) {
+    print "Usage: $0 --filename=FILENAME --coverity-token=TOKEN [--dry-run] [--verbose] [--help]\n";
+    exit($ok);
+}
+
+die "Cannot read $filename_arg"
+    if (! -r $filename_arg);
+
+$verbose_arg = 1
+    if ($debug_arg);
+
+######################################################################
+
+sub verbose {
+    print @_
+        if ($verbose_arg);
+}
+
+# run a command and save the stdout / stderr
+sub safe_system {
+    my $allowed_to_fail = shift;
+    my $cmd = shift;
+    my $stdout_file = shift;
+
+    # Redirect stdout if requested or not verbose
+    if (defined($stdout_file)) {
+        $stdout_file = "$logfile_dir_arg/$stdout_file";
+        unlink($stdout_file);
+        $cmd .= " >$stdout_file";
+    } elsif (!$debug_arg) {
+        $cmd .= " >/dev/null";
+    }
+    $cmd .= " 2>&1";
+
+    my $rc = system($cmd);
+    if (0 != $rc && !$allowed_to_fail) {
+        # If we die/fail, ensure to change out of the temp tree so
+        # that it can be removed upon exit.
+        chdir("/");
+        die "Command $cmd failed: exit status $rc";
+    }
+    system("cat $stdout_file")
+        if ($debug_arg && defined($stdout_file) && -f $stdout_file);
+}
+
+######################################################################
+
+# Make an area to work
+
+my $dir = tempdir(CLEANUP => 1);
+chdir($dir);
+verbose "*** Working in $dir\n";
+
+######################################################################
+
+# Get the coverity tool, put it in our path.
+
+my $cdir = "/home/common/mpiteam/coverity";
+safe_system(0, "mkdir $cdir")
+    if (! -d $cdir);
+
+# Optimization: the tool is pretty large.  If our local copy is less
+# than a day old, just use that without re-downloading.
+my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
+    $atime,$mtime,$ctime,$blksize,$blocks) =
+    stat("$cdir/coverity_tool.tgz");
+my $now = time();
+if (!defined($mtime) || $mtime < $now - 24*60*60) {
+    verbose "*** Downloading new copy of the coverity tool\n";
+    safe_system(0, "wget https://scan.coverity.com/download/linux-64 --post-data \"token=$coverity_token_arg\&project=$coverity_project\" -O coverity_tool.tgz");
+    safe_system(0, "cp coverity_tool.tgz $cdir");
+}
+
+verbose "*** Expanding coverity tool tarball\n";
+safe_system(0, "tar xf $cdir/coverity_tool.tgz");
+opendir(my $dh, ".") ||
+    die "Can't opendir .";
+my @files = grep { /^cov/ && -d "./$_" } readdir($dh);
+closedir($dh);
+
+my $cov_dir = "$dir/$files[0]/bin";
+$ENV{PATH} = "$cov_dir:$ENV{PATH}";
+
+######################################################################
+
+# Expand the PMIX tarball, build it
+
+verbose "*** Extracting PMIX tarball\n";
+safe_system(0, "tar xf $filename_arg");
+my $tarball_filename = basename($filename_arg);
+$tarball_filename =~ m/^pmix-(.+)\.tar.+$/;
+my $pmix_ver = $1;
+chdir("pmix-$pmix_ver");
+
+verbose "*** Configuring PMIX tarball\n";
+safe_system(0, "./configure $configure_args", "configure");
+
+verbose "*** Building PMIX tarball\n";
+safe_system(0, "cov-build --dir cov-int make $make_args", "cov-build");
+
+# Tar up the Coverity results
+verbose "*** Tarring up results\n";
+safe_system(0, "tar jcf $pmix_ver-analyzed.tar.bz2 cov-int");
+
+# If not dry-run, submit to Coverity
+if ($dry_run_arg) {
+    verbose "*** Would have submitted, but this is a dry run\n";
+} else {
+    verbose "*** Submitting results\n";
+    safe_system(0, "curl --form token=$coverity_token_arg " .
+                "--form email=rhc\@open-mpi.org " .
+                "--form file=\@$pmix_ver-analyzed.tar.bz2 " .
+                "--form version=$pmix_ver " .
+                "--form description=nightly-master " .
+                "https://scan.coverity.com/builds?project=$coverity_project",
+                "coverity-submit");
+}
+
+verbose("*** All done\n");
+
+# Chdir out of the tempdir so that it can be removed
+chdir("/");
+
+exit(0);