Merge pull request #171 from brianhlin/SOFTWARE-3783.osg-35

matyasselmeci · web-flow · commit f4521dbadc56 · 2019-08-16T09:54:54.000-05:00
Initial OSG 3.5 support, including Travis tests (SOFTWARE-3783)
diff --git a/.travis.yml b/.travis.yml
@@ -1,11 +1,22 @@
 sudo: required
+
+git:
+  depth: false
+  quiet: true
+
 env:
-  matrix:
-   - OS_TYPE=centos OS_VERSION=6 PACKAGES=osg-ce-condor,rsv,vo-client-lcmaps-voms,llrun
-   - OS_TYPE=centos OS_VERSION=6 PACKAGES=osg-gridftp,rsv,lcmaps,lcmaps-db-templates,vo-client-lcmaps-voms,rsv,llrun
-   - OS_TYPE=centos OS_VERSION=7 PACKAGES=osg-ce-condor,rsv,vo-client-lcmaps-voms,llrun,singularity,osg-oasis
-   - OS_TYPE=centos OS_VERSION=7 PACKAGES=osg-gridftp,rsv,lcmaps,lcmaps-db-templates,vo-client-lcmaps-voms,rsv,llrun,singularity,osg-oasis
-   - OS_TYPE=centos OS_VERSION=7 PACKAGES=xrootd,xrootd-client,xrootd-multiuser,globus-proxy-utils
+  - OSG_RELEASE=3.4 OS_VERSION=6 PKG_SET=htcondor-ce
+  - OSG_RELEASE=3.4 OS_VERSION=6 PKG_SET=gridftp
+  - OSG_RELEASE=3.4 OS_VERSION=7 PKG_SET=htcondor-ce
+  - OSG_RELEASE=3.4 OS_VERSION=7 PKG_SET=gridftp
+  - OSG_RELEASE=3.4 OS_VERSION=7 PKG_SET=singularity
+  - OSG_RELEASE=3.4 OS_VERSION=7 PKG_SET=stashcache
+  - OSG_RELEASE=3.4 OS_VERSION=7 PKG_SET=xrootd
+  - OSG_RELEASE=3.5 OS_VERSION=7 PKG_SET=htcondor-ce
+  - OSG_RELEASE=3.5 OS_VERSION=7 PKG_SET=gridftp
+  - OSG_RELEASE=3.5 OS_VERSION=7 PKG_SET=singularity
+  - OSG_RELEASE=3.5 OS_VERSION=7 PKG_SET=stashcache
+  - OSG_RELEASE=3.5 OS_VERSION=7 PKG_SET=xrootd
 
 services:
   - docker
diff --git a/osgtest/library/core.py b/osgtest/library/core.py
@@ -91,6 +91,7 @@ class PackageVersion:
     def __init__(self, pkgname):
         e,n,v,r,a = get_package_envra(pkgname)
         self.evr = e,v,r
+        self.version = v
 
     def __repr__(self):
         return "%s:%s-%s" % self.evr
@@ -629,16 +630,16 @@ def el_release():
 
 def osg_release(update_state=False):
     """
-    Return the version of osg-release. If the query fails, the test module fails.
+    Return a PackageVersion instance of osg-release. If the query fails, the test module fails.
     """
     if not update_state and 'general.osg_release_ver' in state:
         return state['general.osg_release_ver']
     try:
-        _, _, osg_release_ver, _, _ = get_package_envra('osg-release')
+        release = PackageVersion('osg-release')
     except OSError:
-        _, _, osg_release_ver, _, _ = get_package_envra('osg-release-itb')
-    state['general.osg_release_ver'] = osg_release_ver
-    return osg_release_ver
+        release = PackageVersion('osg-release-itb')
+    state['general.osg_release_ver'] = release
+    return release
 
 
 def get_hostname():
@@ -750,10 +751,10 @@ def test_bar_34_only(self):
     releases = map(str, releases)  # convert float args to str
     def osg_release_decorator(fn):
         def run_fn_if_osg_release_ok(*args, **kwargs):
-            if osg_release() in releases:
+            if osg_release().version in releases:
                 return fn(*args, **kwargs)
             else:
-                msg = "excluding for OSG %s" % osg_release()
+                msg = "excluding for OSG %s" % osg_release().version
                 raise osgunittest.ExcludedException(msg)
         return run_fn_if_osg_release_ok
     return osg_release_decorator
diff --git a/osgtest/tests/special_install.py b/osgtest/tests/special_install.py
@@ -30,10 +30,15 @@ def test_02_install_packages(self):
         fail_msg = ''
         pkg_repo_dict = OrderedDict((x, core.options.extrarepos) for x in core.options.packages)
 
-        # HACK: Install slurm out of development if we're running 'All' tests.
+        # HACK: Install Slurm and osg-tested-internal out of development-like repos.
         # SOFTWARE-1733 may one day give us a generalized solution.
+        if core.osg_release() > '3.4':
+            devops_repo = 'devops-itb'
+        else:
+            devops_repo = 'osg-development'
+
         if 'osg-tested-internal' in pkg_repo_dict or 'slurm' in pkg_repo_dict:
-            pkg_repo_dict.update(dict((x, ['osg-development']) for x in core.SLURM_PACKAGES))
+            pkg_repo_dict.update(dict((x, [devops_repo]) for x in core.SLURM_PACKAGES + ['osg-tested-internal']))
 
         # HACK: Install x509-scitokens-issuer-client out of development (SOFTWARE-3649)
         if 'xrootd-scitokens' in pkg_repo_dict:
diff --git a/travis-ci/gridftp.packages b/travis-ci/gridftp.packages
@@ -0,0 +1,2 @@
+osg-gridftp
+osg-gridftp-xrootd
diff --git a/travis-ci/htcondor-ce.packages b/travis-ci/htcondor-ce.packages
@@ -0,0 +1,2 @@
+osg-ce-condor
+globus-proxy-utils
diff --git a/travis-ci/setup_tests.sh b/travis-ci/setup_tests.sh
@@ -1,23 +1,47 @@
-#!/bin/sh -xe
+#!/bin/bash -xe
 
 # This script starts docker and systemd (if el7)
 
 # Run tests in Container
 # We use `--privileged` for cgroup compatability, which seems to be enabled by default in HTCondor 8.6.x
+
+ENV_FILE=/tmp/travis.env
+for vars in {OSG_RELEASE,OS_VERSION,PKG_SET}; do
+    echo "$vars=${!vars}" >> $ENV_FILE
+done
+
 if [ "${OS_VERSION}" = "6" ]; then
 
-sudo docker run --privileged --rm=true -v /sys/fs/cgroup:/sys/fs/cgroup -v `pwd`:/osg-test:rw centos:centos${OS_VERSION} /bin/bash -c "bash -xe /osg-test/travis-ci/test_inside_docker.sh ${OS_VERSION} ${PACKAGES}"
+    sudo docker run --privileged \
+         --rm=true \
+         --volume=/sys/fs/cgroup:/sys/fs/cgroup \
+         --volume=`pwd`:/osg-test:rw \
+         --env-file=$ENV_FILE \
+         centos:centos${OS_VERSION} \
+         /bin/bash -c "bash -xe /osg-test/travis-ci/test_inside_docker.sh"
 
 elif [ "${OS_VERSION}" = "7" ]; then
 
-docker run --privileged -d -ti -e "container=docker"  -v /sys/fs/cgroup:/sys/fs/cgroup -v `pwd`:/osg-test:rw  centos:centos${OS_VERSION}   /usr/sbin/init
-DOCKER_CONTAINER_ID=$(docker ps | grep centos | awk '{print $1}')
-docker logs $DOCKER_CONTAINER_ID
-docker exec -ti $DOCKER_CONTAINER_ID /bin/bash -xec "bash -xe /osg-test/travis-ci/test_inside_docker.sh ${OS_VERSION} ${PACKAGES};
+    echo "container=docker" >> $ENV_FILE
+    docker run --privileged \
+           --detach=true \
+           --tty \
+           --interactive=true \
+           --volume=/sys/fs/cgroup:/sys/fs/cgroup \
+           --volume `pwd`:/osg-test:rw \
+           --env-file=$ENV_FILE \
+           centos:centos${OS_VERSION} \
+           /usr/sbin/init
+    DOCKER_CONTAINER_ID=$(docker ps | grep centos | awk '{print $1}')
+    docker logs $DOCKER_CONTAINER_ID
+    docker exec --tty=true \
+           --interactive \
+           $DOCKER_CONTAINER_ID \
+           /bin/bash -xec "bash -xe /osg-test/travis-ci/test_inside_docker.sh;
   echo -ne \"------\nEND OSG-TEST TESTS\n\";"
-docker ps -a
-docker stop $DOCKER_CONTAINER_ID
-docker rm -v $DOCKER_CONTAINER_ID
+    docker ps -a
+    docker stop $DOCKER_CONTAINER_ID
+    docker rm -v $DOCKER_CONTAINER_ID
 
 fi
 
diff --git a/travis-ci/singularity.packages b/travis-ci/singularity.packages
@@ -0,0 +1,2 @@
+osg-oasis
+singularity
diff --git a/travis-ci/stashcache.packages b/travis-ci/stashcache.packages
@@ -0,0 +1,3 @@
+stashcache-client
+stash-cache
+stash-origin
diff --git a/travis-ci/test_inside_docker.sh b/travis-ci/test_inside_docker.sh
@@ -1,7 +1,12 @@
 #!/bin/sh -xe
 
-OS_VERSION=$1
-PACKAGES=$2
+if [[ "$OSG_RELEASE" == "3.5" ]]; then
+    devops_repo='--enablerepo=devops-itb'
+    extra_repos='--extra-repo=osg-development'
+else
+    devops_repo=''
+    extra_repos=''
+fi
 
 ls -l /home
 
@@ -10,10 +15,11 @@ yum -y clean all
 yum -y clean expire-cache
 
 # First, install all the needed packages.
-rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-${OS_VERSION}.noarch.rpm
+rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-"$OS_VERSION".noarch.rpm
 
 yum -y install yum-plugin-priorities
-rpm -Uvh https://repo.opensciencegrid.org/osg/3.4/osg-3.4-el${OS_VERSION}-release-latest.rpm
+
+rpm -Uvh https://repo.opensciencegrid.org/osg/"$OSG_RELEASE"/osg-"$OSG_RELEASE"-el"$OS_VERSION"-release-latest.rpm
 yum -y install make git openssl rpm-build
 
 # Prepare the RPM environment
@@ -34,7 +40,9 @@ popd
 rpmbuild --define '_topdir /tmp/rpmbuild' -ba /tmp/rpmbuild/SPECS/osg-test.spec
 
 # After building the RPM, try to install it
-yum localinstall -y /tmp/rpmbuild/RPMS/noarch/osg-test*
+yum localinstall -y \
+    $devops_repo \
+    /tmp/rpmbuild/RPMS/noarch/osg-test*
 
 # HTCondor really, really wants a domain name.  Fake one.
 sed /etc/hosts -e "s/`hostname`/`hostname`.unl.edu `hostname`/" > /etc/hosts.new
@@ -55,6 +63,18 @@ cp /etc/condor/config.d/99-local.conf /etc/condor-ce/config.d/99-local.conf
 export _condor_CONDOR_CE_TRACE_ATTEMPTS=60
 
 # Ok, do actual testing
-INSTALL_STR="--install ${PACKAGES//,/ --install }"
+
+install_str=''
+while read package; do
+    install_str="$install_str --install $package"
+done < /osg-test/travis-ci/"$PKG_SET".packages
+
 echo "------------ OSG Test --------------"
-osg-test -vad --hostcert --no-cleanup ${INSTALL_STR}
+
+osg-test --verbose \
+         --add-user \
+         --dump-output \
+         --hostcert \
+         --no-cleanup \
+         ${extra_repos} \
+         ${install_str}
diff --git a/travis-ci/xrootd.packages b/travis-ci/xrootd.packages
@@ -0,0 +1,10 @@
+xrootd
+xrootd-multiuser
+xrootd-lcmaps
+xrootd-client
+xrootd-fuse
+xrootd-scitokens
+lcmaps-db-templates
+vo-client-lcmaps-voms
+globus-proxy-utils
+osg-gridftp-xrootd

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+osg-ce-condor`
	`2`	`+globus-proxy-utils`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+stashcache-client`
	`2`	`+stash-cache`
	`3`	`+stash-origin`