Merge pull request #191 from opensciencegrid/el8

EL8/Python 3 support

Author: matyasselmeci

Makefile

@@ -13,6 +13,8 @@ VERSION := 3.0.0
 # Other configuration: May need to change for a release
 # ------------------------------------------------------------------------------
 
+PYTHON = python
+
 SBIN_FILES := osg-test osg-test-log-viewer
 INSTALL_SBIN_DIR := usr/sbin
 
@@ -50,7 +52,7 @@ TARBALL_DIR := $(PACKAGE)-$(VERSION)
 TARBALL_NAME := $(PACKAGE)-$(VERSION).tar.gz
 UPSTREAM := /p/vdt/public/html/upstream
 UPSTREAM_DIR := $(UPSTREAM)/$(PACKAGE)/$(VERSION)
-INSTALL_PYTHON_DIR := $(shell python -c 'from distutils.sysconfig import get_python_lib; print get_python_lib()')
+INSTALL_PYTHON_DIR := $(shell $(PYTHON) -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')
 
 
 # ------------------------------------------------------------------------------

osg-test

@@ -1,9 +1,12 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 from __future__ import print_function
 from distutils.sysconfig import get_python_lib
 from optparse import OptionParser
-import ConfigParser
+try:
+    import ConfigParser
+except ImportError:
+    import configparser as ConfigParser
 import os
 import re
 import signal

osg-test-log-viewer

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 """View the output from osg-test in a structured way"""
 from __future__ import print_function
 

osgtest/library/core.py

@@ -20,6 +20,10 @@
 import traceback
 import socket
 import signal
+try:
+    from shlex import quote as shell_quote
+except ImportError:
+    from pipes import quote as shell_quote
 
 from osgtest.library import osgunittest
 
@@ -496,10 +500,8 @@ def __format_command(command):
     return result
 
 
-def __prepare_shell_argument(argument):
-    if re.search(r'\W', argument) or argument == '':
-        return "'" + re.sub(r"'", r"'\''", argument) + "'"
-    return argument
+_devnull = open(os.devnull, "r+b")
+
 
 def __run_command(command, use_test_user, a_input, a_stdout, a_stderr, log_output=True, shell=False, timeout=None, timeout_signal='TERM'):
     global _last_log_had_output
@@ -514,10 +516,10 @@ def __run_command(command, use_test_user, a_input, a_stdout, a_stderr, log_outpu
         except TypeError:
             print('Need list or tuple, got %s' % type(command))
     if use_test_user:
-        command = ['runuser', options.username, '-c', ' '.join(map(__prepare_shell_argument, command))]
+        command = ['runuser', options.username, '-c', ' '.join(map(shell_quote, command))]
 
     # Figure out stdin
-    stdin = None
+    stdin = _devnull
     if a_input is not None:
         stdin = subprocess.PIPE
 
@@ -555,7 +557,8 @@ def __run_command(command, use_test_user, a_input, a_stdout, a_stderr, log_outpu
             os.killpg(p.pid, timeout_signal)
             os._exit(0)
 
-    (stdout, stderr) = p.communicate(a_input)
+    stdout, stderr = p.communicate(to_bytes(a_input))
+    stdout, stderr = to_str(stdout), to_str(stderr)
 
     if timeout is not None:
         if p.returncode >= 0:
@@ -783,11 +786,23 @@ def run_fn_if_el_release_ok(*args, **kwargs):
     return el_release_decorator
 
 
-try:
-    unicode
-except NameError:  # python 3
-    unicode = str
+def to_str(strlike, encoding="latin-1", errors="backslashreplace"):
+    """Turns a bytes into a str (Python 3) or a unicode to a str (Python 2)"""
+    if strlike is None:
+        return
+    if not isinstance(strlike, str):
+        if str is bytes:
+            return strlike.encode(encoding, errors)
+        else:
+            return strlike.decode(encoding, errors)
+    else:
+        return strlike
 
 
-def is_string(var):
-    return isinstance(var, (str, unicode))
+def to_bytes(strlike, encoding="latin-1", errors="backslashreplace"):
+    """Turns a str into bytes (Python 3) or a unicode to a str (Python 2)"""
+    if strlike is None:
+        return
+    if not isinstance(strlike, bytes):
+        return strlike.encode(encoding, errors)
+    return strlike

osgtest/tests/special_install.py

@@ -58,9 +58,13 @@ def test_02_install_packages(self):
 
             retry_fail, _, stdout, _ = yum.retry_command(command)
             if retry_fail == '':   # the command succeeded
-                core.state['install.transaction_ids'].append(yum.get_transaction_id())
-                command = ('rpm', '--verify', pkg)
-                core.check_system(command, 'Verify %s' % (pkg))
+                id = yum.get_transaction_id()
+                if id not in core.state['install.transaction_ids']:
+                    core.state['install.transaction_ids'].append(id)
+                if not pkg.startswith("/"):
+                    # ^^ rpm --verify doesn't work if you asked for a file instead of a package
+                    command = ('rpm', '--verify', pkg)
+                    core.check_system(command, 'Verify %s' % (pkg))
                 yum.parse_output_for_packages(stdout)
 
             fail_msg += retry_fail

osgtest/tests/test_280_gsiopenssh.py

@@ -34,9 +34,9 @@ def test_01_set_config(self):
         core.state['gsisshd.can-run'] = (not (
             core.el_release() >= 7 and
             core.state['selinux.mode'] and
-            not core.rpm_is_installed('policycoreutils-python')))
+            not core.dependency_is_installed("/usr/sbin/semanage")))
         self.skip_ok_unless(core.state['gsisshd.can-run'],
-                            "Can't run with SELinux on EL >= 7 without policycoreutils-python")
+                            "Can't run with SELinux on EL >= 7 without semanage")
 
         files.write(
             SSHD_CONFIG,
@@ -47,7 +47,7 @@ def test_01_set_config(self):
     def test_02_setup_selinux_port(self):
         if not core.state['selinux.mode']:
             self.skip_ok('SELinux disabled')
-        core.skip_ok_unless_installed('policycoreutils-python')
+        core.skip_ok_unless_installed("/usr/sbin/semanage", by_depenency=True)
         port = core.config['gsisshd.port']
         core.check_system(['semanage', 'port', '--add', '-t', 'ssh_port_t', '--proto', 'tcp', port],
                           message="Allow [gsi]sshd to use port %s" % port)

osgtest/tests/test_407_voms.py

@@ -80,6 +80,8 @@ def test_08_voms_proxy_check(self):
 
         pwd_entry = pwd.getpwnam(core.options.username)
         cert_path = os.path.join(pwd_entry.pw_dir, '.globus', 'usercert.pem')
+        # Note: We're only looking for the "Signature Algorithm" line which has the same output format
+        # regardless of openssl version.
         command = ['openssl', 'x509', '-in', cert_path, '-text']
         signature_re = re.compile(r'Signature Algorithm:\s+(\w+)\s')
         stdout = core.check_system(command, 'Check X.509 certificate algorithm', user=True)[0]

osgtest/tests/test_460_stashcache.py

@@ -79,7 +79,7 @@ def test_03_http_fetch_from_cache(self):
             f = urlopen(
                 "http://localhost:%d/%s" % (getcfg("CacheHTTPPort"), path)
             )
-            result = f.read()
+            result = core.to_str(f.read())
         except IOError as e:
             self.fail("Unable to download from cache via http: %s" % e)
         self.assertEqualVerbose(result, contents, "downloaded file mismatch")

osgtest/tests/test_470_rsv.py

@@ -8,7 +8,10 @@
 import pwd
 import shutil
 import socket
-import ConfigParser
+try:
+    import ConfigParser
+except ImportError:
+    import configparser as ConfigParser
 
 """
 Testing number conventions:

osgtest/tests/test_550_condorce.py

@@ -3,7 +3,10 @@
 
 import os
 import re
-import urllib2
+try:
+    from urllib2 import urlopen
+except ImportError:
+    from urllib.request import urlopen
 
 import osgtest.library.condor as condor
 import osgtest.library.core as core
@@ -126,9 +129,9 @@ def test_07_ceview(self):
         core.skip_ok_unless_installed('htcondor-ce-view')
         view_url = 'http://%s:%s' % (core.get_hostname(), int(core.config['condor-ce.view-port']))
         try:
-            src = urllib2.urlopen(view_url).read()
-        except urllib2.URLError:
-            self.fail('Could not reach HTCondor-CE View at %s' % view_url)
+            src = to_str(urlopen(view_url).read())
+        except EnvironmentError as err:
+            self.fail('Could not reach HTCondor-CE View at %s: %s' % (view_url, err))
         self.assert_(re.search(r'HTCondor-CE Overview', src), 'Failed to find expected CE View contents')
         core.config['condor-ce.view-listening'] = True