[LNT] Python 3 support: subprocess IO as text

Summary:
In Python 3, subprocess stdin, stdout and stderr are in binary mode by
default. Trying to print or parse those as text as is done thoughought
in LNT then causes byte/str errors. This commits sets the
universal_newlines parameter to True when invoking subprocess methods
involving input or output to request those to be in text mode. It also
adapt _check_output to work whether universal_newlines is set to True in
the caller or not. Finally, it cleans up 2 cases subprocess method call
where the output file objects are stored in a variable but not used.

Reviewers: cmatthews, hubert.reinterpretcast, kristof.beyls, leandron, PrzemekWirkus

Reviewed By: PrzemekWirkus

Subscribers: llvm-commits

Differential Revision: https://reviews.llvm.org/D68938

Author: Thomas Preud'homme

lnt/server/db/rules/rule_update_profile_stats.py

@@ -28,7 +28,8 @@ def update_profile_stats(session, ts, run_id):
 
     dt = time.time()
     blocks = subprocess.check_output("du -s -k %s" % profile_path,
-                                     shell=True).split('\t')[0]
+                                     shell=True,
+                                     universal_newlines=True).split('\t')[0]
     kb = float(blocks)  # 1024 byte blocks.
 
     history.append((dt, kb))

lnt/testing/util/commands.py

@@ -61,7 +61,8 @@ def capture_with_result(args, include_stderr=False):
     if include_stderr:
         stderr = subprocess.STDOUT
     try:
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=stderr)
+        p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=stderr,
+                             universal_newlines=True)
     except OSError as e:
         if e.errno == errno.ENOENT:
             fatal('no such file or directory: %r when running %s.' %

lnt/tests/compile.py

@@ -81,7 +81,8 @@ def runN(args, N, cwd, preprocess_cmd=None, env=None, sample_mem=False,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE,
                          env=env,
-                         cwd=cwd)
+                         cwd=cwd,
+                         universal_newlines=True)
     runn_stdout, runn_stderr = p.communicate()
     res = p.returncode
 
@@ -332,13 +333,15 @@ def test_build(base_name, run_info, variables, project, build_config, num_jobs,
                                  stdin=None,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE,
-                                 cwd=source_path)
+                                 cwd=source_path,
+                                 universal_newlines=True)
         else:
             p = subprocess.Popen(args=['unzip', '-q', archive_path],
                                  stdin=None,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE,
-                                 cwd=source_path)
+                                 cwd=source_path,
+                                 universal_newlines=True)
         stdout, stderr = p.communicate()
         if p.wait() != 0:
             fatal(("unable to extract archive %r at %r\n"
@@ -356,7 +359,8 @@ def test_build(base_name, run_info, variables, project, build_config, num_jobs,
                                  stdin=None,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE,
-                                 cwd=source_path)
+                                 cwd=source_path,
+                                 universal_newlines=True)
             stdout, stderr = p.communicate()
             if p.wait() != 0:
                 fatal(("unable to apply patch file %r in %r\n"

lnt/tests/nt.py

@@ -513,7 +513,7 @@ def execute_command(test_log, basedir, args, report_dir):
 
     p = subprocess.Popen(args=args, stdin=None, stdout=logfile,
                          stderr=subprocess.STDOUT, cwd=basedir,
-                         env=os.environ)
+                         env=os.environ, universal_newlines=True)
 
     if report_dir is not None:
         while p.poll() is None:

lnt/tests/test_suite.py

@@ -424,7 +424,10 @@ def _check_output(self, *args, **kwargs):
         if 'cwd' in kwargs:
             logger.info('          (In %s)' % kwargs['cwd'])
         output = subprocess.check_output(*args, **kwargs)
-        sys.stdout.write(output)
+        if kwargs.get('universal_newlines', False):
+            sys.stdout.write(output)
+        else:
+            sys.stdout.buffer.write(output)
         return output
 
     def _clean(self, path):
@@ -652,7 +655,8 @@ def _get_lnt_code(self, code):
     def _extract_cmake_vars_from_cache(self):
         assert self.configured is True
         cmake_lah_output = self._check_output(
-            [self.opts.cmake] + ['-LAH', '-N'] + [self._base_path])
+            [self.opts.cmake] + ['-LAH', '-N'] + [self._base_path],
+            universal_newlines=True)
         pattern2var = [
             (re.compile("^%s:[^=]*=(.*)$" % cmakevar), cmakevar)
             for cmakevar in (
@@ -875,13 +879,14 @@ def diagnose(self):
 
         logger.info(' '.join(cmd_temps))
 
-        out = subprocess.check_output(cmd_temps)
+        out = subprocess.check_output(cmd_temps, universal_newlines=True)
         logger.info(out)
 
         # Figure out our test's target.
         make_cmd = [self.opts.make, "VERBOSE=1", 'help']
 
-        make_targets = subprocess.check_output(make_cmd)
+        make_targets = subprocess.check_output(make_cmd,
+                                               universal_newlines=True)
         matcher = re.compile(r"^\.\.\.\s{}$".format(short_name),
                              re.MULTILINE | re.IGNORECASE)
         if not matcher.search(make_targets):
@@ -894,15 +899,17 @@ def diagnose(self):
         logger.info(" ".join(make_deps))
         p = subprocess.Popen(make_deps,
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.STDOUT)
+                             stderr=subprocess.STDOUT,
+                             universal_newlines=True)
         std_out, std_err = p.communicate()
         logger.info(std_out)
 
         make_save_temps = [self.opts.make, "VERBOSE=1", short_name]
         logger.info(" ".join(make_save_temps))
         p = subprocess.Popen(make_save_temps,
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.STDOUT)
+                             stderr=subprocess.STDOUT,
+                             universal_newlines=True)
         std_out, std_err = p.communicate()
         logger.info(std_out)
         with open(report_path + "/build.log", 'w') as f:
@@ -926,13 +933,14 @@ def diagnose(self):
 
         logger.info(' '.join(cmd_time_report))
 
-        out = subprocess.check_output(cmd_time_report)
+        out = subprocess.check_output(cmd_time_report, universal_newlines=True)
         logger.info(out)
 
         make_time_report = [self.opts.make, "VERBOSE=1", short_name]
         p = subprocess.Popen(make_time_report,
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
+                             stderr=subprocess.PIPE,
+                             universal_newlines=True)
         std_out, std_err = p.communicate()
 
         with open(report_path + "/time-report.txt", 'w') as f:
@@ -944,13 +952,15 @@ def diagnose(self):
 
         logger.info(' '.join(cmd_stats_report))
 
-        out = subprocess.check_output(cmd_stats_report)
+        out = subprocess.check_output(cmd_stats_report,
+                                      universal_newlines=True)
         logger.info(out)
 
         make_stats_report = [self.opts.make, "VERBOSE=1", short_name]
         p = subprocess.Popen(make_stats_report,
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
+                             stderr=subprocess.PIPE,
+                             universal_newlines=True)
         std_out, std_err = p.communicate()
 
         with open(report_path + "/stats-report.txt", 'w') as f:
@@ -974,19 +984,20 @@ def diagnose(self):
             cmd_iprofiler = cmd + ['-DTEST_SUITE_RUN_UNDER=' + iprofiler]
             print(' '.join(cmd_iprofiler))
 
-            out = subprocess.check_output(cmd_iprofiler)
+            subprocess.check_output(cmd_iprofiler, universal_newlines=True)
 
             os.chdir(local_path)
             make_iprofiler_temps = [self.opts.make, "VERBOSE=1", short_name]
             p = subprocess.Popen(make_iprofiler_temps,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
-            std_out, std_err = p.communicate()
+            p.communicate()
             logger.warning("Using sudo to collect execution trace.")
             make_save_temps = sudo + [self.opts.lit, short_name + ".test"]
             p = subprocess.Popen(make_save_temps,
                                  stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE)
+                                 stderr=subprocess.PIPE,
+                                 universal_newlines=True)
             std_out, std_err = p.communicate()
             sys.stdout.write(std_out)
             sys.stderr.write(std_err)