Output stdout even when getting RE or TLE (#84)

* Update version to 1.1.0

* Update README

* Update CHANGELOG.md

* fix a bug of Java template (no issue)

* #83 Show stdout even when getting RE or TLE

* #83 Support stderr output

* #83 More recognizable header color

* Treat outputting stderr as special failure case

* autopep

* autopep8

Author: kyuridenamida

atcodertools/tools/tester.py

@@ -1,10 +1,10 @@
 #!/usr/bin/python3
 import argparse
+import glob
 import logging
-import sys
 import os
-import glob
 import subprocess
+import sys
 import time
 from enum import Enum
 from pathlib import Path
@@ -32,16 +32,21 @@ class ExecStatus(Enum):
 
 class ExecResult:
 
-    def __init__(self, status: ExecStatus, output: str = None, elapsed_sec: float = None):
+    def __init__(self, status: ExecStatus, output: str = None, stderr: str = None, elapsed_sec: float = None):
         self.status = status
         self.output = output
+        self.stderr = stderr
+
         if elapsed_sec is not None:
             self.elapsed_ms = int(elapsed_sec * 1000 + 0.5)
         else:
             self.elapsed_ms = None
 
     def is_correct_output(self, answer_text):
-        return answer_text == self.output
+        return self.status == ExecStatus.NORMAL and answer_text == self.output
+
+    def has_stderr(self):
+        return len(self.stderr) > 0
 
 
 def is_executable_file(file_name):
@@ -76,14 +81,23 @@ def infer_case_num(sample_filename: str):
 def run_program(exec_file: str, input_file: str, timeout_sec: int) -> ExecResult:
     try:
         elapsed_sec = -time.time()
-        out_data = subprocess.check_output(
-            [exec_file, ""], stdin=open(input_file, 'r'), universal_newlines=True, timeout=timeout_sec)
+        proc = subprocess.run(
+            [exec_file, ""], stdin=open(input_file, 'r'), universal_newlines=True, timeout=timeout_sec,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        if proc.returncode == 0:
+            code = ExecStatus.NORMAL
+        else:
+            code = ExecStatus.RE
+
         elapsed_sec += time.time()
-        return ExecResult(ExecStatus.NORMAL, out_data, elapsed_sec)
-    except subprocess.TimeoutExpired:
-        return ExecResult(ExecStatus.TLE)
-    except subprocess.CalledProcessError:
-        return ExecResult(ExecStatus.RE)
+        return ExecResult(code, proc.stdout, proc.stderr, elapsed_sec=elapsed_sec)
+    except subprocess.TimeoutExpired as e:
+        return ExecResult(ExecStatus.TLE, e.stdout, e.stderr)
+    except subprocess.CalledProcessError as e:
+        return ExecResult(ExecStatus.RE, e.stdout, e.stderr)
 
 
 def build_details_str(exec_res: ExecResult, input_file: str, output_file: str) -> str:
@@ -93,21 +107,23 @@ def append(text: str, end='\n'):
         nonlocal res
         res += text + end
 
-    append("[Input]")
+    append(with_color("[Input]", Fore.LIGHTMAGENTA_EX))
     with open(input_file, "r") as f:
         append(f.read(), end='')
 
-    append("[Expected]")
+    append(with_color("[Expected]", Fore.LIGHTMAGENTA_EX))
     with open(output_file, "r") as f:
         append(f.read(), end='')
 
-    if exec_res.status == ExecStatus.NORMAL:
-        append("[Received]")
-        if exec_res.status == ExecStatus.NORMAL:
-            append(exec_res.output, end='')
-    else:
-        append("[Log]")
-        append(exec_res.status.name)
+    append(with_color("[Received]", Fore.LIGHTMAGENTA_EX))
+    append(exec_res.output, end='')
+    if exec_res.status != ExecStatus.NORMAL:
+        append(with_color("Aborted ({})\n".format(
+            exec_res.status.name), Fore.LIGHTYELLOW_EX))
+
+    if exec_res.has_stderr():
+        append(with_color("[Error]", Fore.LIGHTYELLOW_EX))
+        append(exec_res.stderr, end='')
     return res
 
 
@@ -123,24 +139,31 @@ def run_for_samples(exec_file: str, sample_pair_list: List[Tuple[str, str]], tim
             answer_text = f.read()
 
         is_correct = exec_res.is_correct_output(answer_text)
-        if is_correct:
+        passed = is_correct and not exec_res.has_stderr()
+
+        if passed:
             message = "{} {elapsed} ms".format(
                 with_color("PASSED", Fore.LIGHTGREEN_EX),
                 elapsed=exec_res.elapsed_ms)
             success_count += 1
         else:
-            if exec_res.status == ExecStatus.NORMAL:
-                message = with_color("WA", Fore.LIGHTRED_EX)
+            if is_correct:
+                message = with_color(
+                    "CORRECT but with stderr (Please remove stderr!)", Fore.LIGHTYELLOW_EX)
             else:
-                message = with_color(exec_res.status.name, Fore.LIGHTYELLOW_EX)
+                if exec_res.status == ExecStatus.NORMAL:
+                    message = with_color("WA", Fore.LIGHTRED_EX)
+                else:
+                    message = with_color(
+                        exec_res.status.name, Fore.LIGHTYELLOW_EX)
 
         print("# {case_name} ... {message}".format(
             case_name=os.path.basename(in_sample_file),
             message=message,
         ))
 
         # Output details for incorrect results.
-        if not is_correct:
+        if not passed:
             print('{}\n'.format(build_details_str(
                 exec_res, in_sample_file, out_sample_file)))
             if knock_out: