Add timeout to Fuzzlyn reductions (#115189)

Author: jakobbotsch

src/coreclr/scripts/fuzzlyn_run.py

@@ -119,7 +119,7 @@ def run(self):
 
             if new_line:
                 evt = json.loads(new_line)
-                # Only reduce BadResult examples since crashes take very long to reduce.
+                # Do not reduce crash examples since those take a very long to reduce.
                 # We will still report crashes, just not with a reduced example.
                 if evt["Kind"] == "ExampleFound":
                     ex = evt["Example"]
@@ -147,7 +147,7 @@ def run(self):
                             "--seed", str(ex['Seed']),
                             "--collect-spmi-to", spmi_collections_path,
                             "--output", output_path]
-                        run_command(cmd)
+                        run_command(cmd, _timeout=60*25)
                         if path.exists(output_path):
                             num_reduced += 1
                             if num_reduced >= 5:
@@ -156,6 +156,8 @@ def run(self):
 
                             if ex_assert_err is not None:
                                 self.reduced_jit_asserts.add(ex_assert_err)
+                        else:
+                            print("  Reduction failed, output file not present")
 
 
 def main(main_args):

src/coreclr/scripts/jitutil.py

@@ -20,6 +20,7 @@
 import logging
 import time
 import tarfile
+import threading
 import urllib
 import urllib.request
 import zipfile
@@ -129,7 +130,7 @@ def decode_and_print(str_to_decode):
         return output
 
 
-def run_command(command_to_run, _cwd=None, _exit_on_fail=False, _output_file=None, _env=None):
+def run_command(command_to_run, _cwd=None, _exit_on_fail=False, _output_file=None, _env=None, _timeout=None):
     """ Runs the command.
 
     Args:
@@ -138,6 +139,7 @@ def run_command(command_to_run, _cwd=None, _exit_on_fail=False, _output_file=Non
         _exit_on_fail (bool): If it should exit on failure.
         _output_file ():
         _env: environment for sub-process, passed to subprocess.Popen()
+        _timeout: timeout in seconds, or None for no timeout
     Returns:
         (string, string, int): Returns a tuple of stdout, stderr, and command return code if _output_file= None
         Otherwise stdout, stderr are empty.
@@ -156,27 +158,44 @@ def run_command(command_to_run, _cwd=None, _exit_on_fail=False, _output_file=Non
     output_type = subprocess.STDOUT if _output_file else subprocess.PIPE
     with subprocess.Popen(command_to_run, env=_env, stdout=subprocess.PIPE, stderr=output_type, cwd=_cwd) as proc:
 
-        # For long running command, continuously print the output
-        if _output_file:
-            while True:
-                with open(_output_file, 'a') as of:
-                    output = proc.stdout.readline()
-                    if proc.poll() is not None:
-                        break
-                    if output:
-                        output_str = decode_and_print(output.strip())
-                        of.write(output_str + "\n")
-        else:
-            command_stdout, command_stderr = proc.communicate()
-            if len(command_stdout) > 0:
-                decode_and_print(command_stdout)
-            if len(command_stderr) > 0:
-                decode_and_print(command_stderr)
+        timer = None
+        if _timeout is not None:
+            def try_kill():
+                try:
+                    print("  Timeout reached; killing process")
+                    proc.kill()
+                except:
+                    pass
+
+            timer = threading.Timer(_timeout, try_kill)
+            timer.start()
+
+        try:
+            # For long running command, continuously print the output
+            if _output_file:
+                while True:
+                    with open(_output_file, 'a') as of:
+                        output = proc.stdout.readline()
+                        if proc.poll() is not None:
+                            break
+                        if output:
+                            output_str = decode_and_print(output.strip())
+                            of.write(output_str + "\n")
+            else:
+                command_stdout, command_stderr = proc.communicate()
+                if len(command_stdout) > 0:
+                    decode_and_print(command_stdout)
+                if len(command_stderr) > 0:
+                    decode_and_print(command_stderr)
+        finally:
+            if timer:
+                timer.cancel()
 
         return_code = proc.returncode
         if _exit_on_fail and return_code != 0:
             print("Command failed. Exiting.")
             sys.exit(1)
+
     return command_stdout, command_stderr, return_code