Implement # sage.doctest: flaky marker

src/sage/algebras/fusion_rings/fusion_ring.py

@@ -1,3 +1,4 @@
+# sage.doctest: flaky
 """
 Fusion Rings
 """

src/sage/doctest/forker.py

@@ -1724,6 +1724,21 @@
     """
     Create parallel :class:`DocTestWorker` processes and dispatches
     doctesting tasks.
+
+    .. NOTE::
+
+        If this is run directly in the normal Sage command-line,
+        it calls :func:`init_sage` which in turn calls
+        :meth:`~sage.repl.rich_output.display_manager.DisplayManager.switch_backend`
+        to the doctest backend, which is incompatible with the IPython-based
+        command-line.  As such, if an error such as
+        ``TypeError: cannot unpack non-iterable NoneType object`` is seen,
+        a workaround is to run the following::
+
+            sage: # not tested
+            sage: from sage.repl.rich_output.backend_ipython import BackendIPythonCommandline
+            sage: backend = BackendIPythonCommandline()
+            sage: get_ipython().display_formatter.dm.switch_backend(backend, shell=get_ipython())
     """
     def __init__(self, controller: DocTestController):
         """
@@ -1851,6 +1866,67 @@
                1 of   1 in ...
                 [1 test, 1 failure, ...s wall]
             Killing test ...
+
+        TESTS:
+
+        Test flaky files. This test should fail the first time and success the second time::
+
+            sage: with NTF(suffix='.py', mode='w+t') as f1:
+            ....:     t = walltime()
+            ....:     _ = f1.write(f"# sage.doctest: flaky\n'''\nsage: sleep(10 if walltime() < {t+1} else 0)\n'''")
+            ....:     f1.flush()
+            ....:     DC = DocTestController(DocTestDefaults(timeout=2),
+            ....:                            [f1.name])
+            ....:     DC.expand_files_into_sources()
+            ....:     DD = DocTestDispatcher(DC)
+            ....:     DR = DocTestReporter(DC)
+            ....:     DC.reporter = DR
+            ....:     DC.dispatcher = DD
+            ....:     DC.timer = Timer().start()
+            ....:     DD.parallel_dispatch()
+                sage -t ...
+                sage -t ...
+                    [1 test, ...s wall]
+
+        This test always fail, so even flaky can't help it::
+
+            sage: with NTF(suffix='.py', mode='w+t') as f1:
+            ....:     _ = f1.write(f"# sage.doctest: flaky\n'''\nsage: sleep(10)\n'''")
+            ....:     f1.flush()
+            ....:     DC = DocTestController(DocTestDefaults(timeout=2),
+            ....:                            [f1.name])
+            ....:     DC.expand_files_into_sources()
+            ....:     DD = DocTestDispatcher(DC)
+            ....:     DR = DocTestReporter(DC)
+            ....:     DC.reporter = DR
+            ....:     DC.dispatcher = DD
+            ....:     DC.timer = Timer().start()
+            ....:     DD.parallel_dispatch()
+            sage -t ...
+            sage -t ...
+                Timed out
+            **********************************************************************
+            ...
+
+        Of course without flaky, the test should fail (since it timeouts the first time)::
+
+            sage: with NTF(suffix='.py', mode='w+t') as f1:
+            ....:     t = walltime()
+            ....:     _ = f1.write(f"'''\nsage: sleep(10 if walltime() < {t+1} else 0)\n'''")
+            ....:     f1.flush()
+            ....:     DC = DocTestController(DocTestDefaults(timeout=2),
+            ....:                            [f1.name])
+            ....:     DC.expand_files_into_sources()
+            ....:     DD = DocTestDispatcher(DC)
+            ....:     DR = DocTestReporter(DC)
+            ....:     DC.reporter = DR
+            ....:     DC.dispatcher = DD
+            ....:     DC.timer = Timer().start()
+            ....:     DD.parallel_dispatch()
+            sage -t ...
+                Timed out
+            **********************************************************************
+            ...
         """
         opt = self.controller.options
 
@@ -1937,6 +2013,28 @@
                     # precision.
                     now = time.time()
 
+                    def start_new_worker(source: DocTestSource, num_retries_left: typing.Optional[int] = None) -> DocTestWorker:
+                        nonlocal opt, target_endtime, now, pending_tests, sel_exit
+                        import copy
+                        worker_options = copy.copy(opt)
+                        baseline = self.controller.source_baseline(source)
+                        if target_endtime is not None:
+                            worker_options.target_walltime = (target_endtime - now) / (max(1, pending_tests / opt.nthreads))
+                        w = DocTestWorker(source, options=worker_options, funclist=[sel_exit], baseline=baseline)
+                        if num_retries_left is not None:
+                            w.num_retries_left = num_retries_left
+                        elif 'flaky' in source.file_optional_tags:
+                            w.num_retries_left = 1
+                        heading = self.controller.reporter.report_head(w.source)
+                        if not self.controller.options.only_errors:
+                            w.messages = heading + "\n"
+                        # Store length of heading to detect if the
+                        # worker has something interesting to report.
+                        w.heading_len = len(w.messages)
+                        w.start()  # This might take some time
+                        w.deadline = time.time() + opt.timeout
+                        return w
+
                     # If there were any substantial changes in the state
                     # (new worker started or finished worker reported),
                     # restart this while loop instead of calling pselect().
@@ -1994,6 +2092,14 @@
                     # Similarly, process finished workers.
                     new_finished = []
                     for w in finished:
+                        if w.killed and w.num_retries_left > 0:
+                            # in this case, the messages from w should be suppressed
+                            # (better handling could be implemented later)
+                            if follow is w:
+                                follow = None
+                            workers.append(start_new_worker(w.source, w.num_retries_left - 1))
+                            continue
+
                         if opt.exitfirst and w.result[1].failures:
                             abort_now = True
                         elif follow is not None and follow is not w:
@@ -2026,28 +2132,13 @@
                     while (source_iter is not None and len(workers) < opt.nthreads
                            and (not job_client or job_client.acquire())):
                         try:
-                            source = next(source_iter)
+                            source: DocTestSource = next(source_iter)
                         except StopIteration:
                             source_iter = None
                             if job_client:
                                 job_client.release()
                         else:
-                            # Start a new worker.
-                            import copy
-                            worker_options = copy.copy(opt)
-                            baseline = self.controller.source_baseline(source)
-                            if target_endtime is not None:
-                                worker_options.target_walltime = (target_endtime - now) / (max(1, pending_tests / opt.nthreads))
-                            w = DocTestWorker(source, options=worker_options, funclist=[sel_exit], baseline=baseline)
-                            heading = self.controller.reporter.report_head(w.source)
-                            if not self.controller.options.only_errors:
-                                w.messages = heading + "\n"
-                            # Store length of heading to detect if the
-                            # worker has something interesting to report.
-                            w.heading_len = len(w.messages)
-                            w.start()  # This might take some time
-                            w.deadline = time.time() + opt.timeout
-                            workers.append(w)
+                            workers.append(start_new_worker(source))
                             restart = True
 
                     # Recompute state if needed
@@ -2181,6 +2272,7 @@
 
     EXAMPLES::
 
+        sage: # long time
         sage: from sage.doctest.forker import DocTestWorker, DocTestTask
         sage: from sage.doctest.sources import FileDocTestSource
         sage: from sage.doctest.reporting import DocTestReporter
@@ -2223,6 +2315,10 @@
         self.funclist = funclist
         self.baseline = baseline
 
+        # This is not used by this class in any way, but DocTestDispatcher
+        # uses this to keep track of reruns for flaky tests.
+        self.num_retries_left = 0
+
         # Open pipe for messages. These are raw file descriptors,
         # not Python file objects!
         self.rmessages, self.wmessages = os.pipe()
@@ -2305,6 +2401,7 @@
 
         TESTS::
 
+            sage: # long time
             sage: from sage.doctest.forker import DocTestWorker, DocTestTask
             sage: from sage.doctest.sources import FileDocTestSource
             sage: from sage.doctest.reporting import DocTestReporter
@@ -2344,6 +2441,7 @@
 
         EXAMPLES::
 
+            sage: # long time
             sage: from sage.doctest.forker import DocTestWorker, DocTestTask
             sage: from sage.doctest.sources import FileDocTestSource
             sage: from sage.doctest.reporting import DocTestReporter
@@ -2378,6 +2476,7 @@
 
         EXAMPLES::
 
+            sage: # long time
             sage: from sage.doctest.forker import DocTestWorker, DocTestTask
             sage: from sage.doctest.sources import FileDocTestSource
             sage: from sage.doctest.reporting import DocTestReporter
@@ -2508,6 +2607,7 @@
 
     EXAMPLES::
 
+        sage: # long time
         sage: from sage.doctest.forker import DocTestTask
         sage: from sage.doctest.sources import FileDocTestSource
         sage: from sage.doctest.control import DocTestDefaults, DocTestController

src/sage/doctest/parsing.py

@@ -56,7 +56,7 @@
 ansi_escape_sequence = re.compile(r"(\x1b[@-Z\\-~]|\x1b\[.*?[@-~]|\x9b.*?[@-~])")
 
 special_optional_regex = (
-    "py2|long time|not implemented|not tested|optional|needs|known bug"
+    "py2|long time|not implemented|not tested|optional|needs|known bug|flaky"
 )
 tag_with_explanation_regex = r"((?:!?\w|[.])*)\s*(?:\((?P<cmd_explanation>.*?)\))?"
 optional_regex = re.compile(
@@ -97,6 +97,7 @@ def parse_optional_tags(
     - ``'not tested'``
     - ``'known bug'`` (possible values are ``None``, ``linux`` and ``macos``)
     - ``'py2'``
+    - ``'flaky'``
     - ``'optional -- FEATURE...'`` or ``'needs FEATURE...'`` --
       the dictionary will just have the key ``'FEATURE'``
 
@@ -166,6 +167,17 @@ def parse_optional_tags(
         sage: parse_optional_tags("sage: #this is not #needs scipy\n....: import scipy",
         ....:                     return_string_sans_tags=True)
         ({'scipy': None}, 'sage: #this is not \n....: import scipy', False)
+
+    TESTS::
+
+        sage: parse_optional_tags("# flaky")
+        {'flaky': None}
+
+    Remember to update the documentation above whenever the following changes::
+
+        sage: from sage.doctest.parsing import special_optional_regex
+        sage: special_optional_regex.pattern
+        'py2|long time|not implemented|not tested|optional|needs|known bug|flaky'
     """
     safe, literals, state = strip_string_literals(string)
     split = safe.split('\n', 1)
@@ -260,6 +272,11 @@ def parse_file_optional_tags(lines):
         sage: with open(filename, "r") as f:
         ....:     parse_file_optional_tags(enumerate(f))
         {'xyz': None}
+        sage: with open(filename, "w") as f:
+        ....:     _ = f.write("# sage.doctest: flaky")
+        sage: with open(filename, "r") as f:
+        ....:     parse_file_optional_tags(enumerate(f))
+        {'flaky': None}
     """
     tags = {}
     for line_count, line in lines:
@@ -990,6 +1007,14 @@ def parse(self, string, *args):
             'C.minimum_distance(algorithm="guava")  # optional - guava\n'
             sage: dte.want
             '...\n24\n'
+
+        Test putting flaky tag in a single test::
+
+            sage: example5 = 'sage: 1 # flaky\n1'
+            sage: parsed5 = DTP.parse(example5)
+            Traceback (most recent call last):
+            ...
+            NotImplementedError: 'flaky' tag is only implemented for whole file, not for single test
         """
         # Regular expressions
         find_sage_prompt = re.compile(r"^(\s*)sage: ", re.M)
@@ -1063,6 +1088,8 @@ def check_and_clear_tag_counts():
         for item in res:
             if isinstance(item, doctest.Example):
                 optional_tags_with_values, _, is_persistent = parse_optional_tags(item.source, return_string_sans_tags=True)
+                if "flaky" in optional_tags_with_values:
+                    raise NotImplementedError("'flaky' tag is only implemented for whole file, not for single test")
                 optional_tags = set(optional_tags_with_values)
                 if is_persistent:
                     check_and_clear_tag_counts()
@@ -1087,6 +1114,11 @@ def check_and_clear_tag_counts():
                             ('not tested' in optional_tags)):
                         continue
 
+                    if 'flaky' in optional_tags:
+                        # since a single test cannot have the 'flaky' tag,
+                        # this must come from file_optional_tags
+                        optional_tags.remove('flaky')
+
                     if 'long time' in optional_tags:
                         if self.long:
                             optional_tags.remove('long time')

src/sage/rings/polynomial/multi_polynomial_libsingular.pyx

@@ -1,3 +1,4 @@
+# sage.doctest: flaky
 r"""
 Multivariate Polynomials via libSINGULAR
 

src/sage/rings/polynomial/polynomial_element.pyx

@@ -1,3 +1,4 @@
+# sage.doctest: flaky
 """
 Univariate polynomial base class
 
@@ -2585,7 +2586,7 @@
             sage: x = polygen(K)
             sage: pol = x^1000000 + x + a
             sage: from sage.doctest.util import ensure_interruptible_after
             sage: with ensure_interruptible_after(0.5): pol.any_root()
 
         Check root computation over large finite fields::