gh-36741:  Doctest hide option: Better detection of hidden packages
    
<!-- ^^^^^
Please provide a concise, informative and self-explanatory title.
Don't put issue numbers in there, do this in the PR body below.
For example, instead of "Fixes #1234" use "Introduce new method to
calculate 1+1"
-->
<!-- Describe your changes here in detail -->

<!-- Why is this change required? What problem does it solve? -->
<!-- If this PR resolves an open issue, please link to it here. For
example "Fixes #12345". -->
<!-- If your change requires a documentation PR, please link it
appropriately. -->

This PR implements the suggestion made in
#36696 (comment).
This means that it introduces a counter in the feature class to detect
the number of events a feature has been asked to be present while it was
hidden. This allows to remove the call of the `is_present` method in
`doctest/control.py`.

In order to test it I ran `sage -tp --all --hide=all`. Ideally this
should give *All tests passed*. But I got two failing files. One of
those was `src/sage/features/databases.py` because
`database_cremona_mini_ellcurve` was detected to be optional even
thought it is a standard package. This is a leftover of
#35820 which I fix here.

Similarily I got 37 failures in
`src/sage/groups/abelian_gps/abelian_group.py` since
`gap_package_polycyclic` was classified optional even though it is a Gap
standard package since Gap 4.10 (as well as `gap_package_atlasrep`). But
in this case a corresponding fix, i.e.:

```
 def all_features():
-    return [GapPackage("atlasrep", spkg="gap_packages"),
+    return [GapPackage("atlasrep", spkg="gap_packages",
type='standard'),
             GapPackage("design", spkg="gap_packages"),
             GapPackage("grape", spkg="gap_packages"),
             GapPackage("guava", spkg="gap_packages"),
             GapPackage("hap", spkg="gap_packages"),
-            GapPackage("polycyclic", spkg="gap_packages"),
+            GapPackage("polycyclic", spkg="gap_packages",
type='standard'),
             GapPackage("qpa", spkg="gap_packages"),
             GapPackage("quagroup", spkg="gap_packages")]
```

is not the correct way (it gives `UserWarning: Feature
gap_package_polycyclic is declared standard, but it is provided by
gap_packages, which is declared optional in SAGE_ROOT/build/pkgs`). I
would say, the correct fix would be to remove both Gap packages from the
`all_features` list and erase the corresponding *optional tags*  in
`permgroup_named.py`, `distance_regular.pyx`, `abelian_aut.py`,
`abelian_group.py` and `abelian_group_gap.py`. If you agree I will open
another PR for this.


BTW: there seem to be more packages with ambiguous type (detected with
current Docker image):

```
Digest:
sha256:790197bb223bd7e20b0a2e055aa7b4c5846dc86d94b2425cd233cb6160a5b944
Status: Downloaded newer image for sagemath/sagemath:develop
┌────────────────────────────────────────────────────────────────────┐
│ SageMath version 10.2.rc4, Release Date: 2023-11-17                │
│ Using Python 3.11.1. Type "help()" for help.                       │
└────────────────────────────────────────────────────────────────────┘
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ Warning: this is a prerelease version, and it may be unstable.     ┃
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
sage: from sage.features.all import all_features
sage: [(f, f._spkg_type()) for f in all_features() if f.is_present() and
f._spkg_type() != 'standard']
[(Feature('sage.misc.cython'), 'optional'),
 (Feature('database_cremona_mini_ellcurve': Cremona's database of
elliptic curves),
  'optional'),
 (Feature('gap_package_atlasrep'), 'optional'),
 (Feature('gap_package_polycyclic'), 'optional'),
 (Feature('sage.libs.ecl'), 'optional'),
 (Feature('sage.libs.gap'), 'optional'),
 (Feature('sage.libs.singular'), 'optional'),
 (Feature('sage.numerical.mip'), 'optional')]
sage:
sage: [(f, f._spkg_type()) for f in all_features() if not f.is_present()
and f._spkg_type() == 'standard']
[(Feature('sagemath_doc_html'), 'standard'), (Feature('sage.sat'),
'standard')]
```


### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->
<!-- If your change requires a documentation PR, please link it
appropriately -->
<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
<!-- Feel free to remove irrelevant items. -->

- [x] The title is concise, informative, and self-explanatory.
- [x] The description explains in detail what this PR is about.
- [x] I have linked a relevant issue or discussion.
- [x] I have created tests covering the changes.
- [x] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on
- #12345: short description why this is a dependency
- #34567: ...
-->

<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
    
URL: #36741
Reported by: Sebastian Oehms
Reviewer(s): Matthias Köppe, Sebastian Oehms

Author: Release Manager

src/sage/doctest/control.py

@@ -416,7 +416,7 @@ def __init__(self, options, args):
             if options.gc:
                 options.timeout *= 2
         if options.nthreads == 0:
-            options.nthreads = int(os.getenv('SAGE_NUM_THREADS_PARALLEL',1))
+            options.nthreads = int(os.getenv('SAGE_NUM_THREADS_PARALLEL', 1))
         if options.failed and not (args or options.new):
             # If the user doesn't specify any files then we rerun all failed files.
             options.all = True
@@ -450,15 +450,11 @@ def __init__(self, options, args):
                 options.hide.discard('all')
                 from sage.features.all import all_features
                 feature_names = {f.name for f in all_features() if not f.is_standard()}
-                from sage.doctest.external import external_software
-                feature_names.difference_update(external_software)
                 options.hide = options.hide.union(feature_names)
             if 'optional' in options.hide:
                 options.hide.discard('optional')
                 from sage.features.all import all_features
                 feature_names = {f.name for f in all_features() if f.is_optional()}
-                from sage.doctest.external import external_software
-                feature_names.difference_update(external_software)
                 options.hide = options.hide.union(feature_names)
 
         options.disabled_optional = set()
@@ -1008,7 +1004,7 @@ def expand():
                 if os.path.isdir(path):
                     for root, dirs, files in os.walk(path):
                         for dir in list(dirs):
-                            if dir[0] == "." or skipdir(os.path.join(root,dir)):
+                            if dir[0] == "." or skipdir(os.path.join(root, dir)):
                                 dirs.remove(dir)
                         for file in files:
                             if not skipfile(os.path.join(root, file),
@@ -1345,9 +1341,9 @@ def run_val_gdb(self, testing=False):
                 flags = os.getenv("SAGE_MEMCHECK_FLAGS")
                 if flags is None:
                     flags = "--leak-resolution=high --leak-check=full --num-callers=25 "
-                    flags += '''--suppressions="%s" ''' % (os.path.join(SAGE_EXTCODE,"valgrind", "pyalloc.supp"))
-                    flags += '''--suppressions="%s" ''' % (os.path.join(SAGE_EXTCODE,"valgrind", "sage.supp"))
-                    flags += '''--suppressions="%s" ''' % (os.path.join(SAGE_EXTCODE,"valgrind", "sage-additional.supp"))
+                    flags += '''--suppressions="%s" ''' % (os.path.join(SAGE_EXTCODE, "valgrind", "pyalloc.supp"))
+                    flags += '''--suppressions="%s" ''' % (os.path.join(SAGE_EXTCODE, "valgrind", "sage.supp"))
+                    flags += '''--suppressions="%s" ''' % (os.path.join(SAGE_EXTCODE, "valgrind", "sage-additional.supp"))
             elif opt.massif:
                 toolname = "massif"
                 flags = os.getenv("SAGE_MASSIF_FLAGS", "--depth=6 ")
@@ -1362,7 +1358,7 @@ def run_val_gdb(self, testing=False):
             if opt.omega:
                 toolname = "omega"
             if "%s" in flags:
-                flags %= toolname + ".%p" # replace %s with toolname
+                flags %= toolname + ".%p"  # replace %s with toolname
         cmd += flags + sage_cmd
 
         sys.stdout.flush()
@@ -1489,6 +1485,25 @@ def run(self):
                 cumulative wall time: ... seconds
             Features detected...
             0
+
+        Test *Features that have been hidden* message::
+
+            sage: DC.run()                              # optional - meataxe
+            Running doctests with ID ...
+            Using --optional=sage
+            Features to be detected: ...
+            Doctesting 1 file.
+            sage -t ....py
+                [4 tests, ... s]
+            ----------------------------------------------------------------------
+            All tests passed!
+            ----------------------------------------------------------------------
+            Total time for all tests: ... seconds
+                cpu time: ... seconds
+                cumulative wall time: ... seconds
+            Features detected...
+            Features that have been hidden: ...meataxe...
+            0
         """
         opt = self.options
         L = (opt.gdb, opt.lldb, opt.valgrind, opt.massif, opt.cachegrind, opt.omega)
@@ -1516,10 +1531,10 @@ def run(self):
                     pass
                 try:
                     ref = subprocess.check_output(["git",
-                                                      "--git-dir=" + SAGE_ROOT_GIT,
-                                                      "describe",
-                                                      "--always",
-                                                      "--dirty"])
+                                                   "--git-dir=" + SAGE_ROOT_GIT,
+                                                   "describe",
+                                                   "--always",
+                                                   "--dirty"])
                     ref = ref.decode('utf-8')
                     self.log("Git ref: " + ref, end="")
                 except subprocess.CalledProcessError:
@@ -1537,12 +1552,11 @@ def run(self):
                     pass
                 else:
                     f = available_software._features[i]
-                    if f.is_present():
-                        f.hide()
-                        self.options.hidden_features.add(f)
-                        for g in f.joined_features():
-                            if g.name in self.options.optional:
-                                self.options.optional.discard(g.name)
+                    f.hide()
+                    self.options.hidden_features.add(f)
+                    for g in f.joined_features():
+                        if g.name in self.options.optional:
+                            self.options.optional.discard(g.name)
 
             for o in self.options.disabled_optional:
                 try:
@@ -1553,8 +1567,6 @@ def run(self):
                     available_software._seen[i] = -1
 
             self.log("Features to be detected: " + ','.join(available_software.detectable()))
-            if self.options.hidden_features:
-                self.log("Hidden features: " + ','.join([f.name for f in self.options.hidden_features]))
             if self.options.probe:
                 self.log("Features to be probed: " + ('all' if self.options.probe is True
                                                       else ','.join(self.options.probe)))
@@ -1564,11 +1576,11 @@ def run(self):
             self.sort_sources()
             self.run_doctests()
 
-            for f in self.options.hidden_features:
-                f.unhide()
-
             self.log("Features detected for doctesting: "
                      + ','.join(available_software.seen()))
+            if self.options.hidden_features:
+                features_hidden = [f.name for f in self.options.hidden_features if f.unhide()]
+                self.log("Features that have been hidden: " + ','.join(features_hidden))
             self.cleanup()
             return self.reporter.error_status
 

src/sage/features/__init__.py

@@ -28,7 +28,7 @@
 Here we test whether the grape GAP package is available::
 
     sage: from sage.features.gap import GapPackage
-    sage: GapPackage("grape", spkg="gap_packages").is_present()  # optional - gap_packages
+    sage: GapPackage("grape", spkg="gap_packages").is_present()  # optional - gap_package_grape
     FeatureTestResult('gap_package_grape', True)
 
 Note that a :class:`FeatureTestResult` acts like a bool in most contexts::
@@ -158,6 +158,12 @@ def __init__(self, name, spkg=None, url=None, description=None, type='optional')
         self._hidden = False
         self._type = type
 
+        # For multiprocessing of doctests, the data self._num_hidings should be
+        # shared among subprocesses. Thus we use the Value class from the
+        # multiprocessing module (cf. self._seen of class AvailableSoftware)
+        from multiprocessing import Value
+        self._num_hidings = Value('i', 0)
+
         try:
             from sage.misc.package import spkg_type
         except ImportError:  # may have been surgically removed in a downstream distribution
@@ -182,7 +188,7 @@ def is_present(self):
         EXAMPLES::
 
             sage: from sage.features.gap import GapPackage
-            sage: GapPackage("grape", spkg="gap_packages").is_present()  # optional - gap_packages
+            sage: GapPackage("grape", spkg="gap_packages").is_present()  # optional - gap_package_grape
             FeatureTestResult('gap_package_grape', True)
             sage: GapPackage("NOT_A_PACKAGE", spkg="gap_packages").is_present()
             FeatureTestResult('gap_package_NOT_A_PACKAGE', False)
@@ -205,15 +211,21 @@ def is_present(self):
             sage: TestFeature("other").is_present()
             FeatureTestResult('other', True)
         """
-        if self._hidden:
-            return FeatureTestResult(self, False, reason="Feature `{name}` is hidden.".format(name=self.name))
         # We do not use @cached_method here because we wish to use
         # Feature early in the build system of sagelib.
         if self._cache_is_present is None:
             res = self._is_present()
             if not isinstance(res, FeatureTestResult):
                 res = FeatureTestResult(self, res)
             self._cache_is_present = res
+
+        if self._hidden:
+            if self._num_hidings.value > 0:
+                self._num_hidings.value += 1
+            elif self._cache_is_present:
+                self._num_hidings.value = 1
+            return FeatureTestResult(self, False, reason="Feature `{name}` is hidden.".format(name=self.name))
+
         return self._cache_is_present
 
     def _is_present(self):
@@ -381,40 +393,34 @@ def hide(self):
             Feature `benzene` is hidden.
             Use method `unhide` to make it available again.
 
-            sage: Benzene().unhide()
+            sage: Benzene().unhide()            # optional - benzene, needs sage.graphs
+            1
             sage: len(list(graphs.fusenes(2)))  # optional - benzene, needs sage.graphs
             1
         """
         self._hidden = True
 
     def unhide(self):
         r"""
-        Revert what :meth:`hide` does.
-
-        EXAMPLES:
+        Revert what :meth:`hide` did.
 
-        PolyCyclic is an optional GAP package. The following test
-        fails if it is hidden, regardless of whether it is installed
-        or not::
+        OUTPUT: The number of events a present feature has been hidden.
 
-            sage: from sage.features.gap import GapPackage
-            sage: Polycyclic = GapPackage("polycyclic", spkg="gap_packages")
-            sage: Polycyclic.hide()
-            sage: libgap(AbelianGroup(3, [0,3,4], names="abc"))                         # needs sage.libs.gap  # optional - gap_packages_polycyclic
-            Traceback (most recent call last):
-            ...
-            FeatureNotPresentError: gap_package_polycyclic is not available.
-            Feature `gap_package_polycyclic` is hidden.
-            Use method `unhide` to make it available again.
-
-        After unhiding the feature, the test should pass again if PolyCyclic
-        is installed and loaded::
+        EXAMPLES:
 
-            sage: Polycyclic.unhide()
-            sage: libgap(AbelianGroup(3, [0,3,4], names="abc"))                         # needs sage.libs.gap  # optional - gap_packages_polycyclic
-            Pcp-group with orders [ 0, 3, 4 ]
+            sage: from sage.features.sagemath import sage__plot
+            sage: sage__plot().hide()
+            sage: sage__plot().is_present()
+            FeatureTestResult('sage.plot', False)
+            sage: sage__plot().unhide()                                                 # needs sage.plot
+            1
+            sage: sage__plot().is_present()                                             # needs sage.plot
+            FeatureTestResult('sage.plot', True)
         """
+        num_hidings = self._num_hidings.value
+        self._num_hidings.value = 0
         self._hidden = False
+        return int(num_hidings)
 
 
 class FeatureNotPresentError(RuntimeError):
@@ -802,7 +808,7 @@ class StaticFile(FileFeature):
         To install no_such_file...you can try to run...sage -i some_spkg...
         Further installation instructions might be available at http://rand.om.
     """
-    def __init__(self, name, filename, *, search_path=None, **kwds):
+    def __init__(self, name, filename, *, search_path=None, type='optional', **kwds):
         r"""
         TESTS::
 
@@ -817,7 +823,7 @@ def __init__(self, name, filename, *, search_path=None, **kwds):
             '/bin/sh'
 
         """
-        Feature.__init__(self, name, **kwds)
+        Feature.__init__(self, name, type=type, **kwds)
         self.filename = filename
         if search_path is None:
             self.search_path = [SAGE_SHARE]

src/sage/features/databases.py

@@ -52,12 +52,12 @@ class DatabaseCremona(StaticFile):
     EXAMPLES::
 
         sage: from sage.features.databases import DatabaseCremona
-        sage: DatabaseCremona('cremona_mini').is_present()
+        sage: DatabaseCremona('cremona_mini', type='standard').is_present()
         FeatureTestResult('database_cremona_mini_ellcurve', True)
         sage: DatabaseCremona().is_present()                                    # optional - database_cremona_ellcurve
         FeatureTestResult('database_cremona_ellcurve', True)
     """
-    def __init__(self, name="cremona"):
+    def __init__(self, name="cremona", spkg="database_cremona_ellcurve", type='optional'):
         r"""
         TESTS::
 
@@ -290,7 +290,7 @@ def __init__(self, name='polytopes_db'):
 def all_features():
     return [PythonModule('conway_polynomials', spkg='conway_polynomials', type='standard'),
             DatabaseCremona(),
-            DatabaseCremona('cremona_mini'),
+            DatabaseCremona('cremona_mini', type='standard'),
             DatabaseEllcurves(),
             DatabaseGraphs(),
             DatabaseJones(),

src/sage/features/gap.py

@@ -53,7 +53,7 @@ def _is_present(self):
         EXAMPLES::
 
             sage: from sage.features.gap import GapPackage
-            sage: GapPackage("grape", spkg="gap_packages")._is_present()  # optional - gap_packages
+            sage: GapPackage("grape", spkg="gap_packages")._is_present()  # optional - gap_package_grape
             FeatureTestResult('gap_package_grape', True)
         """
         try:

src/sage/features/join_feature.py

@@ -152,7 +152,9 @@ def hide(self):
 
     def unhide(self):
         r"""
-        Revert what :meth:`hide` does.
+        Revert what :meth:`hide` did.
+
+        OUTPUT: The number of events a present feature has been hidden.
 
         EXAMPLES::
 
@@ -165,11 +167,14 @@ def unhide(self):
             FeatureTestResult('sage.groups.perm_gps.permgroup', False)
 
             sage: f.unhide()
+            4
             sage: f.is_present()    # optional sage.groups
             FeatureTestResult('sage.groups', True)
             sage: f._features[0].is_present() # optional sage.groups
             FeatureTestResult('sage.groups.perm_gps.permgroup', True)
         """
+        num_hidings = 0
         for f in self._features:
-            f.unhide()
-        super().unhide()
+            num_hidings += f.unhide()
+        num_hidings += super().unhide()
+        return num_hidings