Merge pull request #882 from moreati/macos-11

Fix macOS 11.x and 12.x bootstrap

Author: moreati

.ci/azure-pipelines-steps.yml

@@ -9,11 +9,12 @@ steps:
   displayName: Install python
   inputs:
     versionSpec: '$(python.version)'
+  condition: ne(variables['python.version'], '')
 
 - script: python -mpip install tox
   displayName: Install tooling
 
-- script: tox -e "$(tox.env)"
+- script: python -mtox -e "$(tox.env)"
   displayName: "Run tests"
   env:
     AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)

.ci/azure-pipelines.yml

@@ -9,8 +9,27 @@
   #ANSIBLE_VERBOSITY: 3
 
 jobs:
+- job: Mac1014
+  # vanilla Ansible is really slow
+  timeoutInMinutes: 120
+  steps:
+  - template: azure-pipelines-steps.yml
+  pool:
+    # https://github.com/actions/virtual-environments/blob/main/images/macos/
+    vmImage: macOS-10.14
+  strategy:
+    matrix:
+      Mito_27:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen
+      Mito_36:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen
+      Mito_39:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen
 
-- job: Mac
+- job: Mac1015
   # vanilla Ansible is really slow
   timeoutInMinutes: 120
   steps:
@@ -23,6 +42,12 @@ jobs:
       Mito_27:
         python.version: '2.7'
         tox.env: py27-mode_mitogen
+      Mito_36:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen
+      Mito_39:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen
 
       # TODO: test python3, python3 tests are broken
       Loc_27_210:
@@ -52,6 +77,46 @@ jobs:
         STRATEGY: linear
         ANSIBLE_SKIP_TAGS: resource_intensive
 
+- job: MacLatest
+  # vanilla Ansible is really slow
+  timeoutInMinutes: 120
+  steps:
+  - template: azure-pipelines-steps.yml
+  pool:
+    # https://github.com/actions/virtual-environments/blob/main/images/macos/
+    vmImage: macOS-latest
+  strategy:
+    matrix:
+      Mito_27:
+        tox.env: py27-mode_mitogen
+      Mito_37:
+        python.version: '3.7'
+        tox.env: py37-mode_mitogen
+      Mito_39:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen
+
+      # TODO: test python3, python3 tests are broken
+      Loc_27_210:
+        tox.env: py27-mode_localhost-ansible2.10
+      Loc_27_3:
+        tox.env: py27-mode_localhost-ansible3
+      Loc_27_4:
+        tox.env: py27-mode_localhost-ansible4
+
+      # NOTE: this hangs when ran in Ubuntu 18.04
+      Van_27_210:
+        tox.env: py27-mode_localhost-ansible2.10
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
+      Van_27_3:
+        tox.env: py27-mode_localhost-ansible3
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
+      Van_27_4:
+        tox.env: py27-mode_localhost-ansible4
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
 
 - job: Linux
   pool:

docs/changelog.rst

@@ -21,6 +21,7 @@ To avail of fixes in an unreleased version, please download a ZIP file
 v0.3.1.dev0 (unreleased)
 ------------------------
 
+* :gh:issue:`774` Fix bootstrap failures on macOS 11.x and 12.x, involving Python 2.7 wrapper
 * :gh:issue:`834` Support for Ansible 3 and 4 (ansible-core 2.11)
 * :gh:issue:`869` Continuous Integration tests are now run with Tox
 * :gh:issue:`869` Continuous Integration tests now cover CentOS 6 & 8, Debian 9 & 11, Ubuntu 16.04 & 20.04

mitogen/master.py

@@ -108,7 +108,7 @@ def _stdlib_paths():
     ]
     prefixes = (getattr(sys, a, None) for a in attr_candidates)
     version = 'python%s.%s' % sys.version_info[0:2]
-    s = set(os.path.abspath(os.path.join(p, 'lib', version))
+    s = set(os.path.realpath(os.path.join(p, 'lib', version))
             for p in prefixes if p is not None)
 
     # When running 'unit2 tests/module_finder_test.py' in a Py2 venv on Ubuntu

mitogen/parent.py

@@ -1412,9 +1412,12 @@ def __repr__(self):
     #   * macOS <= 10.14 (Darwin <= 18) install an unreliable Python version
     #     switcher as /usr/bin/python, which introspects argv0. To workaround
     #     it we redirect attempts to call /usr/bin/python with an explicit
-    #     call to /usr/bin/python2.7. macOS 10.15+ (Darwin 19+) removed it.
-    #     On these versions /usr/bin/python is a symlink to
-    #     /System/Library/Frameworks/Python.framework/Versions/2.7/.../Python
+    #     call to /usr/bin/python2.7. macOS 10.15 (Darwin 19) removed it.
+    #   * macOS 11.x (Darwin 20, Big Sur) and macOS 12.x (Darwin 21, Montery)
+    #     do something slightly different. The Python executable is patched to
+    #     perform an extra execvp(). I don't fully understand the details, but
+    #     setting PYTHON_LAUNCHED_FROM_WRAPPER=1 avoids it.
+    #   * macOS 13.x (Darwin 22?) may remove python 2.x entirely.
     #
     # Locals:
     #   R: read side of interpreter stdin.
@@ -1437,7 +1440,8 @@ def _first_stage():
             os.close(r)
             os.close(W)
             os.close(w)
-            if sys.executable+sys.platform=='/usr/bin/pythondarwin'and os.uname()[2]<'19':sys.executable+='2.7'
+            if os.uname()[0]=='Darwin'and os.uname()[2][:2]<'19'and sys.executable=='/usr/bin/python':sys.executable='/usr/bin/python2.7'
+            if os.uname()[0]=='Darwin'and os.uname()[2][:2]in'2021'and sys.version[:3]=='2.7':os.environ['PYTHON_LAUNCHED_FROM_WRAPPER']='1'
             os.environ['ARGV0']=sys.executable
             os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)')
         os.write(1,'MITO000\n'.encode())

tests/testlib.py

@@ -1,4 +1,5 @@
 
+import errno
 import logging
 import os
 import random
@@ -11,6 +12,7 @@
 import time
 import traceback
 
+import psutil
 import unittest2
 
 import mitogen.core
@@ -67,7 +69,6 @@ def get_fd_count():
     """
     Return the number of FDs open by this process.
     """
-    import psutil
     return psutil.Process().num_fds()
 
 
@@ -334,6 +335,10 @@ def setUpClass(cls):
         # Broker() instantiations in setUp() etc.
         mitogen.fork.on_fork()
         cls._fd_count_before = get_fd_count()
+        # Ignore children started by external packages - in particular
+        # multiprocessing.resource_tracker.main()`, started when some Ansible
+        # versions instantiate a `multithreading.Lock()`.
+        cls._children_before = frozenset(psutil.Process().children())
         super(TestCase, cls).setUpClass()
 
     ALLOWED_THREADS = set([
@@ -361,7 +366,10 @@ def _teardown_check_threads(self):
     def _teardown_check_fds(self):
         mitogen.core.Latch._on_fork()
         if get_fd_count() != self._fd_count_before:
-            import os; os.system('lsof +E -w -p %s | grep -vw mem' % (os.getpid(),))
+            if sys.platform == 'linux':
+                os.system('lsof +E -w -p %i | grep -vw mem' % (os.getpid(),))
+            else:
+                os.system('lsof -w -p %i | grep -vw mem' % (os.getpid(),))
             assert 0, "%s leaked FDs. Count before: %s, after: %s" % (
                 self, self._fd_count_before, get_fd_count(),
             )
@@ -374,19 +382,33 @@ def _teardown_check_zombies(self):
         if self.no_zombie_check:
             return
 
+        # pid=0: Wait for any child process in the same process group as us.
+        # WNOHANG: Don't block if no processes ready to report status.
         try:
             pid, status = os.waitpid(0, os.WNOHANG)
-        except OSError:
-            return  # ECHILD
+        except OSError as e:
+            # ECHILD: there are no child processes in our group.
+            if e.errno == errno.ECHILD:
+                return
+            raise
 
         if pid:
             assert 0, "%s failed to reap subprocess %d (status %d)." % (
                 self, pid, status
             )
 
-        print('')
-        print('Children of unit test process:')
-        os.system('ps uww --ppid ' + str(os.getpid()))
+        children_after = frozenset(psutil.Process().children())
+        children_leaked = children_after.difference(self._children_before)
+        if not children_leaked:
+            return
+
+        print('Leaked children of unit test process:')
+        os.system('ps -o "user,pid,%%cpu,%%mem,vsz,rss,tty,stat,start,time,command" -ww -p %s'
+                  % (','.join(str(p.pid) for p in children_leaked),))
+        if self._children_before:
+            print('Pre-existing children of unit test process:')
+            os.system('ps -o "user,pid,%%cpu,%%mem,vsz,rss,tty,stat,start,time,command" -ww -p %s'
+                      % (','.join(str(p.pid) for p in self._children_before),))
         assert 0, "%s leaked still-running subprocesses." % (self,)
 
     def tearDown(self):