PYTHON-2268 Close clients in test suite

Author: ShaneHarvey

pymongo/periodic_executor.py

@@ -50,6 +50,10 @@ def __init__(self, interval, min_interval, target, name=None):
         self._thread_will_exit = False
         self._lock = threading.Lock()
 
+    def __repr__(self):
+        return '<%s(name=%s) object at 0x%x>' % (
+            self.__class__.__name__, self._name, id(self))
+
     def open(self):
         """Start. Multiple calls have no effect.
 

pymongo/settings.py

@@ -15,6 +15,7 @@
 """Represent MongoClient's configuration."""
 
 import threading
+import traceback
 
 from bson.objectid import ObjectId
 from pymongo import common, monitor, pool
@@ -60,6 +61,9 @@ def __init__(self,
         self._heartbeat_frequency = heartbeat_frequency
         self._direct = (len(self._seeds) == 1 and not replica_set_name)
         self._topology_id = ObjectId()
+        # Store the allocation traceback to catch unclosed clients in the
+        # test suite.
+        self._stack = ''.join(traceback.format_stack())
 
     @property
     def seeds(self):

test/__init__.py

@@ -795,6 +795,44 @@ def setup():
     warnings.simplefilter("always")
 
 
+def _get_executors(topology):
+    executors = []
+    for server in topology._servers.values():
+        # Some MockMonitor do not have an _executor.
+        executors.append(getattr(server._monitor, '_executor', None))
+    executors.append(topology._Topology__events_executor)
+    if topology._srv_monitor:
+        executors.append(topology._srv_monitor._executor)
+    return [e for e in executors if e is not None]
+
+
+def all_executors_stopped(topology):
+    running = [e for e in _get_executors(topology) if not e._stopped]
+    if running:
+        print('  Topology %s has THREADS RUNNING: %s, created at: %s' % (
+            topology, running, topology._settings._stack))
+        return False
+    return True
+
+
+def print_unclosed_clients():
+    from pymongo.topology import Topology
+    processed = set()
+    # Call collect to manually cleanup any would-be gc'd clients to avoid
+    # false positives.
+    gc.collect()
+    for obj in gc.get_objects():
+        try:
+            if isinstance(obj, Topology):
+                # Avoid printing the same Topology multiple times.
+                if obj._topology_id in processed:
+                    continue
+                all_executors_stopped(obj)
+                processed.add(obj._topology_id)
+        except ReferenceError:
+            pass
+
+
 def teardown():
     garbage = []
     for g in gc.garbage:
@@ -813,6 +851,10 @@ def teardown():
         c.drop_database("pymongo_test_bernie")
         c.close()
 
+    # Jython does not support gc.get_objects.
+    if not sys.platform.startswith('java'):
+        print_unclosed_clients()
+
 
 class PymongoTestRunner(unittest.TextTestRunner):
     def run(self, test):

test/pymongo_mocks.py

@@ -65,8 +65,9 @@ def __init__(
             topology,
             pool,
             topology_settings):
-        # MockMonitor gets a 'client' arg, regular monitors don't.
-        self.client = client
+        # MockMonitor gets a 'client' arg, regular monitors don't. Weakref it
+        # to avoid cycles.
+        self.client = weakref.proxy(client)
         Monitor.__init__(
             self,
             server_description,
@@ -75,8 +76,9 @@ def __init__(
             topology_settings)
 
     def _check_once(self):
+        client = self.client
         address = self._server_description.address
-        response, rtt = self.client.mock_is_master('%s:%d' % address)
+        response, rtt = client.mock_is_master('%s:%d' % address)
         return ServerDescription(address, IsMaster(response), rtt)
 
 

test/test_auth.py

@@ -695,8 +695,6 @@ def setUp(self):
         client_context.create_user('admin', 'admin', 'pass')
         client_context.create_user(
             'pymongo_test', 'user', 'pass', ['userAdmin', 'readWrite'])
-        self.client = rs_or_single_client_noauth(
-            username='admin', password='pass')
 
     def tearDown(self):
         client_context.drop_user('pymongo_test', 'user')

test/test_client.py

@@ -101,6 +101,10 @@ def setUpClass(cls):
         cls.client = rs_or_single_client(connect=False,
                                          serverSelectionTimeoutMS=100)
 
+    @classmethod
+    def tearDownClass(cls):
+        cls.client.close()
+
     def test_keyword_arg_defaults(self):
         client = MongoClient(socketTimeoutMS=None,
                              connectTimeoutMS=20000,

test/test_collation.py

@@ -105,6 +105,7 @@ def setUpClass(cls):
     def tearDownClass(cls):
         cls.warn_context.__exit__()
         cls.warn_context = None
+        cls.client.close()
 
     def tearDown(self):
         self.listener.results.clear()

test/test_command_monitoring_spec.py

@@ -48,6 +48,10 @@ def setUpClass(cls):
         cls.listener = EventListener()
         cls.client = single_client(event_listeners=[cls.listener])
 
+    @classmethod
+    def tearDownClass(cls):
+        cls.client.close()
+
     def tearDown(self):
         self.listener.results.clear()
 

test/test_connections_survive_primary_stepdown_spec.py

@@ -51,6 +51,10 @@ def setUpClass(cls):
         cls.coll = cls.db.get_collection(
             "step-down", write_concern=WriteConcern("majority"))
 
+    @classmethod
+    def tearDownClass(cls):
+        cls.client.close()
+
     def setUp(self):
         # Note that all ops use same write-concern as self.db (majority).
         self.db.drop_collection("step-down")

test/test_custom_types.py

@@ -911,6 +911,7 @@ def create_targets(self, *args, **kwargs):
             kwargs['type_registry'] = codec_options.type_registry
             kwargs['document_class'] = codec_options.document_class
         self.watched_target = rs_client(*args, **kwargs)
+        self.addCleanup(self.watched_target.close)
         self.input_target = self.watched_target[self.db.name].test
         # Insert a record to ensure db, coll are created.
         self.input_target.insert_one({'data': 'dummy'})