Experimental: Don't send related modules proactively

docs/index.rst

@@ -148,12 +148,6 @@ Common sources of import latency and bandwidth consumption are mitigated:
   statement semantics: children have a list of submodules belonging to a
   package, and ignore requests for submodules that did not exist on the master.
 
-* Imports are extracted from each module, compared to those found in memory,
-  and recursively preloaded into children requesting that module, minimizing
-  round-trips to one per package nesting level. For example,
-  :py:mod:`django.db.models` only requires 3 round-trips to transfer 456KiB,
-  representing 1.7MiB of uncompressed source split across 148 modules.
-
 
 Message Routing
 ###############

mitogen/master.py

@@ -923,9 +923,6 @@ def __init__(self):
         #: results around.
         self._found_cache = {}
 
-        #: Avoid repeated dependency scanning, which is expensive.
-        self._related_cache = {}
-
     def __repr__(self):
         return 'ModuleFinder()'
 
@@ -1006,78 +1003,6 @@ def generate_parent_names(self, fullname):
             fullname, _, _ = str_rpartition(to_text(fullname), u'.')
             yield fullname
 
-    def find_related_imports(self, fullname):
-        """
-        Return a list of non-stdlib modules that are directly imported by
-        `fullname`, plus their parents.
-
-        The list is determined by retrieving the source code of
-        `fullname`, compiling it, and examining all IMPORT_NAME ops.
-
-        :param fullname: Fully qualified name of an *already imported* module
-            for which source code can be retrieved
-        :type fullname: str
-        """
-        related = self._related_cache.get(fullname)
-        if related is not None:
-            return related
-
-        modpath, src, _ = self.get_module_source(fullname)
-        if src is None:
-            return []
-
-        maybe_names = list(self.generate_parent_names(fullname))
-
-        co = compile(src, modpath, 'exec')
-        for level, modname, namelist in scan_code_imports(co):
-            if level == -1:
-                modnames = [modname, '%s.%s' % (fullname, modname)]
-            else:
-                modnames = [
-                    '%s%s' % (self.resolve_relpath(fullname, level), modname)
-                ]
-
-            maybe_names.extend(modnames)
-            maybe_names.extend(
-                '%s.%s' % (mname, name)
-                for mname in modnames
-                for name in namelist
-            )
-
-        return self._related_cache.setdefault(fullname, sorted(
-            set(
-                mitogen.core.to_text(name)
-                for name in maybe_names
-                if sys.modules.get(name) is not None
-                and not is_stdlib_name(name)
-                and u'six.moves' not in name  # TODO: crap
-            )
-        ))
-
-    def find_related(self, fullname):
-        """
-        Return a list of non-stdlib modules that are imported directly or
-        indirectly by `fullname`, plus their parents.
-
-        This method is like :py:meth:`find_related_imports`, but also
-        recursively searches any modules which are imported by `fullname`.
-
-        :param fullname: Fully qualified name of an *already imported* module
-            for which source code can be retrieved
-        :type fullname: str
-        """
-        stack = [fullname]
-        found = set()
-
-        while stack:
-            name = stack.pop(0)
-            names = self.find_related_imports(name)
-            stack.extend(set(names).difference(set(found).union(stack)))
-            found.update(names)
-
-        found.discard(fullname)
-        return sorted(found)
-
 
 class ModuleResponder(object):
     def __init__(self, router):
@@ -1197,18 +1122,13 @@ def _build_tuple(self, fullname):
         if fullname == '__main__':
             source = self.neutralize_main(path, source)
         compressed = mitogen.core.Blob(zlib.compress(source, 9))
-        related = [
-            to_text(name)
-            for name in self._finder.find_related(fullname)
-            if not mitogen.core.is_blacklisted_import(self, name)
-        ]
         # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
         tup = (
             to_text(fullname),
             pkg_present,
             to_text(path),
             compressed,
-            related
+            [],
         )
         self._cache[fullname] = tup
         return tup
@@ -1241,19 +1161,11 @@ def _send_module_load_failed(self, stream, fullname):
             )
         )
 
-    def _send_module_and_related(self, stream, fullname):
+    def _send_module(self, stream, fullname):
         if fullname in stream.protocol.sent_modules:
             return
 
         try:
-            tup = self._build_tuple(fullname)
-            for name in tup[4]:  # related
-                parent, _, _ = str_partition(name, '.')
-                if parent != fullname and parent not in stream.protocol.sent_modules:
-                    # Parent hasn't been sent, so don't load submodule yet.
-                    continue
-
-                self._send_load_module(stream, name)
             self._send_load_module(stream, fullname)
         except Exception:
             LOG.debug('While importing %r', fullname, exc_info=True)
@@ -1276,7 +1188,7 @@ def _on_get_module(self, msg):
 
         t0 = mitogen.core.now()
         try:
-            self._send_module_and_related(stream, fullname)
+            self._send_module(stream, fullname)
         finally:
             self.get_module_secs += mitogen.core.now() - t0
 
@@ -1311,7 +1223,7 @@ def _forward_one_module(self, context, fullname):
             return
 
         for fullname in reversed(path):
-            self._send_module_and_related(stream, fullname)
+            self._send_module(stream, fullname)
             self._send_forward_module(stream, context, fullname)
 
     def _forward_modules(self, context, fullnames):

mitogen/parent.py

@@ -2744,7 +2744,7 @@ def _on_forward_module(self, msg):
 
         LOG.debug('%r._on_forward_module() sending %r to %r via %r',
                   self, fullname, context_id, stream.protocol.remote_id)
-        self._send_module_and_related(stream, fullname)
+        self._send_module(stream, fullname)
         if stream.protocol.remote_id != context_id:
             stream.protocol._send(
                 mitogen.core.Message(
@@ -2766,18 +2766,10 @@ def _on_get_module(self, msg):
     def _on_cache_callback(self, msg, fullname):
         stream = self.router.stream_by_id(msg.src_id)
         LOG.debug('%r: sending %s to %r', self, fullname, stream)
-        self._send_module_and_related(stream, fullname)
+        self._send_module(stream, fullname)
 
-    def _send_module_and_related(self, stream, fullname):
+    def _send_module(self, stream, fullname):
         tup = self.importer._cache[fullname]
-        for related in tup[4]:
-            rtup = self.importer._cache.get(related)
-            if rtup:
-                self._send_one_module(stream, rtup)
-            else:
-                LOG.debug('%r: %s not in cache (for %s)',
-                          self, related, fullname)
-
         self._send_one_module(stream, tup)
 
     def _send_one_module(self, stream, tup):