diff --git a/docs/index.rst b/docs/index.rst index 32083db0a..08379d85e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -148,12 +148,6 @@ Common sources of import latency and bandwidth consumption are mitigated: statement semantics: children have a list of submodules belonging to a package, and ignore requests for submodules that did not exist on the master. -* Imports are extracted from each module, compared to those found in memory, - and recursively preloaded into children requesting that module, minimizing - round-trips to one per package nesting level. For example, - :py:mod:`django.db.models` only requires 3 round-trips to transfer 456KiB, - representing 1.7MiB of uncompressed source split across 148 modules. - Message Routing ############### diff --git a/mitogen/master.py b/mitogen/master.py index 927ccaf11..cbd9bdf15 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -923,9 +923,6 @@ def __init__(self): #: results around. self._found_cache = {} - #: Avoid repeated dependency scanning, which is expensive. - self._related_cache = {} - def __repr__(self): return 'ModuleFinder()' @@ -1006,78 +1003,6 @@ def generate_parent_names(self, fullname): fullname, _, _ = str_rpartition(to_text(fullname), u'.') yield fullname - def find_related_imports(self, fullname): - """ - Return a list of non-stdlib modules that are directly imported by - `fullname`, plus their parents. - - The list is determined by retrieving the source code of - `fullname`, compiling it, and examining all IMPORT_NAME ops. - - :param fullname: Fully qualified name of an *already imported* module - for which source code can be retrieved - :type fullname: str - """ - related = self._related_cache.get(fullname) - if related is not None: - return related - - modpath, src, _ = self.get_module_source(fullname) - if src is None: - return [] - - maybe_names = list(self.generate_parent_names(fullname)) - - co = compile(src, modpath, 'exec') - for level, modname, namelist in scan_code_imports(co): - if level == -1: - modnames = [modname, '%s.%s' % (fullname, modname)] - else: - modnames = [ - '%s%s' % (self.resolve_relpath(fullname, level), modname) - ] - - maybe_names.extend(modnames) - maybe_names.extend( - '%s.%s' % (mname, name) - for mname in modnames - for name in namelist - ) - - return self._related_cache.setdefault(fullname, sorted( - set( - mitogen.core.to_text(name) - for name in maybe_names - if sys.modules.get(name) is not None - and not is_stdlib_name(name) - and u'six.moves' not in name # TODO: crap - ) - )) - - def find_related(self, fullname): - """ - Return a list of non-stdlib modules that are imported directly or - indirectly by `fullname`, plus their parents. - - This method is like :py:meth:`find_related_imports`, but also - recursively searches any modules which are imported by `fullname`. - - :param fullname: Fully qualified name of an *already imported* module - for which source code can be retrieved - :type fullname: str - """ - stack = [fullname] - found = set() - - while stack: - name = stack.pop(0) - names = self.find_related_imports(name) - stack.extend(set(names).difference(set(found).union(stack))) - found.update(names) - - found.discard(fullname) - return sorted(found) - class ModuleResponder(object): def __init__(self, router): @@ -1197,18 +1122,13 @@ def _build_tuple(self, fullname): if fullname == '__main__': source = self.neutralize_main(path, source) compressed = mitogen.core.Blob(zlib.compress(source, 9)) - related = [ - to_text(name) - for name in self._finder.find_related(fullname) - if not mitogen.core.is_blacklisted_import(self, name) - ] # 0:fullname 1:pkg_present 2:path 3:compressed 4:related tup = ( to_text(fullname), pkg_present, to_text(path), compressed, - related + [], ) self._cache[fullname] = tup return tup @@ -1241,19 +1161,11 @@ def _send_module_load_failed(self, stream, fullname): ) ) - def _send_module_and_related(self, stream, fullname): + def _send_module(self, stream, fullname): if fullname in stream.protocol.sent_modules: return try: - tup = self._build_tuple(fullname) - for name in tup[4]: # related - parent, _, _ = str_partition(name, '.') - if parent != fullname and parent not in stream.protocol.sent_modules: - # Parent hasn't been sent, so don't load submodule yet. - continue - - self._send_load_module(stream, name) self._send_load_module(stream, fullname) except Exception: LOG.debug('While importing %r', fullname, exc_info=True) @@ -1276,7 +1188,7 @@ def _on_get_module(self, msg): t0 = mitogen.core.now() try: - self._send_module_and_related(stream, fullname) + self._send_module(stream, fullname) finally: self.get_module_secs += mitogen.core.now() - t0 @@ -1311,7 +1223,7 @@ def _forward_one_module(self, context, fullname): return for fullname in reversed(path): - self._send_module_and_related(stream, fullname) + self._send_module(stream, fullname) self._send_forward_module(stream, context, fullname) def _forward_modules(self, context, fullnames): diff --git a/mitogen/parent.py b/mitogen/parent.py index 7ac14fb11..379a5593f 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -2744,7 +2744,7 @@ def _on_forward_module(self, msg): LOG.debug('%r._on_forward_module() sending %r to %r via %r', self, fullname, context_id, stream.protocol.remote_id) - self._send_module_and_related(stream, fullname) + self._send_module(stream, fullname) if stream.protocol.remote_id != context_id: stream.protocol._send( mitogen.core.Message( @@ -2766,18 +2766,10 @@ def _on_get_module(self, msg): def _on_cache_callback(self, msg, fullname): stream = self.router.stream_by_id(msg.src_id) LOG.debug('%r: sending %s to %r', self, fullname, stream) - self._send_module_and_related(stream, fullname) + self._send_module(stream, fullname) - def _send_module_and_related(self, stream, fullname): + def _send_module(self, stream, fullname): tup = self.importer._cache[fullname] - for related in tup[4]: - rtup = self.importer._cache.get(related) - if rtup: - self._send_one_module(stream, rtup) - else: - LOG.debug('%r: %s not in cache (for %s)', - self, related, fullname) - self._send_one_module(stream, tup) def _send_one_module(self, stream, tup): diff --git a/tests/data/importer/webproject/manage.py b/tests/data/importer/webproject/manage.py deleted file mode 100755 index 92a4a08fe..000000000 --- a/tests/data/importer/webproject/manage.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "webproject.settings") - - from django.core.management import execute_from_command_line - - execute_from_command_line(sys.argv) diff --git a/tests/data/importer/webproject/modules_expected_py2x.json b/tests/data/importer/webproject/modules_expected_py2x.json deleted file mode 100644 index 4bf0bd399..000000000 --- a/tests/data/importer/webproject/modules_expected_py2x.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "find_related_imports": { - "django.db": [ - "django", - "django.core", - "django.core.signals", - "django.db.utils" - ], - "django.db.models": [ - "django", - "django.core.exceptions", - "django.db", - "django.db.models", - "django.db.models.aggregates", - "django.db.models.base", - "django.db.models.deletion", - "django.db.models.expressions", - "django.db.models.fields", - "django.db.models.fields.files", - "django.db.models.fields.proxy", - "django.db.models.fields.related", - "django.db.models.indexes", - "django.db.models.lookups", - "django.db.models.manager", - "django.db.models.query", - "django.db.models.signals" - ] - }, - "find_related": { - "django.db": [ - "django", - "django.conf", - "django.conf.global_settings", - "django.core", - "django.core.exceptions", - "django.core.signals", - "django.db.utils", - "django.dispatch", - "django.dispatch.dispatcher", - "django.dispatch.weakref_backports", - "django.utils", - "django.utils._os", - "django.utils.deprecation", - "django.utils.encoding", - "django.utils.functional", - "django.utils.inspect", - "django.utils.lru_cache", - "django.utils.module_loading", - "django.utils.six", - "django.utils.version" - ], - "django.db.models": [ - "django", - "django.apps", - "django.apps.config", - "django.apps.registry", - "django.conf", - "django.conf.global_settings", - "django.core", - "django.core.cache", - "django.core.cache.backends", - "django.core.cache.backends.base", - "django.core.checks", - "django.core.checks.caches", - "django.core.checks.compatibility", - "django.core.checks.compatibility.django_1_10", - "django.core.checks.compatibility.django_1_8_0", - "django.core.checks.database", - "django.core.checks.messages", - "django.core.checks.model_checks", - "django.core.checks.registry", - "django.core.checks.security", - "django.core.checks.security.base", - "django.core.checks.security.csrf", - "django.core.checks.security.sessions", - "django.core.checks.templates", - "django.core.checks.urls", - "django.core.checks.utils", - "django.core.exceptions", - "django.core.files", - "django.core.files.base", - "django.core.files.images", - "django.core.files.locks", - "django.core.files.move", - "django.core.files.storage", - "django.core.files.utils", - "django.core.signals", - "django.core.validators", - "django.db", - "django.db.backends", - "django.db.backends.utils", - "django.db.models.aggregates", - "django.db.models.base", - "django.db.models.constants", - "django.db.models.deletion", - "django.db.models.expressions", - "django.db.models.fields", - "django.db.models.fields.files", - "django.db.models.fields.proxy", - "django.db.models.fields.related", - "django.db.models.fields.related_descriptors", - "django.db.models.fields.related_lookups", - "django.db.models.fields.reverse_related", - "django.db.models.functions", - "django.db.models.functions.base", - "django.db.models.functions.datetime", - "django.db.models.indexes", - "django.db.models.lookups", - "django.db.models.manager", - "django.db.models.options", - "django.db.models.query", - "django.db.models.query_utils", - "django.db.models.signals", - "django.db.models.sql", - "django.db.models.sql.constants", - "django.db.models.sql.datastructures", - "django.db.models.sql.query", - "django.db.models.sql.subqueries", - "django.db.models.sql.where", - "django.db.models.utils", - "django.db.transaction", - "django.db.utils", - "django.dispatch", - "django.dispatch.dispatcher", - "django.dispatch.weakref_backports", - "django.forms", - "django.forms.boundfield", - "django.forms.fields", - "django.forms.forms", - "django.forms.formsets", - "django.forms.models", - "django.forms.renderers", - "django.forms.utils", - "django.forms.widgets", - "django.template", - "django.template.backends", - "django.template.backends.base", - "django.template.backends.django", - "django.template.base", - "django.template.context", - "django.template.engine", - "django.template.exceptions", - "django.template.library", - "django.template.loader", - "django.template.utils", - "django.templatetags", - "django.templatetags.static", - "django.utils", - "django.utils._os", - "django.utils.crypto", - "django.utils.datastructures", - "django.utils.dateformat", - "django.utils.dateparse", - "django.utils.dates", - "django.utils.datetime_safe", - "django.utils.deconstruct", - "django.utils.decorators", - "django.utils.deprecation", - "django.utils.duration", - "django.utils.encoding", - "django.utils.formats", - "django.utils.functional", - "django.utils.html", - "django.utils.html_parser", - "django.utils.http", - "django.utils.inspect", - "django.utils.ipv6", - "django.utils.itercompat", - "django.utils.lru_cache", - "django.utils.module_loading", - "django.utils.numberformat", - "django.utils.safestring", - "django.utils.six", - "django.utils.text", - "django.utils.timezone", - "django.utils.translation", - "django.utils.tree", - "django.utils.version", - "pytz", - "pytz.exceptions", - "pytz.lazy", - "pytz.tzfile", - "pytz.tzinfo" - ] - } -} diff --git a/tests/data/importer/webproject/modules_expected_py3x-legacy.json b/tests/data/importer/webproject/modules_expected_py3x-legacy.json deleted file mode 100644 index 06e94e620..000000000 --- a/tests/data/importer/webproject/modules_expected_py3x-legacy.json +++ /dev/null @@ -1,209 +0,0 @@ -{ - "find_related_imports": { - "django.db": [ - "django", - "django.core", - "django.core.signals", - "django.db.utils", - "django.utils.connection" - ], - "django.db.models": [ - "django", - "django.core.exceptions", - "django.db", - "django.db.models", - "django.db.models.aggregates", - "django.db.models.base", - "django.db.models.constraints", - "django.db.models.deletion", - "django.db.models.enums", - "django.db.models.expressions", - "django.db.models.fields", - "django.db.models.fields.files", - "django.db.models.fields.json", - "django.db.models.fields.proxy", - "django.db.models.fields.related", - "django.db.models.indexes", - "django.db.models.lookups", - "django.db.models.manager", - "django.db.models.query", - "django.db.models.query_utils", - "django.db.models.signals" - ] - }, - "find_related": { - "django.db": [ - "asgiref", - "asgiref.compatibility", - "asgiref.current_thread_executor", - "asgiref.local", - "asgiref.sync", - "django", - "django.conf", - "django.conf.global_settings", - "django.core", - "django.core.exceptions", - "django.core.signals", - "django.db.utils", - "django.dispatch", - "django.dispatch.dispatcher", - "django.utils", - "django.utils.connection", - "django.utils.deprecation", - "django.utils.functional", - "django.utils.hashable", - "django.utils.inspect", - "django.utils.itercompat", - "django.utils.module_loading", - "django.utils.regex_helper", - "django.utils.version" - ], - "django.db.models": [ - "asgiref", - "asgiref.compatibility", - "asgiref.current_thread_executor", - "asgiref.local", - "asgiref.sync", - "django", - "django.apps", - "django.apps.config", - "django.apps.registry", - "django.conf", - "django.conf.global_settings", - "django.conf.locale", - "django.core", - "django.core.cache", - "django.core.cache.backends", - "django.core.cache.backends.base", - "django.core.cache.backends.filebased", - "django.core.checks", - "django.core.checks.async_checks", - "django.core.checks.caches", - "django.core.checks.database", - "django.core.checks.messages", - "django.core.checks.model_checks", - "django.core.checks.registry", - "django.core.checks.security", - "django.core.checks.security.base", - "django.core.checks.security.csrf", - "django.core.checks.security.sessions", - "django.core.checks.templates", - "django.core.checks.translation", - "django.core.checks.urls", - "django.core.exceptions", - "django.core.files", - "django.core.files.base", - "django.core.files.images", - "django.core.files.locks", - "django.core.files.move", - "django.core.files.storage", - "django.core.files.utils", - "django.core.signals", - "django.core.validators", - "django.db", - "django.db.backends", - "django.db.backends.utils", - "django.db.models.aggregates", - "django.db.models.base", - "django.db.models.constants", - "django.db.models.constraints", - "django.db.models.deletion", - "django.db.models.enums", - "django.db.models.expressions", - "django.db.models.fields", - "django.db.models.fields.files", - "django.db.models.fields.json", - "django.db.models.fields.mixins", - "django.db.models.fields.proxy", - "django.db.models.fields.related", - "django.db.models.fields.related_descriptors", - "django.db.models.fields.related_lookups", - "django.db.models.fields.reverse_related", - "django.db.models.functions", - "django.db.models.functions.comparison", - "django.db.models.functions.datetime", - "django.db.models.functions.math", - "django.db.models.functions.mixins", - "django.db.models.functions.text", - "django.db.models.functions.window", - "django.db.models.indexes", - "django.db.models.lookups", - "django.db.models.manager", - "django.db.models.options", - "django.db.models.query", - "django.db.models.query_utils", - "django.db.models.signals", - "django.db.models.sql", - "django.db.models.sql.constants", - "django.db.models.sql.datastructures", - "django.db.models.sql.query", - "django.db.models.sql.subqueries", - "django.db.models.sql.where", - "django.db.models.utils", - "django.db.transaction", - "django.db.utils", - "django.dispatch", - "django.dispatch.dispatcher", - "django.forms", - "django.forms.boundfield", - "django.forms.fields", - "django.forms.forms", - "django.forms.formsets", - "django.forms.models", - "django.forms.renderers", - "django.forms.utils", - "django.forms.widgets", - "django.template", - "django.template.backends", - "django.template.backends.base", - "django.template.backends.django", - "django.template.base", - "django.template.context", - "django.template.engine", - "django.template.exceptions", - "django.template.library", - "django.template.loader", - "django.template.utils", - "django.templatetags", - "django.templatetags.static", - "django.utils", - "django.utils._os", - "django.utils.autoreload", - "django.utils.connection", - "django.utils.crypto", - "django.utils.datastructures", - "django.utils.dateformat", - "django.utils.dateparse", - "django.utils.dates", - "django.utils.datetime_safe", - "django.utils.deconstruct", - "django.utils.deprecation", - "django.utils.duration", - "django.utils.encoding", - "django.utils.formats", - "django.utils.functional", - "django.utils.hashable", - "django.utils.html", - "django.utils.http", - "django.utils.inspect", - "django.utils.ipv6", - "django.utils.itercompat", - "django.utils.module_loading", - "django.utils.numberformat", - "django.utils.regex_helper", - "django.utils.safestring", - "django.utils.text", - "django.utils.timezone", - "django.utils.topological_sort", - "django.utils.translation", - "django.utils.translation.trans_real", - "django.utils.tree", - "django.utils.version", - "pytz", - "pytz.exceptions", - "pytz.lazy", - "pytz.tzfile", - "pytz.tzinfo" - ] - } -} diff --git a/tests/data/importer/webproject/modules_expected_py3x-new.json b/tests/data/importer/webproject/modules_expected_py3x-new.json deleted file mode 100644 index 614a7f9c3..000000000 --- a/tests/data/importer/webproject/modules_expected_py3x-new.json +++ /dev/null @@ -1,207 +0,0 @@ -{ - "find_related_imports": { - "django.db": [ - "django", - "django.core", - "django.core.signals", - "django.db.utils", - "django.utils.connection" - ], - "django.db.models": [ - "django", - "django.core.exceptions", - "django.db", - "django.db.models", - "django.db.models.aggregates", - "django.db.models.base", - "django.db.models.constraints", - "django.db.models.deletion", - "django.db.models.enums", - "django.db.models.expressions", - "django.db.models.fields", - "django.db.models.fields.files", - "django.db.models.fields.json", - "django.db.models.fields.proxy", - "django.db.models.fields.related", - "django.db.models.indexes", - "django.db.models.lookups", - "django.db.models.manager", - "django.db.models.query", - "django.db.models.query_utils", - "django.db.models.signals" -] - }, - "find_related": { - "django.db": [ - "asgiref", - "asgiref.current_thread_executor", - "asgiref.local", - "asgiref.sync", - "django", - "django.conf", - "django.conf.global_settings", - "django.core", - "django.core.exceptions", - "django.core.signals", - "django.db.utils", - "django.dispatch", - "django.dispatch.dispatcher", - "django.utils", - "django.utils.connection", - "django.utils.deprecation", - "django.utils.functional", - "django.utils.hashable", - "django.utils.inspect", - "django.utils.itercompat", - "django.utils.module_loading", - "django.utils.regex_helper", - "django.utils.version" - ], - "django.db.models": [ - "asgiref", - "asgiref.current_thread_executor", - "asgiref.local", - "asgiref.sync", - "django", - "django.apps", - "django.apps.config", - "django.apps.registry", - "django.conf", - "django.conf.global_settings", - "django.conf.locale", - "django.core", - "django.core.cache", - "django.core.cache.backends", - "django.core.cache.backends.base", - "django.core.cache.backends.filebased", - "django.core.checks", - "django.core.checks.async_checks", - "django.core.checks.caches", - "django.core.checks.database", - "django.core.checks.messages", - "django.core.checks.model_checks", - "django.core.checks.registry", - "django.core.checks.security", - "django.core.checks.security.base", - "django.core.checks.security.csrf", - "django.core.checks.security.sessions", - "django.core.checks.templates", - "django.core.checks.translation", - "django.core.checks.urls", - "django.core.exceptions", - "django.core.files", - "django.core.files.base", - "django.core.files.images", - "django.core.files.locks", - "django.core.files.move", - "django.core.files.storage", - "django.core.files.utils", - "django.core.signals", - "django.core.validators", - "django.db", - "django.db.backends", - "django.db.backends.utils", - "django.db.models.aggregates", - "django.db.models.base", - "django.db.models.constants", - "django.db.models.constraints", - "django.db.models.deletion", - "django.db.models.enums", - "django.db.models.expressions", - "django.db.models.fields", - "django.db.models.fields.files", - "django.db.models.fields.json", - "django.db.models.fields.mixins", - "django.db.models.fields.proxy", - "django.db.models.fields.related", - "django.db.models.fields.related_descriptors", - "django.db.models.fields.related_lookups", - "django.db.models.fields.reverse_related", - "django.db.models.functions", - "django.db.models.functions.comparison", - "django.db.models.functions.datetime", - "django.db.models.functions.math", - "django.db.models.functions.mixins", - "django.db.models.functions.text", - "django.db.models.functions.window", - "django.db.models.indexes", - "django.db.models.lookups", - "django.db.models.manager", - "django.db.models.options", - "django.db.models.query", - "django.db.models.query_utils", - "django.db.models.signals", - "django.db.models.sql", - "django.db.models.sql.constants", - "django.db.models.sql.datastructures", - "django.db.models.sql.query", - "django.db.models.sql.subqueries", - "django.db.models.sql.where", - "django.db.models.utils", - "django.db.transaction", - "django.db.utils", - "django.dispatch", - "django.dispatch.dispatcher", - "django.forms", - "django.forms.boundfield", - "django.forms.fields", - "django.forms.forms", - "django.forms.formsets", - "django.forms.models", - "django.forms.renderers", - "django.forms.utils", - "django.forms.widgets", - "django.template", - "django.template.backends", - "django.template.backends.base", - "django.template.backends.django", - "django.template.base", - "django.template.context", - "django.template.engine", - "django.template.exceptions", - "django.template.library", - "django.template.loader", - "django.template.utils", - "django.templatetags", - "django.templatetags.static", - "django.utils", - "django.utils._os", - "django.utils.autoreload", - "django.utils.connection", - "django.utils.crypto", - "django.utils.datastructures", - "django.utils.dateformat", - "django.utils.dateparse", - "django.utils.dates", - "django.utils.datetime_safe", - "django.utils.deconstruct", - "django.utils.deprecation", - "django.utils.duration", - "django.utils.encoding", - "django.utils.formats", - "django.utils.functional", - "django.utils.hashable", - "django.utils.html", - "django.utils.http", - "django.utils.inspect", - "django.utils.ipv6", - "django.utils.itercompat", - "django.utils.module_loading", - "django.utils.numberformat", - "django.utils.regex_helper", - "django.utils.safestring", - "django.utils.text", - "django.utils.timezone", - "django.utils.topological_sort", - "django.utils.translation", - "django.utils.translation.trans_real", - "django.utils.tree", - "django.utils.version", - "pytz", - "pytz.exceptions", - "pytz.lazy", - "pytz.tzfile", - "pytz.tzinfo" - ] - } -} diff --git a/tests/data/importer/webproject/serve_django_app.py b/tests/data/importer/webproject/serve_django_app.py deleted file mode 100644 index a9ad1e804..000000000 --- a/tests/data/importer/webproject/serve_django_app.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -import sys - -import mitogen -import mitogen.master -import mitogen.utils - - -import sys -sys.path.insert(0, '..') - - -def serve_django_app(settings_name): - os.listdir = lambda path: [] - - os.environ['DJANGO_SETTINGS_MODULE'] = settings_name - import django - args = ['manage.py', 'runserver', '0:9191', '--noreload'] - from django.conf import settings - #settings.configure() - django.setup() - from django.core.management.commands import runserver - runserver.Command().run_from_argv(args) - #django.core.management.execute_from_command_line(args) - - -def main(broker): - import logging - mitogen.utils.log_to_file(io=False) - context = mitogen.master.connect(broker) - context.call(os.chdir, '/') - #context.call(mitogen.utils.log_to_file, '/tmp/log') - context.call(serve_django_app, 'webproject.settings') - -if __name__ == '__main__' and mitogen.is_master: - mitogen.utils.run_with_broker(main) diff --git a/tests/data/importer/webproject/webapp/__init__.py b/tests/data/importer/webproject/webapp/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/data/importer/webproject/webapp/admin.py b/tests/data/importer/webproject/webapp/admin.py deleted file mode 100644 index 8c38f3f3d..000000000 --- a/tests/data/importer/webproject/webapp/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/tests/data/importer/webproject/webapp/apps.py b/tests/data/importer/webproject/webapp/apps.py deleted file mode 100644 index 627128104..000000000 --- a/tests/data/importer/webproject/webapp/apps.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import unicode_literals - -from django.apps import AppConfig - - -class WebappConfig(AppConfig): - name = 'webapp' diff --git a/tests/data/importer/webproject/webapp/migrations/__init__.py b/tests/data/importer/webproject/webapp/migrations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/data/importer/webproject/webapp/models.py b/tests/data/importer/webproject/webapp/models.py deleted file mode 100644 index bd4b2abe9..000000000 --- a/tests/data/importer/webproject/webapp/models.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import unicode_literals - -from django.db import models - -# Create your models here. diff --git a/tests/data/importer/webproject/webapp/tests.py b/tests/data/importer/webproject/webapp/tests.py deleted file mode 100644 index 7ce503c2d..000000000 --- a/tests/data/importer/webproject/webapp/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/tests/data/importer/webproject/webapp/views.py b/tests/data/importer/webproject/webapp/views.py deleted file mode 100644 index d0f9cc491..000000000 --- a/tests/data/importer/webproject/webapp/views.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.http import HTTPResponse - - -def index(request): - return HTTPResponse('hello, world') - - -def shutdown(request): - raise KeyboardInterrupt diff --git a/tests/data/importer/webproject/webproject/__init__.py b/tests/data/importer/webproject/webproject/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/data/importer/webproject/webproject/settings.py b/tests/data/importer/webproject/webproject/settings.py deleted file mode 100644 index 18f1e7257..000000000 --- a/tests/data/importer/webproject/webproject/settings.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -Django settings for webproject project. - -Generated by 'django-admin startproject' using Django 1.9.1. - -For more information on this file, see -https://docs.djangoproject.com/en/1.9/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.9/ref/settings/ -""" - -import os - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'db)-3@sbmu5d91tpio#h*9=iew@12-n1bh-de!xbrb0e4la(9n' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] -AUTH_PASSWORD_VALIDATORS = ['cats'] - - -# Application definition - -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', -] - -MIDDLEWARE_CLASSES = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', -] - -ROOT_URLCONF = 'webproject.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -WSGI_APPLICATION = 'webproject.wsgi.application' - - -# Database -# https://docs.djangoproject.com/en/1.9/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': '' - } -} - - -# Password validation -# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - - -# Internationalization -# https://docs.djangoproject.com/en/1.9/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.9/howto/static-files/ - -STATIC_URL = '/static/' diff --git a/tests/data/importer/webproject/webproject/urls.py b/tests/data/importer/webproject/webproject/urls.py deleted file mode 100644 index f4b8e560b..000000000 --- a/tests/data/importer/webproject/webproject/urls.py +++ /dev/null @@ -1,24 +0,0 @@ -"""webproject URL Configuration - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/1.9/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.conf.urls import url, include - 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) -""" -from django.conf.urls import url -from django.contrib import admin -import webapp.views - -urlpatterns = [ - url(r'^$', webapp.views.index), - url(r'^shutdown/$', webapp.views.shutdown), - url(r'^admin/', admin.site.urls), -] diff --git a/tests/data/importer/webproject/webproject/wsgi.py b/tests/data/importer/webproject/webproject/wsgi.py deleted file mode 100644 index 2f67662a0..000000000 --- a/tests/data/importer/webproject/webproject/wsgi.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -WSGI config for webproject project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/ -""" - -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "webproject.settings") - -application = get_wsgi_application() diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index 67e937ed0..23fda06e5 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -1,5 +1,4 @@ import inspect -import json import os import sys import unittest @@ -262,108 +261,3 @@ def test_rel2(self): def test_rel_overflow(self): self.assertEqual('', self.call('email.utils', 3)) - - -class FakeSshTest(testlib.TestCase): - klass = mitogen.master.ModuleFinder - - def call(self, fullname): - return self.klass().find_related_imports(fullname) - - def test_simple(self): - import mitogen.fakessh - related = self.call('mitogen.fakessh') - self.assertEqual(related, [ - 'mitogen', - 'mitogen.core', - 'mitogen.parent', - ]) - - -class FindRelatedTest(testlib.TestCase): - klass = mitogen.master.ModuleFinder - - def call(self, fullname): - return self.klass().find_related(fullname) - - SIMPLE_EXPECT = set([ - u'mitogen', - u'mitogen.core', - u'mitogen.parent', - ]) - - if sys.version_info < (2, 7): - SIMPLE_EXPECT.add('mitogen.compat') - SIMPLE_EXPECT.add('mitogen.compat.tokenize') - if sys.version_info < (2, 6): - SIMPLE_EXPECT.add('mitogen.compat') - SIMPLE_EXPECT.add('mitogen.compat.pkgutil') - - def test_simple(self): - import mitogen.fakessh - related = self.call('mitogen.fakessh') - self.assertEqual(set(related), self.SIMPLE_EXPECT) - - -class DjangoMixin(object): - WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject') - - @classmethod - def modules_expected_path(cls): - if sys.version_info[0:2] < (3, 0): - modules_expected_filename = 'modules_expected_py2x.json' - elif sys.version_info[0:2] <= (3, 6): - modules_expected_filename = 'modules_expected_py3x-legacy.json' - elif sys.version_info[0:2] >= (3, 10): - modules_expected_filename = 'modules_expected_py3x-new.json' - return os.path.join(cls.WEBPROJECT_PATH, modules_expected_filename) - - @classmethod - def setUpClass(cls): - super(DjangoMixin, cls).setUpClass() - sys.path.append(cls.WEBPROJECT_PATH) - os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' - with open(cls.modules_expected_path(), 'rb') as f: - cls.MODULES_EXPECTED = json.load(f) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.WEBPROJECT_PATH) - del os.environ['DJANGO_SETTINGS_MODULE'] - super(DjangoMixin, cls).tearDownClass() - - -class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): - maxDiff = None - - def test_django_db(self): - import django.db - module_finder = mitogen.master.ModuleFinder() - related = module_finder.find_related('django.db') - expected = self.MODULES_EXPECTED['find_related']['django.db'] - self.assertEqual(related, expected) - - def test_django_db_models(self): - import django.db.models - module_finder = mitogen.master.ModuleFinder() - related = module_finder.find_related('django.db.models') - expected = self.MODULES_EXPECTED['find_related']['django.db.models'] - self.assertEqual(related, expected) - - -class DjangoFindRelatedImportsTest(DjangoMixin, testlib.TestCase): - maxDiff = None - - def test_django_db(self): - import django.db - module_finder = mitogen.master.ModuleFinder() - related = module_finder.find_related_imports('django.db') - expected = self.MODULES_EXPECTED['find_related_imports']['django.db'] - self.assertEqual(related, expected) - - def test_django_db_models(self): - import django.db.models - module_finder = mitogen.master.ModuleFinder() - related = module_finder.find_related_imports('django.db.models') - expected = self.MODULES_EXPECTED['find_related_imports']['django.db.models'] - self.assertEqual(related, expected) diff --git a/tests/requirements.txt b/tests/requirements.txt index c5671b37e..3c5eaf79a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,9 +6,6 @@ coverage==6.2; python_version == '3.6' coverage==7.2.7; python_version == '3.7' coverage==7.4.3; python_version >= '3.8' -Django==1.11.29; python_version < '3.0' -Django==3.2.20; python_version >= '3.6' - mock==3.0.5; python_version == '2.7' mock==5.1.0; python_version >= '3.6' diff --git a/tests/responder_test.py b/tests/responder_test.py index d1e658167..9cb7a50ce 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -87,8 +87,12 @@ def test_simple_pkg(self): context = self.router.local() self.assertEqual(3, context.call(simple_pkg.a.subtract_one_add_two, 2)) + + # If we don't proactively send related modules, then we expect + # count(GET_MODULE) == count(LOAD_MODULE) == count(succesful modules). + # simple_pkg, simple_pkg.a, simple_pkg.b, & mitogen.os_fork as necessary os_fork = int(sys.version_info < (2, 6)) # mitogen.os_fork - self.assertEqual(2+os_fork, self.router.responder.get_module_count) + self.assertEqual(3+os_fork, self.router.responder.get_module_count) self.assertEqual(3+os_fork, self.router.responder.good_load_module_count) self.assertEqual(0, self.router.responder.bad_load_module_count) self.assertLess(450, self.router.responder.good_load_module_size)