diff --git a/src/util/misc.py b/src/util/misc.py index dd0063469..1fce48f10 100644 --- a/src/util/misc.py +++ b/src/util/misc.py @@ -4,9 +4,12 @@ import collections import datetime import functools +import hashlib +import inspect import logging import os import re +import sys import textwrap import uuid from contextlib import contextmanager @@ -21,7 +24,7 @@ from openerp.modules.module import get_module_path from openerp.tools.parse_version import parse_version -from .exceptions import SleepyDeveloperError +from .exceptions import MigrationError, SleepyDeveloperError # python3 shim try: @@ -391,6 +394,31 @@ def log(chunk_num, size=chunk_size): log(i // chunk_size + 1, i % chunk_size) +def make_pickleable_callback(callback): + """ + Make a callable importable. + + `ProcessPoolExecutor.map` arguments needs to be pickleable + Functions can only be pickled if they are importable. + However, the callback's file is not importable due to the dash in the filename. + We should then put the executed function in its own importable file. + + :meta private: exclude from online docs + """ + callback_filepath = inspect.getfile(callback) + name = "_upgrade_" + hashlib.sha256(callback_filepath.encode()).hexdigest() + if name not in sys.modules: + sys.modules[name] = import_script(callback_filepath, name=name) + try: + return getattr(sys.modules[name], callback.__name__) + except AttributeError: + error_msg = ( + "The converter callback `{}` is a nested function in `{}`.\n" + "Move it outside the `migrate()` function to make it top-level." + ).format(callback.__name__, callback.__module__) + raise MigrationError(error_msg) + + class SelfPrint(object): """ Class that will return a self representing string. Used to evaluate domains. diff --git a/src/util/snippets.py b/src/util/snippets.py index a4e091546..8d690e7b2 100644 --- a/src/util/snippets.py +++ b/src/util/snippets.py @@ -1,9 +1,6 @@ # -*- coding: utf-8 -*- -import inspect import logging import re -import sys -import uuid from concurrent.futures import ProcessPoolExecutor from lxml import etree, html @@ -12,9 +9,8 @@ from psycopg2.extras import Json from .const import NEARLYWARN -from .exceptions import MigrationError from .helpers import table_of_model -from .misc import import_script, log_progress +from .misc import log_progress, make_pickleable_callback from .pg import column_exists, column_type, get_max_workers, table_exists _logger = logging.getLogger(__name__) @@ -161,28 +157,6 @@ def html_converter(transform_callback, selector=None): return HTMLConverter(make_pickleable_callback(transform_callback), selector) -def make_pickleable_callback(callback): - """ - Make a callable importable. - - `ProcessPoolExecutor.map` arguments needs to be pickleable - Functions can only be pickled if they are importable. - However, the callback's file is not importable due to the dash in the filename. - We should then put the executed function in its own importable file. - """ - callback_filepath = inspect.getfile(callback) - name = f"_upgrade_{uuid.uuid4().hex}" - mod = sys.modules[name] = import_script(callback_filepath, name=name) - try: - return getattr(mod, callback.__name__) - except AttributeError: - error_msg = ( - f"The converter callback `{callback.__name__}` is a nested function in `{callback.__module__}`.\n" - "Move it outside the `migrate()` function to make it top-level." - ) - raise MigrationError(error_msg) from None - - class BaseConverter: def __init__(self, callback, selector=None): self.callback = callback