From 9c4f699674cf6228d176a7250e78c1e9e832057f Mon Sep 17 00:00:00 2001 From: Richard deMeester Date: Sat, 8 Oct 2022 17:51:17 +0500 Subject: [PATCH 1/8] DB filter mod updated to v16 --- odoo/http.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/odoo/http.py b/odoo/http.py index 754251a82ba4c..64ef3d5a20fdf 100644 --- a/odoo/http.py +++ b/odoo/http.py @@ -340,6 +340,26 @@ def db_filter(dbs, host=None): :rtype: List[str] """ + if config.get('db_filter_multi'): + host_m = host + + if host_m is None: + host_m = request.httprequest.environ.get('HTTP_HOST', '') + host_m = host_m.partition(':')[0] + if host_m.startswith('www.'): + host_m = host_m[4:] + domain = host_m.partition('.')[0] + + db_dict = json.loads(config["db_filter_multi"]) + if isinstance(db_dict, dict) and host_m in db_dict: + ndbs = [] + for dbfilter in db_dict[host_m]: + dbfilter_re = re.compile( + dbfilter.replace("%h", re.escape(host_m)) + .replace("%d", re.escape(domain))) + ndbs.extend([db for db in dbs if dbfilter_re.match(db)]) + return sorted(list(set(ndbs))) + if config['dbfilter']: # host # ----------- From ef9e50f40fcd7c30b24e52fbf597677329eba0b8 Mon Sep 17 00:00:00 2001 From: Richard deMeester Date: Sun, 9 Oct 2022 17:03:58 +0500 Subject: [PATCH 2/8] Whitelisting dbs for email and crons fix email whilelist related test failures when installing base --- addons/mail/models/mail_template.py | 6 ++++++ odoo/addons/base/models/ir_cron.py | 17 +++++++++++++++++ odoo/addons/base/models/ir_mail_server.py | 11 +++++++++++ 3 files changed, 34 insertions(+) diff --git a/addons/mail/models/mail_template.py b/addons/mail/models/mail_template.py index c7b9512f07f4f..6cabcf6290e33 100644 --- a/addons/mail/models/mail_template.py +++ b/addons/mail/models/mail_template.py @@ -11,6 +11,8 @@ from odoo.tools import is_html_empty from odoo.tools.safe_eval import safe_eval, time +from odoo.addons.base.models.ir_cron import db_whitelisted + _logger = logging.getLogger(__name__) @@ -596,6 +598,10 @@ def send_mail(self, res_id, force_send=False, raise_exception=False, email_value Attachment = self.env['ir.attachment'] # TDE FIXME: should remove default_type from context + if force_send and raise_exception and not db_whitelisted(self.env.cr.dbname): + # Allows auto functions, like create users, to continue without failing + raise_exception = False + # create a mail_mail based on values, without attachments values = self._generate_template( [res_id], diff --git a/odoo/addons/base/models/ir_cron.py b/odoo/addons/base/models/ir_cron.py index 9e9c2fabc0a6b..a8dab75c6d5f6 100644 --- a/odoo/addons/base/models/ir_cron.py +++ b/odoo/addons/base/models/ir_cron.py @@ -9,6 +9,9 @@ from dateutil.relativedelta import relativedelta from psycopg2 import sql +import json +import re + import odoo from odoo import api, fields, models, _ from odoo.exceptions import UserError @@ -22,6 +25,16 @@ ODOO_NOTIFY_FUNCTION = os.getenv('ODOO_NOTIFY_FUNCTION', 'pg_notify') +def db_whitelisted(db_name): + cron_whitelist = odoo.tools.config.get("db_cron_whitelist") and json.loads(odoo.tools.config["db_cron_whitelist"]) or [] + if db_name not in cron_whitelist: + for cw_name in cron_whitelist: + if re.match(cw_name, db_name): + break + else: + return False + return True + class BadVersion(Exception): pass @@ -112,6 +125,10 @@ def method_direct_trigger(self): @classmethod def _process_jobs(cls, db_name): + + if not db_whitelisted(db_name): + return False + """ Execute every job ready to be run on this database. """ try: db = odoo.sql_db.db_connect(db_name) diff --git a/odoo/addons/base/models/ir_mail_server.py b/odoo/addons/base/models/ir_mail_server.py index 7f49ad7517e9d..edc0ebaf21334 100644 --- a/odoo/addons/base/models/ir_mail_server.py +++ b/odoo/addons/base/models/ir_mail_server.py @@ -20,11 +20,14 @@ from OpenSSL.crypto import Error as SSLCryptoError, FILETYPE_PEM from OpenSSL.SSL import Error as SSLError from urllib3.contrib.pyopenssl import PyOpenSSLContext +from OpenSSL.SSL import Context as SSLContext, Error as SSLError from odoo import api, fields, models, tools, _ from odoo.exceptions import UserError from odoo.tools import ustr, pycompat, formataddr, email_normalize, encapsulate_email, email_domain_extract, email_domain_normalize +from odoo.addons.base.models.ir_cron import db_whitelisted + _logger = logging.getLogger(__name__) _test_logger = logging.getLogger('odoo.tests') @@ -696,6 +699,14 @@ def send_email(self, message, mail_server_id=None, smtp_server=None, smtp_port=N _test_logger.info("skip sending email in test mode") return message['Message-Id'] + # Some odoo tests in base explicitly patch ir.mail_server to return False from _is_test_mode() + if ( + not db_whitelisted(self.env.cr.dbname) + and not isinstance(self.connect, MagicMock) + and smtp.__class__.__name__ != "FakeSMTP" + ): + raise UserError(_("Whitelist Error") + "\n" + _("Database cannot send emails as it is not on the whitelist.")) + try: message_id = message['Message-Id'] From 707e99e3818781b6f8e2df6be72ce4076fccce36 Mon Sep 17 00:00:00 2001 From: James Bos Date: Wed, 11 Jan 2023 21:41:53 +1100 Subject: [PATCH 3/8] return out of send() If db not whitelisted --- addons/mail/models/mail_mail.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/addons/mail/models/mail_mail.py b/addons/mail/models/mail_mail.py index 5225587dfa395..d998d9757fdb7 100644 --- a/addons/mail/models/mail_mail.py +++ b/addons/mail/models/mail_mail.py @@ -17,6 +17,7 @@ from odoo import _, api, fields, models from odoo import tools from odoo.addons.base.models.ir_mail_server import MailDeliveryException +from odoo.addons.base.models.ir_cron import db_whitelisted _logger = logging.getLogger(__name__) _UNFOLLOW_REGEX = re.compile(r'', re.DOTALL) @@ -551,6 +552,12 @@ def send(self, auto_commit=False, raise_exception=False): email sending process has failed :return: True """ + + # Return out of send() if db not in whitelist + if not db_whitelisted(self.env.cr.dbname): + _logger.warning('Database cannot send emails as it is not on the whitelist.') + return + for mail_server_id, alias_domain_id, smtp_from, batch_ids in self._split_by_mail_configuration(): smtp_session = None try: From e28a7b6bb3c290997f2dbb90cbdb9a6e0ff27c16 Mon Sep 17 00:00:00 2001 From: Richard deMeester Date: Fri, 13 Jan 2023 09:02:52 +1100 Subject: [PATCH 4/8] Add support for pre_install tests --- odoo/modules/graph.py | 41 +++++++++++++++++++++++++++++++++++++++++ odoo/modules/loading.py | 35 +++++++++++++++++++++++++++++++++++ odoo/service/server.py | 9 +++++++++ odoo/tests/loader.py | 13 +++++++++++++ 4 files changed, 98 insertions(+) diff --git a/odoo/modules/graph.py b/odoo/modules/graph.py index 752237b4b0a93..8dd497b6374c0 100644 --- a/odoo/modules/graph.py +++ b/odoo/modules/graph.py @@ -21,6 +21,11 @@ def _ignored_modules(cr): result += [m[0] for m in cr.fetchall()] return result + +def check_package_delayed(info): + return int(info.get('load_priority', 0)) > 0 + + class Graph(dict): """ Modules dependency graph. @@ -30,8 +35,17 @@ class Graph(dict): def add_node(self, name, info): max_depth, father = 0, None + + package_delayed = check_package_delayed(info) + for d in info['depends']: n = self.get(d) or Node(d, self, None) # lazy creation, do not use default value for get() + + if package_delayed: + deepest_nodes = n.deepest_nodes() + if deepest_nodes: + n = deepest_nodes[-1] + if n.depth >= max_depth: father = n max_depth = n.depth @@ -77,12 +91,30 @@ def add_modules(self, cr, module_list, force=None): dependencies = dict([(p, info['depends']) for p, info in packages]) current, later = set([p for p, info in packages]), set() + delayed_packages = [] + do_delayed_package = False + while packages and current > later: package, info = packages[0] deps = info['depends'] + # Have we looped through all without adding anything? + if delayed_packages and package == delayed_packages[0][0]: + delayed_packages.sort(key=lambda p: int(p[1]['load_priority'])) + do_delayed_package = delayed_packages[0][0] + delayed_packages = [] + # if all dependencies of 'package' are already in the graph, add 'package' in the graph if all(dep in self for dep in deps): + if check_package_delayed(info) and package != do_delayed_package: + delayed_packages.append((package, info)) + packages.append((package, info)) + packages.pop(0) + continue + + delayed_packages = [] + do_delayed_package = False + if not package in current: packages.pop(0) continue @@ -159,6 +191,15 @@ def add_child(self, name, info): self.children.sort(key=lambda x: x.name) return node + def deepest_nodes(self): + next_level = [self] + while next_level: + last_level = next_level + next_level = [] + for node in last_level: + next_level.extend(node.children) + return last_level + def __setattr__(self, name, value): super(Node, self).__setattr__(name, value) if name in ('init', 'update', 'demo'): diff --git a/odoo/modules/loading.py b/odoo/modules/loading.py index ad724f1d6e480..00405d8393bcc 100644 --- a/odoo/modules/loading.py +++ b/odoo/modules/loading.py @@ -182,6 +182,29 @@ def load_module_graph(env, graph, status=None, perform_checks=True, if package.name != 'base': env.flush_all() + # Before Loading, check if any other modules have a "pre-install" test to be run + loader = odoo.tests.loader + updating = tools.config.options['init'] or tools.config.options['update'] + test_results = None + if tools.config.options['test_enable'] and (needs_update or not updating): + module_names = (sorted(registry._init_modules)) + for test_module_name in module_names: + preinstalls = loader.find_pre_install_tests(test_module_name) + if module_name in preinstalls: + _logger.info("Starting pre install tests") + tests_before = registry._assertion_report.testsRun + tests_t0, tests_q0 = time.time(), odoo.sql_db.sql_counter + with odoo.api.Environment.manage(): + result = loader.run_suite(loader.make_suite(test_module_name, 'pre_install_%s' % module_name), test_module_name) + registry._assertion_report.update(result) + _logger.info( + "%d pre-install-tests in %.2fs, %s queries", + registry._assertion_report.testsRun - tests_before, + time.time() - tests_t0, + odoo.sql_db.sql_counter - tests_q0) + + # End of pre-install tests + load_openerp_module(package.name) if new_install: @@ -288,6 +311,18 @@ def load_module_graph(env, graph, status=None, perform_checks=True, # tests may have reset the environment module = env['ir.module.module'].browse(module_id) + # If this module has pre-install tests, but that module is already installed, + # then we have either a circular reference, or need a load_priority in the + # manifest + + preinstalls = loader.find_pre_install_tests(module_name) + loaded_modules = (sorted(registry._init_modules)) + if any(n in loaded_modules for n in preinstalls): + _logger.error( + "Module %s: Preinstall test not run as module already installed", + module_name + ) + if needs_update: processed_modules.append(package.name) diff --git a/odoo/service/server.py b/odoo/service/server.py index e1c9b3a3c1862..6e46326fc9af4 100644 --- a/odoo/service/server.py +++ b/odoo/service/server.py @@ -1330,6 +1330,15 @@ def preload_registries(dbnames): module_names = (registry.updated_modules if update_module else sorted(registry._init_modules)) _logger.info("Starting post tests") + + # Run pre_install tests which were missed because the module was never installed + for module_name in module_names: + preinstalls = loader.find_pre_install_tests(module_name) + for preinstall in preinstalls: + if preinstall not in module_names: + result = loader.run_suite(loader.make_suite(module_name, 'pre_install_%s' % preinstall), module_name) + registry._assertion_report.update(result) + tests_before = registry._assertion_report.testsRun post_install_suite = loader.make_suite(module_names, 'post_install') if post_install_suite.has_http_case(): diff --git a/odoo/tests/loader.py b/odoo/tests/loader.py index 0fb462c030a28..ce2e4b9b7da2b 100644 --- a/odoo/tests/loader.py +++ b/odoo/tests/loader.py @@ -76,6 +76,19 @@ def make_suite(module_names, position='at_install'): return OdooSuite(sorted(tests, key=lambda t: t.test_sequence)) +def find_pre_install_tests(module_name): + mods = get_test_modules(module_name) + pre_installs = set() + for mod in mods: + for test in unwrap_suite(unittest.TestLoader().loadTestsFromModule(mod)): + for tag in test.test_tags if hasattr(test, 'test_tags') else []: + if tag.startswith('+'): + tag = tag.replace('+', '') + if tag.startswith('pre_install_'): + pre_installs.add(tag.replace('pre_install_', '')) + return pre_installs + + def run_suite(suite, module_name=None): # avoid dependency hell from ..modules import module From b40e9ff0451235e00bb990b07573afb99da25eb5 Mon Sep 17 00:00:00 2001 From: David James Date: Wed, 11 Oct 2023 14:37:45 +1100 Subject: [PATCH 5/8] allow setting a mail server for local development --- addons/mail/models/mail_mail.py | 4 +- odoo/addons/base/models/ir_mail_server.py | 59 ++++++++++++++++++++--- 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/addons/mail/models/mail_mail.py b/addons/mail/models/mail_mail.py index d998d9757fdb7..1b5a0cc6d978b 100644 --- a/addons/mail/models/mail_mail.py +++ b/addons/mail/models/mail_mail.py @@ -16,7 +16,7 @@ from odoo import _, api, fields, models from odoo import tools -from odoo.addons.base.models.ir_mail_server import MailDeliveryException +from odoo.addons.base.models.ir_mail_server import MailDeliveryException, MailDeliveryWhitelistException from odoo.addons.base.models.ir_cron import db_whitelisted _logger = logging.getLogger(__name__) @@ -562,6 +562,8 @@ def send(self, auto_commit=False, raise_exception=False): smtp_session = None try: smtp_session = self.env['ir.mail_server'].connect(mail_server_id=mail_server_id, smtp_from=smtp_from) + except MailDeliveryWhitelistException: + pass except Exception as exc: if raise_exception: # To be consistent and backward compatible with mail_mail.send() raised diff --git a/odoo/addons/base/models/ir_mail_server.py b/odoo/addons/base/models/ir_mail_server.py index edc0ebaf21334..67e4d6b0e1822 100644 --- a/odoo/addons/base/models/ir_mail_server.py +++ b/odoo/addons/base/models/ir_mail_server.py @@ -14,6 +14,7 @@ import ssl import sys import threading +import os from socket import gaierror, timeout from OpenSSL import crypto as SSLCrypto @@ -39,6 +40,10 @@ class MailDeliveryException(Exception): """Specific exception subclass for mail delivery errors""" +class MailDeliveryWhitelistException(MailDeliveryException): + """Specific exception subclass for non whitelisted mail delivery attempts""" + + def make_wrap_property(name): return property( lambda self: getattr(self.__obj__, name), @@ -416,6 +421,8 @@ def connect(self, host=None, port=None, user=None, password=None, encryption=Non _("Please define at least one SMTP server, " "or provide the SMTP parameters explicitly."))) + self._is_allowed_to_send(smtp_server, raise_exception=True) + if smtp_encryption == 'ssl': if 'SMTP_SSL' not in smtplib.__all__: raise UserError( @@ -699,14 +706,6 @@ def send_email(self, message, mail_server_id=None, smtp_server=None, smtp_port=N _test_logger.info("skip sending email in test mode") return message['Message-Id'] - # Some odoo tests in base explicitly patch ir.mail_server to return False from _is_test_mode() - if ( - not db_whitelisted(self.env.cr.dbname) - and not isinstance(self.connect, MagicMock) - and smtp.__class__.__name__ != "FakeSMTP" - ): - raise UserError(_("Whitelist Error") + "\n" + _("Database cannot send emails as it is not on the whitelist.")) - try: message_id = message['Message-Id'] @@ -730,6 +729,8 @@ def send_email(self, message, mail_server_id=None, smtp_server=None, smtp_port=N smtp.quit() except smtplib.SMTPServerDisconnected: raise + except MailDeliveryWhitelistException: + raise except Exception as e: params = (ustr(smtp_server), e.__class__.__name__, ustr(e)) msg = _("Mail delivery failed via SMTP server '%s'.\n%s: %s", *params) @@ -848,3 +849,45 @@ def _is_test_mode(self): outgoing mail server. """ return getattr(threading.current_thread(), 'testing', False) or self.env.registry.in_test_mode() + + def _is_allowed_to_send(self, smtp_server: str = None, raise_exception: bool = False) -> bool: + """ + Return True if the database is allowed to send email. + + Emails are not allowed unless the database is whitelisted by using the + `db_cron_whitelist` option in the odoo config file. + + During development, we don't want to enable this option, and we do not want + to send emails to real users, but we do want to test emails using local + development mail servers such as MailHog. + + To avoid accidentally allowing real local email servers such as postfix + to send emails, the mail server must be explicitly configured with the + environment variable `ODOO_DEV_SMTP_SERVER` if the database is not + whitelisted. + """ + + if db_whitelisted(self.env.cr.dbname): + return True + + # Some odoo tests in base explicitly patch ir.mail_server to return False from + # _is_test_mode() in which case we want to allow sending mails. + if isinstance(self.connect, MagicMock): + return True + + dev_smtp_server = os.environ.get("ODOO_DEV_SMTP_SERVER") + if dev_smtp_server in ["localhost", "127.0.0.1"] and ( + (smtp_server and smtp_server == dev_smtp_server) + or + (len(self) <= 1 and self.smtp_host == dev_smtp_server) + ): + _logger.info("Allowing local development SMTP server: %s", smtp_server) + return True + + msg = _("Database cannot send emails as it is not on the whitelist.") + _logger.warning(msg) + + if raise_exception: + raise MailDeliveryWhitelistException(msg) + + return False From 5ad10f4b4c0656da0acc9f66449af4d752535b9a Mon Sep 17 00:00:00 2001 From: James Bos Date: Thu, 28 Mar 2024 10:52:48 +1100 Subject: [PATCH 6/8] Ensure unmet depends are logged as warnings --- odoo/modules/graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odoo/modules/graph.py b/odoo/modules/graph.py index 8dd497b6374c0..ef86e090343bd 100644 --- a/odoo/modules/graph.py +++ b/odoo/modules/graph.py @@ -133,7 +133,7 @@ def add_modules(self, cr, module_list, force=None): for package in later: unmet_deps = [p for p in dependencies[package] if p not in self] - _logger.info('module %s: Unmet dependencies: %s', package, ', '.join(unmet_deps)) + _logger.warning('module %s: Unmet dependencies: %s', package, ', '.join(unmet_deps)) return len(self) - len_graph From 02efb39513957ce6789cbfe122d94af84c21b259 Mon Sep 17 00:00:00 2001 From: James Bos Date: Fri, 26 Jul 2024 10:51:12 +1000 Subject: [PATCH 7/8] Missing unit test MagicMock import --- odoo/addons/base/models/ir_mail_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/odoo/addons/base/models/ir_mail_server.py b/odoo/addons/base/models/ir_mail_server.py index 67e4d6b0e1822..823c89fd643f5 100644 --- a/odoo/addons/base/models/ir_mail_server.py +++ b/odoo/addons/base/models/ir_mail_server.py @@ -22,6 +22,7 @@ from OpenSSL.SSL import Error as SSLError from urllib3.contrib.pyopenssl import PyOpenSSLContext from OpenSSL.SSL import Context as SSLContext, Error as SSLError +from unittest.mock import MagicMock from odoo import api, fields, models, tools, _ from odoo.exceptions import UserError From 852316f897965a00a8f780a011e9e77bfaeb80b1 Mon Sep 17 00:00:00 2001 From: James Bos Date: Mon, 5 Aug 2024 11:04:16 +1000 Subject: [PATCH 8/8] Revert to python3-lxml for compat --- debian/control | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 61d3f0913abce..93ee4caeb7e5d 100644 --- a/debian/control +++ b/debian/control @@ -35,7 +35,8 @@ Depends: python3-jinja2, python3-libsass, # After lxml 5.2, lxml-html-clean is in a separate package - python3-lxml-html-clean | python3-lxml, +# WilldooIT Patch: reverting to original package for compatibility + python3-lxml, python3-num2words, python3-ofxparse, python3-passlib,