diff --git a/src/util/convert_bootstrap.py b/src/util/convert_bootstrap.py index b86d001ba..424ef7fb1 100644 --- a/src/util/convert_bootstrap.py +++ b/src/util/convert_bootstrap.py @@ -1,1143 +1,10 @@ -"""Convert an XML/HTML document Bootstrap code from an older version to a newer one.""" +import warnings -import logging -import os.path -import re -from functools import lru_cache -from typing import Iterable +from .views.bootstrap import * # noqa: F403 -from lxml import etree - -try: - from packaging.version import Version -except ImportError: - from distutils.version import StrictVersion as Version # N.B. deprecated, will be removed in py3.12 - - -_logger = logging.getLogger(__name__) - - -# Regex boundary patterns for class names within attributes (`\b` is not good enough -# because it matches `-` etc.). It's probably enough to detect if the class name -# is surrounded by either whitespace or beginning/end of string. -# Includes boundary chars that can appear in t-att(f)-* attributes. -BS = r"(?:^|(?<=[\s}'\"]))" -BE = r"(?:$|(?=[\s{#'\"]))" -B = rf"(?:{BS}|{BE})" - - -def _xpath_has_class(context, *cls): - """Extension function for xpath to check if the context node has all the classes passed as arguments.""" - node_classes = set(context.context_node.attrib.get("class", "").split()) - return node_classes.issuperset(cls) - - -@lru_cache(maxsize=128) # does >99% hits over ~10mln calls -def _xpath_has_t_class_inner(attrs_values, classes): - """ - Inner implementation of :func:`_xpath_has_t_class`, suitable for caching. - - :param tuple[str] attrs_values: the values of the ``t-att-class`` and ``t-attf-class`` attributes - :param tuple[str] classes: the classes to check - """ - return any( - all(re.search(rf"{BS}{escaped_cls}{BE}", attr_value or "") for escaped_cls in map(re.escape, classes)) - for attr_value in attrs_values - ) - - -def _xpath_has_t_class(context, *cls): - """Extension function for xpath to check if the context node has all the classes passed as arguments in one of ``class`` or ``t-att(f)-class`` attributes.""" - return _xpath_has_class(context, *cls) or _xpath_has_t_class_inner( - tuple(map(context.context_node.attrib.get, ("t-att-class", "t-attf-class"))), cls - ) - - -@lru_cache(maxsize=1024) # does >88% hits over ~1.5mln calls -def _xpath_regex_inner(pattern, item): - """Inner implementation of :func:`_xpath_regex`, suitable for caching.""" - return bool(re.search(pattern, item)) - - -def _xpath_regex(context, item, pattern): - """Extension function for xpath to check if the passed item (attribute or text) matches the passed regex pattern.""" - if not item: - return False - if isinstance(item, list): - item = item[0] # only first attribute is valid - return _xpath_regex_inner(pattern, item) - - -xpath_utils = etree.FunctionNamespace(None) -xpath_utils["hasclass"] = _xpath_has_class -xpath_utils["has-class"] = _xpath_has_class -xpath_utils["has-t-class"] = _xpath_has_t_class -xpath_utils["regex"] = _xpath_regex - -html_utf8_parser = etree.HTMLParser(encoding="utf-8") - - -def innerxml(element, is_html=False): - """ - Return the inner XML of an element as a string. - - :param etree.ElementBase element: the element to convert. - :param bool is_html: whether to use HTML for serialization, XML otherwise. Defaults to False. - :rtype: str - """ - return (element.text or "") + "".join( - etree.tostring(child, encoding=str, method="html" if is_html else None) for child in element - ) - - -def split_classes(*joined_classes): - """Return a list of classes given one or more strings of joined classes separated by spaces.""" - return [c for classes in joined_classes for c in (classes or "").split(" ") if c] - - -def get_classes(element): - """Return the list of classes from the ``class`` attribute of an element.""" - return split_classes(element.get("class", "")) - - -def join_classes(classes): - """Return a string of classes joined by space given a list of classes.""" - return " ".join(classes) - - -def set_classes(element, classes): - """ - Set the ``class`` attribute of an element from a list of classes. - - If the list is empty, the attribute is removed. - """ - if classes: - element.attrib["class"] = join_classes(classes) - else: - element.attrib.pop("class", None) - - -def edit_classlist(classes, add, remove): - """ - Edit a class list, adding and removing classes. - - :param str | typing.List[str] classes: the original classes list or str to edit. - :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the list. - :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) - from the list. The `ALL` sentinel value can be specified to remove all classes. - :rtype: typing.List[str] - :return: the new class list. - """ - if remove is ALL: - classes = [] - remove = None - else: - if isinstance(classes, str): - classes = [classes] - classes = split_classes(*classes) - - remove_index = None - if isinstance(remove, str): - remove = [remove] - for classname in remove or []: - while classname in classes: - remove_index = max(remove_index or 0, classes.index(classname)) - classes.remove(classname) - - insert_index = remove_index if remove_index is not None else len(classes) - if isinstance(add, str): - add = [add] - for classname in add or []: - if classname not in classes: - classes.insert(insert_index, classname) - insert_index += 1 - - return classes - - -def edit_element_t_classes(element, add, remove): - """ - Edit inplace qweb ``t-att-class`` and ``t-attf-class`` attributes of an element, adding and removing the specified classes. - - N.B. adding new classes will not work if neither ``t-att-class`` nor ``t-attf-class`` are present. - - :param etree.ElementBase element: the element to edit. - :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the element. - :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) - from the element. The `ALL` sentinel value can be specified to remove all classes. - """ - if isinstance(add, str): - add = [add] - if add: - add = split_classes(*add) - if isinstance(remove, str): - remove = [remove] - if remove and remove is not ALL: - remove = split_classes(*remove) - - if not add and not remove: - return # nothing to do - - for attr in ("t-att-class", "t-attf-class"): - if attr in element.attrib: - value = element.attrib.pop(attr) - - # if we need to remove all classes, just remove or replace the attribute - # with the literal string of classes to add, removing all logic - if remove is ALL: - if not add: - value = None - else: - value = " ".join(add or []) - if attr.startswith("t-att-"): - value = f"'{value}'" - - else: - joined_adds = join_classes(add or []) - # if there's no classes to remove, try to append the new classes in a sane way - if not remove: - if add: - if attr.startswith("t-att-"): - value = f"{value} + ' {joined_adds}'" if value else f"'{joined_adds}'" - else: - value = f"{value} {joined_adds}" - # otherwise use regexes to replace the classes in the attribute - # if there's just one replace, do a string replacement with the new classes - elif len(remove) == 1: - value = re.sub(rf"{BS}{re.escape(remove[0])}{BE}", joined_adds, value) - # else there's more than one class to remove and split at matches, - # then rejoin with new ones at the position of the last removed one. - else: - value = re.split(rf"{BS}(?:{'|'.join(map(re.escape, remove))}){BE}", value) - value = "".join(value[:-1] + [joined_adds] + value[-1:]) - - if value is not None: - element.attrib[attr] = value - - -def edit_element_classes(element, add, remove, is_qweb=False): - """ - Edit inplace the "class" attribute of an element, adding and removing classes. - - :param etree.ElementBase element: the element to edit. - :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the element. - :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) - from the element. The `ALL` sentinel value can be specified to remove all classes. - :param bool is_qweb: if True also edit classes in ``t-att-class`` and ``t-attf-class`` attributes. - Defaults to False. - """ - if not is_qweb or not set(element.attrib) & {"t-att-class", "t-attf-class"}: - set_classes(element, edit_classlist(get_classes(element), add, remove)) - if is_qweb: - edit_element_t_classes(element, add, remove) - - -ALL = object() -"""Sentinel object to indicate "all items" in a collection""" - - -def simple_css_selector_to_xpath(selector, prefix="//"): - """ - Convert a basic CSS selector cases to an XPath expression. - - Supports node names, classes, ``>`` and ``,`` combinators. - - :param str selector: the CSS selector to convert. - :param str prefix: the prefix to add to the XPath expression. Defaults to ``//``. - :return: the resulting XPath expression. - :rtype: str - """ - separator = prefix - xpath_parts = [] - combinators = "+>,~ " - for selector_part in map(str.strip, re.split(rf"(\s*[{combinators}]\s*)", selector)): - if not selector_part: - separator = "//" - elif selector_part == ">": - separator = "/" - elif selector_part == ",": - separator = "|" + prefix - elif re.search(r"^(?:[a-z](-?\w+)*|[*.])", selector_part, flags=re.I): - element, *classes = selector_part.split(".") - if not element: - element = "*" - class_predicates = [f"[hasclass('{classname}')]" for classname in classes if classname] - xpath_parts += [separator, element + "".join(class_predicates)] - else: - raise NotImplementedError(f"Unsupported CSS selector syntax: {selector}") - - return "".join(xpath_parts) - - -CSS = simple_css_selector_to_xpath - - -def regex_xpath(pattern, attr=None, xpath=None): - """ - Return an XPath expression that matches elements with an attribute value matching the given regex pattern. - - :param str pattern: a regex pattern to match the attribute value against. - :param str | None attr: the attribute to match the pattern against. - If not given, the pattern is matched against the element's text. - :param str | None xpath: an optional XPath expression to further filter the elements to match. - :rtype: str - """ - # TODO abt: investigate lxml xpath variables interpolation (xpath.setcontext? registerVariables?) - if "'" in pattern and '"' in pattern: - quoted_pattern = "concat('" + "', \"'\", '".join(pattern.split("'")) + "')" - elif "'" in pattern: - quoted_pattern = '"' + pattern + '"' - else: - quoted_pattern = "'" + pattern + "'" - xpath_pre = xpath or "//*" - attr_or_text = f"@{attr}" if attr is not None else "text()" - return xpath_pre + f"[regex({attr_or_text}, {quoted_pattern})]" - - -def adapt_xpath_for_qweb(xpath): - """Adapts a xpath to enable matching on qweb ``t-att(f)-`` attributes.""" - xpath = re.sub(r"\bhas-?class(?=\()", "has-t-class", xpath) - # supposing that there's only one of `class`, `t-att-class`, `t-attf-class`, - # joining all of them with a space and removing trailing whitespace should behave - # similarly to COALESCE, and result in ORing the values for matching - xpath = re.sub( - r"(?<=\()\s*@(? - ... - - - after:: - .. code-block:: html - - ... - - """ - - def __call__(self, element, converter): - parent = element.getparent() - if parent is None: - raise ValueError(f"Cannot pull up contents of xml element with no parent: {element}") - - prev_sibling = element.getprevious() - if prev_sibling is not None: - prev_sibling.tail = ((prev_sibling.tail or "") + (element.text or "")) or None - else: - parent.text = ((parent.text or "") + (element.text or "")) or None - - for child in element: - element.addprevious(child) - - parent.remove(element) - - -class RenameAttribute(ElementOperation): - """Rename an attribute. Silently ignores elements that do not have the attribute.""" - - def __init__(self, old_name, new_name, extra_xpath=""): - self.old_name = old_name - self.new_name = new_name - self.extra_xpath = extra_xpath - - def __call__(self, element, converter): - rename_map = {self.old_name: self.new_name} - if converter.is_qweb: - rename_map = { - f"{prefix}{old}": f"{prefix}{new}" - for old, new in rename_map.items() - for prefix in ("",) + (("t-att-", "t-attf-") if not old.startswith("t-") else ()) - if f"{prefix}{old}" in element.attrib - } - if rename_map: - # to preserve attributes order, iterate+rename on a copy and reassign (clear+update, bc readonly property) - attrib_before = dict(element.attrib) - element.attrib.clear() - element.attrib.update({rename_map.get(k, k): v for k, v in attrib_before.items()}) - return element - - def xpath(self, xpath=None): - """ - Return an XPath expression that matches elements with the old attribute name. - - :param str | None xpath: an optional XPath expression to further filter the elements to match. - :rtype: str - """ - return (xpath or "//*") + f"[@{self.old_name}]{self.extra_xpath}" - - -class RegexReplace(ElementOperation): - """ - Uses `re.sub` to modify an attribute or the text of an element. - - N.B. no checks are made to ensure the attribute to replace is actually present on the elements. - - :param str pattern: the regex pattern to match. - :param str repl: the replacement string. - :param str | None attr: the attribute to replace. If not specified, the text of the element is replaced. - """ - - def __init__(self, pattern, sub, attr=None): - self.pattern = pattern - self.repl = sub - self.attr = attr - - def __call__(self, element, converter): - if self.attr is None: - # TODO abt: what about tail? - element.text = re.sub(self.pattern, self.repl, element.text or "") - else: - for attr in (self.attr,) + ((f"t-att-{self.attr}", f"t-attf-{self.attr}") if converter.is_qweb else ()): - if attr in element.attrib: - element.attrib[attr] = re.sub(self.pattern, self.repl, element.attrib[attr]) - return element - - def xpath(self, xpath=None): - """ - Return an XPath expression that matches elements with the old attribute name. - - :param str | None xpath: an optional XPath expression to further filter the elements to match. - :rtype: str - """ - return regex_xpath(self.pattern, self.attr, xpath) - - -class RegexReplaceClass(RegexReplace): - """ - Uses `re.sub` to modify the class. - - Basically, same as `RegexReplace`, but with `attr="class"`. - """ - - def __init__(self, pattern, sub, attr="class"): - super().__init__(pattern, sub, attr) - - -class BS3to4ConvertBlockquote(ElementOperation): - """Convert a BS3 ``
`` element to a BS4 ``
`` element with the ``blockquote`` class.""" - - def __call__(self, element, converter): - blockquote = converter.copy_element(element, tag="div", add_classes="blockquote", copy_attrs=False) - element.addnext(blockquote) - element.getparent().remove(element) - return blockquote - - -# TODO abt: merge MakeCard and ConvertCard into one operation class -class BS3to4MakeCard(ElementOperation): - """ - Pre-processe a BS3 panel, thumbnail, or well element to be converted to a BS4 card. - - Card components conversion is then handled by the ``ConvertCard`` operation class. - """ - - def __call__(self, element, converter): - card = converter.element_factory("
") - card_body = converter.copy_element( - element, tag="div", add_classes="card-body", remove_classes=ALL, copy_attrs=False - ) - card.append(card_body) - element.addnext(card) - element.getparent().remove(element) - return card - - -# TODO abt: refactor code -class BS3to4ConvertCard(ElementOperation): - """Fully convert a BS3 panel, thumbnail, or well element and their contents to a BS4 card.""" - - POST_CONVERSIONS = { - "title": ["card-title"], - "description": ["card-description"], - "category": ["card-category"], - "panel-danger": ["card", "bg-danger", "text-white"], - "panel-warning": ["card", "bg-warning"], - "panel-info": ["card", "bg-info", "text-white"], - "panel-success": ["card", "bg-success", "text-white"], - "panel-primary": ["card", "bg-primary", "text-white"], - "panel-footer": ["card-footer"], - "panel-body": ["card-body"], - "panel-title": ["card-title"], - "panel-heading": ["card-header"], - "panel-default": [], - "panel": ["card"], - } - - def _convert_child(self, child, old_card, new_card, converter): - old_card_classes = get_classes(old_card) - - classes = get_classes(child) - - if "header" in classes or ("image" in classes and len(child)): - add_classes = "card-header" - remove_classes = ["header", "image"] - elif "content" in classes: - add_classes = "card-img-overlay" if "card-background" in old_card_classes else "card-body" - remove_classes = "content" - elif {"card-footer", "footer", "text-center"} & set(classes): - add_classes = "card-footer" - remove_classes = "footer" - else: - new_card.append(child) - return - - new_child = converter.copy_element( - child, "div", add_classes=add_classes, remove_classes=remove_classes, copy_attrs=True - ) - - if "image" in classes: - [img_el] = new_child.xpath("./img")[:1] or [None] - if img_el is not None and "src" in img_el: - new_child.attrib["style"] = ( - f'background-image: url("{img_el.attrib["src"]}"); ' - "background-position: center center; " - "background-size: cover;" - ) - new_child.remove(img_el) - - new_card.append(new_child) - - if "content" in classes: # TODO abt: consider skipping for .card-background - [footer] = new_child.xpath(converter.adapt_xpath("./*[hasclass('footer')]"))[:1] or [None] - if footer is not None: - self._convert_child(footer, old_card, new_card, converter) - new_child.remove(footer) - - def _postprocess(self, new_card, converter): - for old_class, new_classes in self.POST_CONVERSIONS.items(): - for element in new_card.xpath(converter.adapt_xpath(f"(.|.//*)[hasclass('{old_class}')]")): - edit_element_classes(element, add=new_classes, remove=old_class) - - def __call__(self, element, converter): - classes = get_classes(element) - new_card = converter.copy_element(element, tag="div", copy_attrs=True, copy_contents=False) - wrapper = new_card - if "card-horizontal" in classes: - wrapper = etree.SubElement(new_card, "div", {"class": "row"}) - - for child in element: - self._convert_child(child, element, wrapper, converter) - - self._postprocess(new_card, converter) - element.addnext(new_card) - element.getparent().remove(element) - return new_card - - -class BS4to5ConvertCloseButton(ElementOperation): - """ - Convert BS4 ``button.close`` elements to BS5 ``button.btn-close``. - - Also fixes the ``data-dismiss`` attribute to ``data-bs-dismiss``, and removes any inner contents. - """ - - def __call__(self, element, converter): - new_btn = converter.copy_element(element, remove_classes="close", add_classes="btn-close", copy_contents=False) - - if "data-dismiss" in element.attrib: - new_btn.attrib["data-bs-dismiss"] = element.attrib["data-dismiss"] - del new_btn.attrib["data-dismiss"] - - element.addnext(new_btn) - element.getparent().remove(element) - - return new_btn - - -class BS4to5ConvertCardDeck(ElementOperation): - """Convert BS4 ``.card-deck`` elements to grid components (``.row``, ``.col``, etc.).""" - - def __call__(self, element, converter): - cards = element.xpath(converter.adapt_xpath("./*[hasclass('card')]")) - - cols_class = f"row-cols-{len(cards)}" if len(cards) in range(1, 7) else "row-cols-auto" - edit_element_classes(element, add=["row", cols_class], remove="card-deck", is_qweb=converter.is_qweb) - - for card in cards: - new_col = converter.build_element("div", classes=["col"]) - card.addprevious(new_col) - new_col.append(card) - - return element - - -class BS4to5ConvertFormInline(ElementOperation): - """Convert BS4 ``.form-inline`` elements to grid components (``.row``, ``.col``, etc.).""" - - def __call__(self, element, converter): - edit_element_classes(element, add="row row-cols-lg-auto", remove="form-inline", is_qweb=converter.is_qweb) - - children_selector = converter.adapt_xpath( - CSS(".form-control,.form-group,.form-check,.input-group,.custom-select,button", prefix="./") - ) - indexed_children = sorted([(element.index(c), c) for c in element.xpath(children_selector)], key=lambda x: x[0]) - - nest_groups = [] - last_idx = -1 - for idx, child in indexed_children: - nest_start, nest_end = idx, idx - labels = [label for label in child.xpath("preceding-sibling::label") if element.index(label) > last_idx] - labels = [ - label - for label in labels - if "for" in label.attrib and child.xpath(f"descendant-or-self::*[@id='{label.attrib['for']}']") - ] or labels[-1:] - if labels: - first_label = labels[0] - assert last_idx < element.index(first_label) < idx, "label must be between last group and current" - nest_start = element.index(first_label) - - assert nest_start <= nest_end, f"expected start {nest_start} to be <= end {nest_end}" - nest_groups.append(element[nest_start : nest_end + 1]) - last_idx = nest_end - - for els in nest_groups: - wrapper = converter.build_element("div", classes=["col-12"]) - els[0].addprevious(wrapper) - for el in els: - wrapper.append(el) - assert el not in element, f"expected {el!r} to be removed from {element!r}" - - return element - - -class BootstrapConverter: - """ - Class for converting XML or HTML Bootstrap code across versions. - - :param etree.ElementTree tree: the parsed XML or HTML tree to convert. - :param bool is_html: whether the tree is an HTML document. - """ - - MIN_VERSION = "3.0" - """Minimum supported Bootstrap version.""" - - # Conversions definitions by destination version. - # It's a dictionary of version strings to a list of (xpath, operations_list) tuples. - # For operations that implement the `xpath()` method, the `op()` class method can be used - # to directly define the operation and return the tuple with the corresponding XPath expression - # and the operation list. - # The `convert()` method will then process the conversions list in order, and for each tuple - # match the elements in the tree using the XPath expression and apply the operations list to them. - CONVERSIONS = { - "4.0": [ - # inputs - (CSS(".form-group .control-label"), [ReplaceClasses("control-label", "form-control-label")]), - (CSS(".form-group .text-help"), [ReplaceClasses("text-help", "form-control-feedback")]), - (CSS(".control-group .help-block"), [ReplaceClasses("help-block", "form-text")]), - ReplaceClasses.op("form-group-sm", "form-control-sm"), - ReplaceClasses.op("form-group-lg", "form-control-lg"), - (CSS(".form-control .input-sm"), [ReplaceClasses("input-sm", "form-control-sm")]), - (CSS(".form-control .input-lg"), [ReplaceClasses("input-lg", "form-control-lg")]), - # hide - ReplaceClasses.op("hidden-xs", "d-none"), - ReplaceClasses.op("hidden-sm", "d-sm-none"), - ReplaceClasses.op("hidden-md", "d-md-none"), - ReplaceClasses.op("hidden-lg", "d-lg-none"), - ReplaceClasses.op("visible-xs", "d-block d-sm-none"), - ReplaceClasses.op("visible-sm", "d-block d-md-none"), - ReplaceClasses.op("visible-md", "d-block d-lg-none"), - ReplaceClasses.op("visible-lg", "d-block d-xl-none"), - # image - ReplaceClasses.op("img-rounded", "rounded"), - ReplaceClasses.op("img-circle", "rounded-circle"), - ReplaceClasses.op("img-responsive", ("d-block", "img-fluid")), - # buttons - ReplaceClasses.op("btn-default", "btn-secondary"), - ReplaceClasses.op("btn-xs", "btn-sm"), - (CSS(".btn-group.btn-group-xs"), [ReplaceClasses("btn-group-xs", "btn-group-sm")]), - (CSS(".dropdown .divider"), [ReplaceClasses("divider", "dropdown-divider")]), - ReplaceClasses.op("badge", "badge badge-pill"), - ReplaceClasses.op("label", "badge"), - RegexReplaceClass.op(rf"{BS}label-(default|primary|success|info|warning|danger){BE}", r"badge-\1"), - (CSS(".breadcrumb > li"), [ReplaceClasses("breadcrumb", "breadcrumb-item")]), - # li - (CSS(".list-inline > li"), [AddClasses("list-inline-item")]), - # pagination - (CSS(".pagination > li"), [AddClasses("page-item")]), - (CSS(".pagination > li > a"), [AddClasses("page-link")]), - # carousel - (CSS(".carousel .carousel-inner > .item"), [ReplaceClasses("item", "carousel-item")]), - # pull - ReplaceClasses.op("pull-right", "float-right"), - ReplaceClasses.op("pull-left", "float-left"), - ReplaceClasses.op("center-block", "mx-auto"), - # well - (CSS(".well"), [BS3to4MakeCard()]), - (CSS(".thumbnail"), [BS3to4MakeCard()]), - # blockquote - (CSS("blockquote"), [BS3to4ConvertBlockquote()]), - (CSS(".blockquote.blockquote-reverse"), [ReplaceClasses("blockquote-reverse", "text-right")]), - # dropdown - (CSS(".dropdown-menu > li > a"), [AddClasses("dropdown-item")]), - (CSS(".dropdown-menu > li"), [PullUp()]), - # in - ReplaceClasses.op("in", "show"), - # table - (CSS("tr.active, td.active"), [ReplaceClasses("active", "table-active")]), - (CSS("tr.success, td.success"), [ReplaceClasses("success", "table-success")]), - (CSS("tr.info, td.info"), [ReplaceClasses("info", "table-info")]), - (CSS("tr.warning, td.warning"), [ReplaceClasses("warning", "table-warning")]), - (CSS("tr.danger, td.danger"), [ReplaceClasses("danger", "table-danger")]), - (CSS("table.table-condesed"), [ReplaceClasses("table-condesed", "table-sm")]), - # navbar - (CSS(".nav.navbar > li > a"), [AddClasses("nav-link")]), - (CSS(".nav.navbar > li"), [AddClasses("nav-intem")]), - ReplaceClasses.op("navbar-btn", "nav-item"), - (CSS(".navbar-nav"), [ReplaceClasses("navbar-right nav", "ml-auto")]), - ReplaceClasses.op("navbar-toggler-right", "ml-auto"), - (CSS(".navbar-nav > li > a"), [AddClasses("nav-link")]), - (CSS(".navbar-nav > li"), [AddClasses("nav-item")]), - (CSS(".navbar-nav > a"), [AddClasses("navbar-brand")]), - ReplaceClasses.op("navbar-fixed-top", "fixed-top"), - ReplaceClasses.op("navbar-toggle", "navbar-toggler"), - ReplaceClasses.op("nav-stacked", "flex-column"), - (CSS("nav.navbar"), [AddClasses("navbar-expand-lg")]), - (CSS("button.navbar-toggle"), [ReplaceClasses("navbar-toggle", "navbar-expand-md")]), - # card - (CSS(".panel"), [BS3to4ConvertCard()]), - (CSS(".card"), [BS3to4ConvertCard()]), - # grid - RegexReplaceClass.op(rf"{BS}col((?:-\w{{2}})?)-offset-(\d{{1,2}}){BE}", r"offset\1-\2"), - ], - "5.0": [ - # links - RegexReplaceClass.op(rf"{BS}text-(?!o-)", "link-", xpath=CSS("a")), - (CSS(".nav-item.active > .nav-link"), [AddClasses("active")]), - (CSS(".nav-link.active") + CSS(".nav-item.active", prefix="/parent::"), [RemoveClasses("active")]), - # badges - ReplaceClasses.op("badge-pill", "rounded-pill"), - RegexReplaceClass.op(rf"{BS}badge-", r"text-bg-"), - # buttons - ("//*[hasclass('btn-block')]/parent::div", [AddClasses("d-grid gap-2")]), - ("//*[hasclass('btn-block')]/parent::p[count(./*)=1]", [AddClasses("d-grid gap-2")]), - RemoveClasses.op("btn-block"), - (CSS("button.close"), [BS4to5ConvertCloseButton()]), - # card - # TODO abt: .card-columns (unused in odoo) - (CSS(".card-deck"), [BS4to5ConvertCardDeck()]), - # jumbotron - ReplaceClasses.op("jumbotron", "container-fluid py-5"), - # new data-bs- attributes - RenameAttribute.op("data-display", "data-bs-display", "[not(@data-snippet='s_countdown')]"), - *[ - RenameAttribute.op(f"data-{attr}", f"data-bs-{attr}") - for attr in ( - "animation attributes autohide backdrop body container content delay dismiss focus" - " interval margin-right no-jquery offset original-title padding-right parent placement" - " ride sanitize show slide slide-to spy target toggle touch trigger whatever" - ).split(" ") - ], - # popover - (CSS(".popover .arrow"), [ReplaceClasses("arrow", "popover-arrow")]), - # form - ReplaceClasses.op("form-row", "row"), - ("//*[hasclass('form-group')]/parent::form", [AddClasses("row")]), - ReplaceClasses.op("form-group", "col-12 py-2"), - (CSS(".form-inline"), [BS4to5ConvertFormInline()]), - ReplaceClasses.op("custom-checkbox", "form-check"), - RegexReplaceClass.op(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"), - RegexReplaceClass.op(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"), - RegexReplaceClass.op(rf"{BS}custom-(check|select|range){BE}", r"form-\1"), - (CSS(".custom-switch"), [ReplaceClasses("custom-switch", "form-check form-switch")]), - ReplaceClasses.op("custom-radio", "form-check"), - RemoveClasses.op("custom-control"), - (CSS(".custom-file"), [PullUp()]), - RegexReplaceClass.op(rf"{BS}custom-file-", r"form-file-"), - RegexReplaceClass.op(rf"{BS}form-file(?:-input)?{BE}", r"form-control"), - (CSS("label.form-file-label"), [RemoveElement()]), - (regex_xpath(rf"{BS}input-group-(prepend|append){BE}", "class"), [PullUp()]), - ("//label[not(hasclass('form-check-label'))]", [AddClasses("form-label")]), - ReplaceClasses.op("form-control-file", "form-control"), - ReplaceClasses.op("form-control-range", "form-range"), - # TODO abt: .form-text no loger sets display, add some class? - # table - RegexReplaceClass.op(rf"{BS}thead-(light|dark){BE}", r"table-\1"), - # grid - RegexReplaceClass.op(rf"{BS}col-((?:\w{{2}}-)?)offset-(\d{{1,2}}){BE}", r"offset-\1\2"), # from BS4 - # gutters - ReplaceClasses.op("no-gutters", "g-0"), - # logical properties - RegexReplaceClass.op(rf"{BS}left-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"start-\1"), - RegexReplaceClass.op(rf"{BS}right-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"end-\1"), - RegexReplaceClass.op(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-left{BE}", r"\1-start"), - RegexReplaceClass.op(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-right{BE}", r"\1-end"), - RegexReplaceClass.op(rf"{BS}rounded-sm(-(?:start|end|top|bottom))?", r"rounded\1-1"), - RegexReplaceClass.op(rf"{BS}rounded-lg(-(?:start|end|top|bottom))?", r"rounded\1-3"), - RegexReplaceClass.op(rf"{BS}([mp])l-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1s-\2"), - RegexReplaceClass.op(rf"{BS}([mp])r-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1e-\2"), - ReplaceClasses.op("dropdown-menu-left", "dropdown-menu-start"), - ReplaceClasses.op("dropdown-menu-right", "dropdown-menu-end"), - ReplaceClasses.op("dropleft", "dropstart"), - ReplaceClasses.op("dropright", "dropend"), - # tooltips - ( - "//*[hasclass('tooltip') or @role='tooltip']//*[hasclass('arrow')]", - [ReplaceClasses("arrow", "tooltip-arrow")], - ), - # utilities - ReplaceClasses.op("text-monospace", "font-monospace"), - RegexReplaceClass.op(rf"{BS}font-weight-", r"fw-"), - RegexReplaceClass.op(rf"{BS}font-style-", r"fst-"), - ReplaceClasses.op("font-italic", "fst-italic"), - # helpers - RegexReplaceClass.op(rf"{BS}embed-responsive-(\d+)by(\d+)", r"ratio-\1x\2"), - RegexReplaceClass.op(rf"{BS}ratio-(\d+)by(\d+)", r"ratio-\1x\2"), - RegexReplaceClass.op(rf"{BS}embed-responsive(?!-)", r"ratio"), - RegexReplaceClass.op(rf"{BS}sr-only(-focusable)?", r"visually-hidden\1"), - # media - ReplaceClasses.op("media-body", "flex-grow-1"), - ReplaceClasses.op("media", "d-flex"), - ], - } - - def __init__(self, tree, is_html=False, is_qweb=False): - self.tree = tree - self.is_html = is_html - self.is_qweb = is_qweb - - @classmethod - def _get_sorted_conversions(cls): - """Return the conversions dict sorted by version, from oldest to newest.""" - return sorted(cls.CONVERSIONS.items(), key=lambda kv: Version(kv[0])) - - @classmethod - @lru_cache(maxsize=8) - def get_conversions(cls, src_ver, dst_ver, is_qweb=False): - """ - Return the list of conversions to convert Bootstrap from ``src_ver`` to ``dst_ver``, with compiled XPaths. - - :param str src_ver: the source Bootstrap version. - :param str dst_ver: the destination Bootstrap version. - :param bool is_qweb: whether to adapt conversions for QWeb (to support ``t-att(f)-`` conversions). - :rtype: list[(etree.XPath, list[ElementOperation])] - """ - if Version(dst_ver) < Version(src_ver): - raise NotImplementedError("Downgrading Bootstrap versions is not supported.") - if Version(src_ver) < Version(cls.MIN_VERSION): - raise NotImplementedError(f"Conversion from Bootstrap version {src_ver} is not supported") - result = [] - for version, conversions in BootstrapConverter._get_sorted_conversions(): - if Version(src_ver) < Version(version) <= Version(dst_ver): - result.extend(conversions) - if not result: - if Version(src_ver) == Version(dst_ver): - _logger.info("Source and destination versions are the same, no conversion needed.") - else: - raise NotImplementedError(f"Conversion from {src_ver} to {dst_ver} is not supported") - if is_qweb: - result = [(adapt_xpath_for_qweb(xpath), conversions) for xpath, conversions in result] - return [(etree.XPath(xpath), conversions) for xpath, conversions in result] - - def convert(self, src_version, dst_version): - """ - Convert the loaded document inplace from the source version to the destination, returning the converted document and the number of conversion operations applied. - - :param str src_version: the source Bootstrap version. - :param str dst_version: the destination Bootstrap version. - :rtype: etree.ElementTree, int - """ - conversions = self.get_conversions(src_version, dst_version, is_qweb=self.is_qweb) - applied_operations_count = 0 - for xpath, operations in conversions: - for element in xpath(self.tree): - for operation in operations: - if element is None: # previous operations that returned None (i.e. deleted element) - raise ValueError("Matched xml element is not available anymore! Check operations.") - element = operation(element, self) # noqa: PLW2901 - applied_operations_count += 1 - return self.tree, applied_operations_count - - @classmethod - def convert_arch(cls, arch, src_version, dst_version, is_html=False, **converter_kwargs): - """ - Class method for converting a string of XML or HTML code. - - :param str arch: the XML or HTML code to convert. - :param str src_version: the source Bootstrap version. - :param str dst_version: the destination Bootstrap version. - :param bool is_html: whether the arch is an HTML document. - :param dict converter_kwargs: additional keyword arguments to pass to the converter. - :return: the converted XML or HTML code. - :rtype: str - """ - stripped_arch = arch.strip() - doc_header_match = re.search(r"^<\?xml .+\?>\s*", stripped_arch) - doc_header = doc_header_match.group(0) if doc_header_match else "" - stripped_arch = stripped_arch[doc_header_match.end() :] if doc_header_match else stripped_arch - - tree = etree.fromstring(f"{stripped_arch}", parser=html_utf8_parser if is_html else None) - - tree, ops_count = cls(tree, is_html, **converter_kwargs).convert(src_version, dst_version) - if not ops_count: - return arch - - wrap_node = tree.xpath("//wrap")[0] - return doc_header + "\n".join( - etree.tostring(child, encoding="unicode", with_tail=True, method="html" if is_html else None) - for child in wrap_node - ) - - @classmethod - def convert_file(cls, path, src_version, dst_version, is_html=None, **converter_kwargs): - """ - Class method for converting an XML or HTML file inplace. - - :param str path: the path to the XML or HTML file to convert. - :param str src_version: the source Bootstrap version. - :param str dst_version: the destination Bootstrap version. - :param bool is_html: whether the file is an HTML document. - If not set, will be detected from the file extension. - :param dict converter_kwargs: additional keyword arguments to pass to the converter. - :rtype: None - """ - if is_html is None: - is_html = os.path.splitext(path)[1].startswith("htm") - tree = etree.parse(path, parser=html_utf8_parser if is_html else None) - - tree, ops_count = cls(tree, is_html, **converter_kwargs).convert(src_version, dst_version) - if not ops_count: - logging.info("No conversion operations applied, skipping file %s", path) - return - - tree.write(path, encoding="utf-8", method="html" if is_html else None, xml_declaration=not is_html) - - def element_factory(self, *args, **kwargs): - """ - Create new elements using the correct document type. - - Basically a wrapper for either etree.XML or etree.HTML depending on the type of document loaded. - - :param args: positional arguments to pass to the etree.XML or etree.HTML function. - :param kwargs: keyword arguments to pass to the etree.XML or etree.HTML function. - :return: the created element. - """ - return etree.HTML(*args, **kwargs) if self.is_html else etree.XML(*args, **kwargs) - - def build_element(self, tag, classes=None, contents=None, **attributes): - """ - Create a new element with the given tag, classes, contents and attributes. - - Like :meth:`~.element_factory`, can be used by operations to create elements abstracting away the document type. - - :param str tag: the tag of the element to create. - :param typing.Iterable[str] | None classes: the classes to set on the new element. - :param str | None contents: the contents of the new element (i.e. inner text/HTML/XML). - :param dict[str, str] attributes: attributes to set on the new element, provided as keyword arguments. - :return: the created element. - :rtype: etree.ElementBase - """ - element = self.element_factory(f"<{tag}>{contents or ''}") - for name, value in attributes.items(): - element.attrib[name] = value - if classes: - set_classes(element, classes) - return element - - def copy_element( - self, - element, - tag=None, - add_classes=None, - remove_classes=None, - copy_attrs=True, - copy_contents=True, - **attributes, - ): - """ - Create a copy of an element, optionally changing the tag, classes, contents and attributes. - - Like :meth:`~.element_factory`, can be used by operations to copy elements abstracting away the document type. - - :param etree.ElementBase element: the element to copy. - :param str | None tag: if specified, overrides the tag of the new element. - :param str | typing.Iterable[str] | None add_classes: if specified, adds the given class(es) to the new element. - :param str | typing.Iterable[str] | ALL | None remove_classes: if specified, removes the given class(es) - from the new element. The `ALL` sentinel value can be specified to remove all classes. - :param bool copy_attrs: if True, copies the attributes of the source element to the new one. Defaults to True. - :param bool copy_contents: if True, copies the contents of the source element to the new one. Defaults to True. - :param dict[str, str] attributes: attributes to set on the new element, provided as keyword arguments. - Will be str merged with the attributes of the source element, overriding the latter. - :return: the new copied element. - :rtype: etree.ElementBase - """ - tag = tag or element.tag - contents = innerxml(element, is_html=self.is_html) if copy_contents else None - if copy_attrs: - attributes = {**element.attrib, **attributes} - new_element = self.build_element(tag, contents=contents, **attributes) - edit_element_classes(new_element, add_classes, remove_classes, is_qweb=self.is_qweb) - return new_element - - def adapt_xpath(self, xpath): - """Adapts an xpath to match qweb ``t-att(f)-*`` attributes, if ``is_qweb`` is True.""" - return adapt_xpath_for_qweb(xpath) if self.is_qweb else xpath - - -def convert_tree(tree, src_version, dst_version, **converter_kwargs): - """ - Convert an already parsed lxml tree from Bootstrap v3 to v4 inplace. - - :param etree.ElementTree tree: the lxml tree to convert. - :param str src_version: the version of Bootstrap the document is currently using. - :param str dst_version: the version of Bootstrap to convert the document to. - :param dict converter_kwargs: additional keyword arguments to initialize :class:`~.BootstrapConverter`. - :return: the converted lxml tree. - :rtype: etree.ElementTree - """ - tree, ops_count = BootstrapConverter(tree, **converter_kwargs).convert(src_version, dst_version) - return tree - - -convert_arch = BootstrapConverter.convert_arch -convert_file = BootstrapConverter.convert_file - - -class BootstrapHTMLConverter: - def __init__(self, src, dst): - self.src = src - self.dst = dst - - def __call__(self, content): - if not content: - return False, content - converted_content = convert_arch(content, self.src, self.dst, is_html=True, is_qweb=True) - return content != converted_content, converted_content +warnings.warn( + "`util.convert_bootstrap` module has been deprecated in favor of `util.views.bootstrap`. " + "Consider adjusting your imports.", + category=DeprecationWarning, + stacklevel=1, +) diff --git a/src/util/domains.py b/src/util/domains.py index c7fa004fa..728809621 100644 --- a/src/util/domains.py +++ b/src/util/domains.py @@ -37,7 +37,7 @@ from .inherit import for_each_inherit from .misc import SelfPrintEvalContext from .pg import column_exists, get_value_or_en_translation, table_exists -from .records import edit_view +from .views.records import edit_view # python3 shims try: diff --git a/src/util/models.py b/src/util/models.py index b593beff1..2d47c4c09 100644 --- a/src/util/models.py +++ b/src/util/models.py @@ -31,8 +31,9 @@ # avoid namespace clash from .pg import rename_table as pg_rename_table -from .records import _rm_refs, remove_records, remove_view, replace_record_references_batch +from .records import _rm_refs, remove_records, replace_record_references_batch from .report import add_to_migration_reports +from .views.records import remove_view _logger = logging.getLogger(__name__) diff --git a/src/util/modules.py b/src/util/modules.py index 32c8ddfff..248002ad6 100644 --- a/src/util/modules.py +++ b/src/util/modules.py @@ -48,7 +48,8 @@ from .models import delete_model from .orm import env, flush from .pg import column_exists, table_exists, target_of -from .records import ref, remove_menus, remove_records, remove_view, replace_record_references_batch +from .records import ref, remove_menus, remove_records, replace_record_references_batch +from .views.records import remove_view INSTALLED_MODULE_STATES = ("installed", "to install", "to upgrade") NO_AUTOINSTALL = str2bool(os.getenv("UPG_NO_AUTOINSTALL", "0")) if version_gte("15.0") else False diff --git a/src/util/records.py b/src/util/records.py index 02d6a50dd..ab05c82bf 100644 --- a/src/util/records.py +++ b/src/util/records.py @@ -4,7 +4,6 @@ import logging import os import re -from contextlib import contextmanager from operator import itemgetter import lxml @@ -14,7 +13,6 @@ from odoo import release from odoo.tools.convert import xml_import from odoo.tools.misc import file_open - from odoo.tools.translate import xml_translate except ImportError: from openerp import release from openerp.tools.convert import xml_import @@ -44,7 +42,7 @@ table_exists, target_of, ) -from .report import add_to_migration_reports +from .views.records import add_view, edit_view, remove_view # noqa: F401 _logger = logging.getLogger(__name__) @@ -55,249 +53,6 @@ basestring = unicode = str -def remove_view(cr, xml_id=None, view_id=None, silent=False, key=None): - """ - Remove a view and all its descendants. - - This function recursively deletes the given view and its inherited views, as long as - they are part of a module. It will fail as soon as a custom view exists anywhere in - the hierarchy. It also removes multi-website COWed views. - - :param str xml_id: optional, the xml_id of the view to remove - :param int view_id: optional, the ID of the view to remove - :param bool silent: whether to show in the logs disabled custom views - :param str or None key: key used to detect multi-website COWed views, if `None` then - set to `xml_id` if provided, otherwise set to the xml_id - referencing the view with ID `view_id` if any - - .. warning:: - Either `xml_id` or `view_id` must be set. Specifying both will raise an error. - """ - assert bool(xml_id) ^ bool(view_id) - if xml_id: - view_id = ref(cr, xml_id) - if view_id: - module, _, name = xml_id.partition(".") - cr.execute("SELECT model FROM ir_model_data WHERE module=%s AND name=%s", [module, name]) - - [model] = cr.fetchone() - if model != "ir.ui.view": - raise ValueError("%r should point to a 'ir.ui.view', not a %r" % (xml_id, model)) - else: - # search matching xmlid for logging or renaming of custom views - xml_id = "?" - if not key: - cr.execute("SELECT module, name FROM ir_model_data WHERE model='ir.ui.view' AND res_id=%s", [view_id]) - if cr.rowcount: - xml_id = "%s.%s" % cr.fetchone() - - # From given or determined xml_id, the views duplicated in a multi-website - # context are to be found and removed. - if xml_id != "?" and column_exists(cr, "ir_ui_view", "key"): - cr.execute("SELECT id FROM ir_ui_view WHERE key = %s AND id != %s", [xml_id, view_id]) - for [v_id] in cr.fetchall(): - remove_view(cr, view_id=v_id, silent=silent, key=xml_id) - - if not view_id: - return - - cr.execute( - """ - SELECT v.id, x.module || '.' || x.name, v.name - FROM ir_ui_view v LEFT JOIN - ir_model_data x ON (v.id = x.res_id AND x.model = 'ir.ui.view' AND x.module !~ '^_') - WHERE v.inherit_id = %s; - """, - [view_id], - ) - for child_id, child_xml_id, child_name in cr.fetchall(): - if child_xml_id: - if not silent: - _logger.info( - "remove deprecated built-in view %s (ID %s) as parent view %s (ID %s) is going to be removed", - child_xml_id, - child_id, - xml_id, - view_id, - ) - remove_view(cr, child_xml_id, silent=True) - else: - if not silent: - _logger.warning( - "deactivate deprecated custom view with ID %s as parent view %s (ID %s) is going to be removed", - child_id, - xml_id, - view_id, - ) - disable_view_query = """ - UPDATE ir_ui_view - SET name = (name || ' - old view, inherited from ' || %%s), - inherit_id = NULL - %s - WHERE id = %%s - """ - # In 8.0, disabling requires setting mode to 'primary' - extra_set_sql = "" - if column_exists(cr, "ir_ui_view", "mode"): - extra_set_sql = ", mode = 'primary' " - - # Column was not present in v7 and it's older version - if column_exists(cr, "ir_ui_view", "active"): - extra_set_sql += ", active = false " - - disable_view_query = disable_view_query % extra_set_sql - cr.execute(disable_view_query, (key or xml_id, child_id)) - add_to_migration_reports( - {"id": child_id, "name": child_name}, - "Disabled views", - ) - if not silent: - _logger.info("remove deprecated %s view %s (ID %s)", key and "COWed" or "built-in", key or xml_id, view_id) - - remove_records(cr, "ir.ui.view", [view_id]) - - -@contextmanager -def edit_view(cr, xmlid=None, view_id=None, skip_if_not_noupdate=True, active=True): - """ - Context manager to edit a view's arch. - - This function returns a context manager that may yield a parsed arch of a view as an - `etree Element `_. Any changes done - in the returned object will be written back to the database upon exit of the context - manager, updating also the translated versions of the arch. Since the function may not - yield, use :func:`~odoo.upgrade.util.misc.skippable_cm` to avoid errors. - - .. code-block:: python - - with util.skippable_cm(), util.edit_view(cr, "xml.id") as arch: - arch.attrib["string"] = "My Form" - - To select the target view to edit use either `xmlid` or `view_id`, not both. - - When the view is identified by `view_id`, the arch is always yielded if the view - exists, with disregard to any `noupdate` flag it may have associated. When `xmlid` is - set, if the view `noupdate` flag is `True` then the arch will not be yielded *unless* - `skip_if_not_noupdate` is set to `False`. If `noupdate` is `False`, the view will be - yielded for edit. - - If the `active` argument is not `None`, the `active` flag of the view will be set - accordingly. - - .. warning:: - The default value of `active` is `True`, therefore views are always *activated* by - default. To avoid inadvertently activating views, pass `None` as `active` parameter. - - :param str xmlid: optional, xml_id of the view edit - :param int view_id: optional, ID of the view to edit - :param bool skip_if_not_noupdate: whether to force the edit of views requested via - `xmlid` parameter even if they are flagged as - `noupdate=True`, ignored if `view_id` is set - :param bool or None active: active flag value to set, nothing is set when `None` - :return: a context manager that yields the parsed arch, upon exit the context manager - writes back the changes. - """ - assert bool(xmlid) ^ bool(view_id), "You Must specify either xmlid or view_id" - noupdate = True - if xmlid: - if "." not in xmlid: - raise ValueError("Please use fully qualified name .") - - module, _, name = xmlid.partition(".") - cr.execute( - """ - SELECT res_id, noupdate - FROM ir_model_data - WHERE module = %s - AND name = %s - """, - [module, name], - ) - data = cr.fetchone() - if data: - view_id, noupdate = data - - if view_id and not (skip_if_not_noupdate and not noupdate): - arch_col = "arch_db" if column_exists(cr, "ir_ui_view", "arch_db") else "arch" - jsonb_column = column_type(cr, "ir_ui_view", arch_col) == "jsonb" - cr.execute( - """ - SELECT {arch} - FROM ir_ui_view - WHERE id=%s - """.format( - arch=arch_col, - ), - [view_id], - ) - [arch] = cr.fetchone() or [None] - if arch: - - def parse(arch): - arch = arch.encode("utf-8") if isinstance(arch, unicode) else arch - return lxml.etree.fromstring(arch.replace(b" \n", b"\n").strip()) - - if jsonb_column: - - def get_trans_terms(value): - terms = [] - xml_translate(terms.append, value) - return terms - - translation_terms = {lang: get_trans_terms(value) for lang, value in arch.items()} - arch_etree = parse(arch["en_US"]) - yield arch_etree - new_arch = lxml.etree.tostring(arch_etree, encoding="unicode") - terms_en = translation_terms["en_US"] - arch_column_value = Json( - { - lang: xml_translate(dict(zip(terms_en, terms)).get, new_arch) - for lang, terms in translation_terms.items() - } - ) - else: - arch_etree = parse(arch) - yield arch_etree - arch_column_value = lxml.etree.tostring(arch_etree, encoding="unicode") - - set_active = ", active={}".format(bool(active)) if active is not None else "" - cr.execute( - "UPDATE ir_ui_view SET {arch}=%s{set_active} WHERE id=%s".format(arch=arch_col, set_active=set_active), - [arch_column_value, view_id], - ) - - -def add_view(cr, name, model, view_type, arch_db, inherit_xml_id=None, priority=16): - inherit_id = None - if inherit_xml_id: - inherit_id = ref(cr, inherit_xml_id) - if not inherit_id: - raise ValueError( - "Unable to add view '%s' because its inherited view '%s' cannot be found!" % (name, inherit_xml_id) - ) - arch_col = "arch_db" if column_exists(cr, "ir_ui_view", "arch_db") else "arch" - jsonb_column = column_type(cr, "ir_ui_view", arch_col) == "jsonb" - arch_column_value = Json({"en_US": arch_db}) if jsonb_column else arch_db - cr.execute( - """ - INSERT INTO ir_ui_view(name, "type", model, inherit_id, mode, active, priority, %s) - VALUES(%%(name)s, %%(view_type)s, %%(model)s, %%(inherit_id)s, %%(mode)s, 't', %%(priority)s, %%(arch_db)s) - RETURNING id - """ - % arch_col, - { - "name": name, - "view_type": view_type, - "model": model, - "inherit_id": inherit_id, - "mode": "extension" if inherit_id else "primary", - "priority": priority, - "arch_db": arch_column_value, - }, - ) - return cr.fetchone()[0] - - # fmt:off if version_gte("saas~14.3"): def remove_asset(cr, name): diff --git a/src/util/snippets.py b/src/util/snippets.py index a12ed7b2f..ba349e0f2 100644 --- a/src/util/snippets.py +++ b/src/util/snippets.py @@ -12,7 +12,9 @@ from psycopg2.extras import Json from .exceptions import MigrationError -from odoo.upgrade import util +from .helpers import table_of_model +from .misc import import_script, log_progress +from .pg import column_exists, column_type, get_max_workers, table_exists _logger = logging.getLogger(__name__) utf8_parser = html.HTMLParser(encoding="utf-8") @@ -38,7 +40,7 @@ def add_snippet_names(cr, table, column, snippets, select_query): _logger.info("Add snippet names on %s.%s", table, column) cr.execute(select_query) - it = util.log_progress(cr.fetchall(), _logger, qualifier="rows", size=cr.rowcount, log_hundred_percent=True) + it = log_progress(cr.fetchall(), _logger, qualifier="rows", size=cr.rowcount, log_hundred_percent=True) def quote(ident): return quote_ident(ident, cr._cnx) @@ -103,11 +105,11 @@ def html_fields(cr): """ ) for model, columns in cr.fetchall(): - table = util.table_of_model(cr, model) - if not util.table_exists(cr, table): + table = table_of_model(cr, model) + if not table_exists(cr, table): # an SQL VIEW continue - existing_columns = [column for column in columns if util.column_exists(cr, table, column)] + existing_columns = [column for column in columns if column_exists(cr, table, column)] if existing_columns: yield table, existing_columns @@ -169,7 +171,7 @@ def make_pickleable_callback(callback): """ callback_filepath = inspect.getfile(callback) name = f"_upgrade_{uuid.uuid4().hex}" - mod = sys.modules[name] = util.import_script(callback_filepath, name=name) + mod = sys.modules[name] = import_script(callback_filepath, name=name) try: return getattr(mod, callback.__name__) except AttributeError: @@ -257,7 +259,7 @@ def convert_html_columns(cr, table, columns, converter_callback, where_column="I """ assert "id" not in columns - converters = {column: "->>'en_US'" if util.column_type(cr, table, column) == "jsonb" else "" for column in columns} + converters = {column: "->>'en_US'" if column_type(cr, table, column) == "jsonb" else "" for column in columns} select = ", ".join(f'"{column}"' for column in columns) where = " OR ".join(f'"{column}"{converters[column]} {where_column}' for column in columns) @@ -275,13 +277,19 @@ def convert_html_columns(cr, table, columns, converter_callback, where_column="I update_sql = ", ".join(f'"{column}" = %({column})s' for column in columns) update_query = f"UPDATE {table} SET {update_sql} WHERE id = %(id)s" - with ProcessPoolExecutor(max_workers=util.get_max_workers()) as executor: + matched_count = 0 + converted_count = 0 + with ProcessPoolExecutor(max_workers=get_max_workers()) as executor: convert = Convertor(converters, converter_callback) - for query in util.log_progress(split_queries, logger=_logger, qualifier=f"{table} updates"): + for query in log_progress(split_queries, logger=_logger, qualifier=f"{table} updates"): cr.execute(query) for data in executor.map(convert, cr.fetchall()): + matched_count += 1 if "id" in data: cr.execute(update_query, data) + converted_count += 1 + + return matched_count, converted_count def determine_chunk_limit_ids(cr, table, column_arr, where): @@ -304,6 +312,7 @@ def convert_html_content( cr, converter_callback, where_column="IS NOT NULL", + verbose=False, **kwargs, ): r""" @@ -316,9 +325,16 @@ def convert_html_content( :param str where_column: filtering such as - "like '%abc%xyz%'" - "~* '\yabc.*xyz\y'" + :param bool verbose: print stats about the conversion :param dict kwargs: extra keyword arguments to pass to :func:`convert_html_column` """ - convert_html_columns( + if verbose: + _logger.info("Converting html fields data using %s", repr(converter_callback)) + + matched_count = 0 + converted_count = 0 + + matched, converted = convert_html_columns( cr, "ir_ui_view", ["arch_db"], @@ -326,6 +342,18 @@ def convert_html_content( where_column=where_column, **dict(kwargs, extra_where="type = 'qweb'"), ) + matched_count += matched + converted_count += converted for table, columns in html_fields(cr): - convert_html_columns(cr, table, columns, converter_callback, where_column=where_column, **kwargs) + matched, converted = convert_html_columns( + cr, table, columns, converter_callback, where_column=where_column, **kwargs + ) + matched_count += matched + converted_count += converted + + if verbose: + if matched_count: + _logger.info("Converted %d/%d matched html fields values", converted_count, matched_count) + else: + _logger.info("Did not match any html fields values to convert") diff --git a/src/util/views/__init__.py b/src/util/views/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/util/views/bootstrap.py b/src/util/views/bootstrap.py new file mode 100644 index 000000000..dc6d5d4d4 --- /dev/null +++ b/src/util/views/bootstrap.py @@ -0,0 +1,491 @@ +"""Convert an XML/HTML document Bootstrap code from an older version to a newer one.""" + +import logging +from functools import lru_cache + +from lxml import etree + +try: + from packaging.version import Version +except ImportError: + from distutils.version import StrictVersion as Version # N.B. deprecated, will be removed in py3.12 + +from .convert import ( + ALL, + BE, + BS, + CSS, + AddClasses, + ElementOperation, + EtreeConverter, + PullUp, + RegexReplaceClass, + RemoveClasses, + RemoveElement, + RenameAttribute, + ReplaceClasses, + edit_element_classes, + get_classes, + regex_xpath, +) + +_logger = logging.getLogger(__name__) + + +class BS3to4ConvertBlockquote(ElementOperation): + """ + Convert a BS3 ``
`` element to a BS4 ``
`` element with the ``blockquote`` class. + + :meta private: exclude from online docs + """ + + def __call__(self, element, converter): + blockquote = converter.copy_element(element, tag="div", add_classes="blockquote", copy_attrs=False) + element.addnext(blockquote) + element.getparent().remove(element) + return blockquote + + +# TODO abt: merge MakeCard and ConvertCard into one operation class +class BS3to4MakeCard(ElementOperation): + """ + Pre-processe a BS3 panel, thumbnail, or well element to be converted to a BS4 card. + + Card components conversion is then handled by the ``ConvertCard`` operation class. + + :meta private: exclude from online docs + """ + + def __call__(self, element, converter): + card = converter.element_factory("
") + card_body = converter.copy_element( + element, tag="div", add_classes="card-body", remove_classes=ALL, copy_attrs=False + ) + card.append(card_body) + element.addnext(card) + element.getparent().remove(element) + return card + + +# TODO abt: refactor code +class BS3to4ConvertCard(ElementOperation): + """ + Fully convert a BS3 panel, thumbnail, or well element and their contents to a BS4 card. + + :meta private: exclude from online docs + """ + + POST_CONVERSIONS = { + "title": ["card-title"], + "description": ["card-description"], + "category": ["card-category"], + "panel-danger": ["card", "bg-danger", "text-white"], + "panel-warning": ["card", "bg-warning"], + "panel-info": ["card", "bg-info", "text-white"], + "panel-success": ["card", "bg-success", "text-white"], + "panel-primary": ["card", "bg-primary", "text-white"], + "panel-footer": ["card-footer"], + "panel-body": ["card-body"], + "panel-title": ["card-title"], + "panel-heading": ["card-header"], + "panel-default": [], + "panel": ["card"], + } + + def _convert_child(self, child, old_card, new_card, converter): + old_card_classes = get_classes(old_card) + + classes = get_classes(child) + + if "header" in classes or ("image" in classes and len(child)): + add_classes = "card-header" + remove_classes = ["header", "image"] + elif "content" in classes: + add_classes = "card-img-overlay" if "card-background" in old_card_classes else "card-body" + remove_classes = "content" + elif {"card-footer", "footer", "text-center"} & set(classes): + add_classes = "card-footer" + remove_classes = "footer" + else: + new_card.append(child) + return + + new_child = converter.copy_element( + child, "div", add_classes=add_classes, remove_classes=remove_classes, copy_attrs=True + ) + + if "image" in classes: + [img_el] = new_child.xpath("./img")[:1] or [None] + if img_el is not None and "src" in img_el: + new_child.attrib["style"] = ( + f'background-image: url("{img_el.attrib["src"]}"); ' + "background-position: center center; " + "background-size: cover;" + ) + new_child.remove(img_el) + + new_card.append(new_child) + + if "content" in classes: # TODO abt: consider skipping for .card-background + [footer] = new_child.xpath(converter.adapt_xpath("./*[hasclass('footer')]"))[:1] or [None] + if footer is not None: + self._convert_child(footer, old_card, new_card, converter) + new_child.remove(footer) + + def _postprocess(self, new_card, converter): + for old_class, new_classes in self.POST_CONVERSIONS.items(): + for element in new_card.xpath(converter.adapt_xpath(f"(.|.//*)[hasclass('{old_class}')]")): + edit_element_classes(element, add=new_classes, remove=old_class) + + def __call__(self, element, converter): + classes = get_classes(element) + new_card = converter.copy_element(element, tag="div", copy_attrs=True, copy_contents=False) + wrapper = new_card + if "card-horizontal" in classes: + wrapper = etree.SubElement(new_card, "div", {"class": "row"}) + + for child in element: + self._convert_child(child, element, wrapper, converter) + + self._postprocess(new_card, converter) + element.addnext(new_card) + element.getparent().remove(element) + return new_card + + +class BS4to5ConvertCloseButton(ElementOperation): + """ + Convert BS4 ``button.close`` elements to BS5 ``button.btn-close``. + + Also fixes the ``data-dismiss`` attribute to ``data-bs-dismiss``, and removes any inner contents. + + :meta private: exclude from online docs + """ + + def __call__(self, element, converter): + new_btn = converter.copy_element(element, remove_classes="close", add_classes="btn-close", copy_contents=False) + + if "data-dismiss" in element.attrib: + new_btn.attrib["data-bs-dismiss"] = element.attrib["data-dismiss"] + del new_btn.attrib["data-dismiss"] + + element.addnext(new_btn) + element.getparent().remove(element) + + return new_btn + + +class BS4to5ConvertCardDeck(ElementOperation): + """ + Convert BS4 ``.card-deck`` elements to grid components (``.row``, ``.col``, etc.). + + :meta private: exclude from online docs + """ + + def __call__(self, element, converter): + cards = element.xpath(converter.adapt_xpath("./*[hasclass('card')]")) + + cols_class = f"row-cols-{len(cards)}" if len(cards) in range(1, 7) else "row-cols-auto" + edit_element_classes(element, add=["row", cols_class], remove="card-deck", is_qweb=converter.is_qweb) + + for card in cards: + new_col = converter.build_element("div", classes=["col"]) + card.addprevious(new_col) + new_col.append(card) + + return element + + +class BS4to5ConvertFormInline(ElementOperation): + """ + Convert BS4 ``.form-inline`` elements to grid components (``.row``, ``.col``, etc.). + + :meta private: exclude from online docs + """ + + def __call__(self, element, converter): + edit_element_classes(element, add="row row-cols-lg-auto", remove="form-inline", is_qweb=converter.is_qweb) + + children_selector = converter.adapt_xpath( + CSS(".form-control,.form-group,.form-check,.input-group,.custom-select,button", prefix="./") + ) + indexed_children = sorted([(element.index(c), c) for c in element.xpath(children_selector)], key=lambda x: x[0]) + + nest_groups = [] + last_idx = -1 + for idx, child in indexed_children: + nest_start, nest_end = idx, idx + labels = [label for label in child.xpath("preceding-sibling::label") if element.index(label) > last_idx] + labels = [ + label + for label in labels + if "for" in label.attrib and child.xpath(f"descendant-or-self::*[@id='{label.attrib['for']}']") + ] or labels[-1:] + if labels: + first_label = labels[0] + assert last_idx < element.index(first_label) < idx, "label must be between last group and current" + nest_start = element.index(first_label) + + assert nest_start <= nest_end, f"expected start {nest_start} to be <= end {nest_end}" + nest_groups.append(element[nest_start : nest_end + 1]) + last_idx = nest_end + + for els in nest_groups: + wrapper = converter.build_element("div", classes=["col-12"]) + els[0].addprevious(wrapper) + for el in els: + wrapper.append(el) + assert el not in element, f"expected {el!r} to be removed from {element!r}" + + return element + + +class BootstrapConverter(EtreeConverter): + """ + Class for converting XML or HTML Bootstrap code across versions. + + :param str src_version: the source Bootstrap version to convert from. + :param str dst_version: the destination Bootstrap version to convert to. + :param bool is_html: whether the tree is an HTML document. + :param bool is_qweb: whether the tree contains QWeb directives. See :class:`EtreeConverter` for more info. + + :meta private: exclude from online docs + """ + + MIN_VERSION = "3.0" + """Minimum supported Bootstrap version.""" + + # Conversions definitions by destination version. + # It's a dictionary of version strings to a conversions list. + # Each item in the conversions list must either be an :class:`ElementOperation`, + # that can provide its own XPath, or a tuple of ``(xpath, operation(s))`` with the XPath + # and a single operation or a list of operations to apply to the nodes matching the XPath. + CONVERSIONS = { + "4.0": [ + # inputs + (CSS(".form-group .control-label"), ReplaceClasses("control-label", "form-control-label")), + (CSS(".form-group .text-help"), ReplaceClasses("text-help", "form-control-feedback")), + (CSS(".control-group .help-block"), ReplaceClasses("help-block", "form-text")), + ReplaceClasses("form-group-sm", "form-control-sm"), + ReplaceClasses("form-group-lg", "form-control-lg"), + (CSS(".form-control .input-sm"), ReplaceClasses("input-sm", "form-control-sm")), + (CSS(".form-control .input-lg"), ReplaceClasses("input-lg", "form-control-lg")), + # hide + ReplaceClasses("hidden-xs", "d-none"), + ReplaceClasses("hidden-sm", "d-sm-none"), + ReplaceClasses("hidden-md", "d-md-none"), + ReplaceClasses("hidden-lg", "d-lg-none"), + ReplaceClasses("visible-xs", "d-block d-sm-none"), + ReplaceClasses("visible-sm", "d-block d-md-none"), + ReplaceClasses("visible-md", "d-block d-lg-none"), + ReplaceClasses("visible-lg", "d-block d-xl-none"), + # image + ReplaceClasses("img-rounded", "rounded"), + ReplaceClasses("img-circle", "rounded-circle"), + ReplaceClasses("img-responsive", ("d-block", "img-fluid")), + # buttons + ReplaceClasses("btn-default", "btn-secondary"), + ReplaceClasses("btn-xs", "btn-sm"), + (CSS(".btn-group.btn-group-xs"), ReplaceClasses("btn-group-xs", "btn-group-sm")), + (CSS(".dropdown .divider"), ReplaceClasses("divider", "dropdown-divider")), + ReplaceClasses("badge", "badge badge-pill"), + ReplaceClasses("label", "badge"), + RegexReplaceClass(rf"{BS}label-(default|primary|success|info|warning|danger){BE}", r"badge-\1"), + (CSS(".breadcrumb > li"), ReplaceClasses("breadcrumb", "breadcrumb-item")), + # li + (CSS(".list-inline > li"), AddClasses("list-inline-item")), + # pagination + (CSS(".pagination > li"), AddClasses("page-item")), + (CSS(".pagination > li > a"), AddClasses("page-link")), + # carousel + (CSS(".carousel .carousel-inner > .item"), ReplaceClasses("item", "carousel-item")), + # pull + ReplaceClasses("pull-right", "float-right"), + ReplaceClasses("pull-left", "float-left"), + ReplaceClasses("center-block", "mx-auto"), + # well + (CSS(".well"), BS3to4MakeCard()), + (CSS(".thumbnail"), BS3to4MakeCard()), + # blockquote + (CSS("blockquote"), BS3to4ConvertBlockquote()), + (CSS(".blockquote.blockquote-reverse"), ReplaceClasses("blockquote-reverse", "text-right")), + # dropdown + (CSS(".dropdown-menu > li > a"), AddClasses("dropdown-item")), + (CSS(".dropdown-menu > li"), PullUp()), + # in + ReplaceClasses("in", "show"), + # table + (CSS("tr.active, td.active"), ReplaceClasses("active", "table-active")), + (CSS("tr.success, td.success"), ReplaceClasses("success", "table-success")), + (CSS("tr.info, td.info"), ReplaceClasses("info", "table-info")), + (CSS("tr.warning, td.warning"), ReplaceClasses("warning", "table-warning")), + (CSS("tr.danger, td.danger"), ReplaceClasses("danger", "table-danger")), + (CSS("table.table-condesed"), ReplaceClasses("table-condesed", "table-sm")), + # navbar + (CSS(".nav.navbar > li > a"), AddClasses("nav-link")), + (CSS(".nav.navbar > li"), AddClasses("nav-intem")), + ReplaceClasses("navbar-btn", "nav-item"), + (CSS(".navbar-nav"), ReplaceClasses("navbar-right nav", "ml-auto")), + ReplaceClasses("navbar-toggler-right", "ml-auto"), + (CSS(".navbar-nav > li > a"), AddClasses("nav-link")), + (CSS(".navbar-nav > li"), AddClasses("nav-item")), + (CSS(".navbar-nav > a"), AddClasses("navbar-brand")), + ReplaceClasses("navbar-fixed-top", "fixed-top"), + ReplaceClasses("navbar-toggle", "navbar-toggler"), + ReplaceClasses("nav-stacked", "flex-column"), + (CSS("nav.navbar"), AddClasses("navbar-expand-lg")), + (CSS("button.navbar-toggle"), ReplaceClasses("navbar-toggle", "navbar-expand-md")), + # card + (CSS(".panel"), BS3to4ConvertCard()), + (CSS(".card"), BS3to4ConvertCard()), + # grid + RegexReplaceClass(rf"{BS}col((?:-\w{{2}})?)-offset-(\d{{1,2}}){BE}", r"offset\1-\2"), + ], + "5.0": [ + # links + RegexReplaceClass(rf"{BS}text-(?!o-)", "link-", xpath=CSS("a")), + (CSS(".nav-item.active > .nav-link"), AddClasses("active")), + (CSS(".nav-link.active") + CSS(".nav-item.active", prefix="/parent::"), RemoveClasses("active")), + # badges + ReplaceClasses("badge-pill", "rounded-pill"), + RegexReplaceClass(rf"{BS}badge-", r"text-bg-"), + # buttons + ("//*[hasclass('btn-block')]/parent::div", AddClasses("d-grid gap-2")), + ("//*[hasclass('btn-block')]/parent::p[count(./*)=1]", AddClasses("d-grid gap-2")), + RemoveClasses("btn-block"), + (CSS("button.close"), BS4to5ConvertCloseButton()), + # card + # TODO abt: .card-columns (unused in odoo) + (CSS(".card-deck"), BS4to5ConvertCardDeck()), + # jumbotron + ReplaceClasses("jumbotron", "container-fluid py-5"), + # new data-bs- attributes + RenameAttribute("data-display", "data-bs-display", xpath="//*[not(@data-snippet='s_countdown')]"), + *[ + RenameAttribute(f"data-{attr}", f"data-bs-{attr}") + for attr in ( + "animation attributes autohide backdrop body container content delay dismiss focus" + " interval margin-right no-jquery offset original-title padding-right parent placement" + " ride sanitize show slide slide-to spy target toggle touch trigger whatever" + ).split(" ") + ], + # popover + (CSS(".popover .arrow"), ReplaceClasses("arrow", "popover-arrow")), + # form + ReplaceClasses("form-row", "row"), + ("//*[hasclass('form-group')]/parent::form", AddClasses("row")), + ReplaceClasses("form-group", "col-12 py-2"), + (CSS(".form-inline"), BS4to5ConvertFormInline()), + ReplaceClasses("custom-checkbox", "form-check"), + RegexReplaceClass(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"), + RegexReplaceClass(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"), + RegexReplaceClass(rf"{BS}custom-(check|select|range){BE}", r"form-\1"), + (CSS(".custom-switch"), ReplaceClasses("custom-switch", "form-check form-switch")), + ReplaceClasses("custom-radio", "form-check"), + RemoveClasses("custom-control"), + (CSS(".custom-file"), PullUp()), + RegexReplaceClass(rf"{BS}custom-file-", r"form-file-"), + RegexReplaceClass(rf"{BS}form-file(?:-input)?{BE}", r"form-control"), + (CSS("label.form-file-label"), RemoveElement()), + (regex_xpath(rf"{BS}input-group-(prepend|append){BE}", "class"), PullUp()), + ("//label[not(hasclass('form-check-label'))]", AddClasses("form-label")), + ReplaceClasses("form-control-file", "form-control"), + ReplaceClasses("form-control-range", "form-range"), + # TODO abt: .form-text no loger sets display, add some class? + # table + RegexReplaceClass(rf"{BS}thead-(light|dark){BE}", r"table-\1"), + # grid + RegexReplaceClass(rf"{BS}col-((?:\w{{2}}-)?)offset-(\d{{1,2}}){BE}", r"offset-\1\2"), # from BS4 + # gutters + ReplaceClasses("no-gutters", "g-0"), + # logical properties + RegexReplaceClass(rf"{BS}left-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"start-\1"), + RegexReplaceClass(rf"{BS}right-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"end-\1"), + RegexReplaceClass(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-left{BE}", r"\1-start"), + RegexReplaceClass(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-right{BE}", r"\1-end"), + RegexReplaceClass(rf"{BS}rounded-sm(-(?:start|end|top|bottom))?", r"rounded\1-1"), + RegexReplaceClass(rf"{BS}rounded-lg(-(?:start|end|top|bottom))?", r"rounded\1-3"), + RegexReplaceClass(rf"{BS}([mp])l-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1s-\2"), + RegexReplaceClass(rf"{BS}([mp])r-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1e-\2"), + ReplaceClasses("dropdown-menu-left", "dropdown-menu-start"), + ReplaceClasses("dropdown-menu-right", "dropdown-menu-end"), + ReplaceClasses("dropleft", "dropstart"), + ReplaceClasses("dropright", "dropend"), + # tooltips + ( + "//*[hasclass('tooltip') or @role='tooltip']//*[hasclass('arrow')]", + ReplaceClasses("arrow", "tooltip-arrow"), + ), + # utilities + ReplaceClasses("text-monospace", "font-monospace"), + RegexReplaceClass(rf"{BS}font-weight-", r"fw-"), + RegexReplaceClass(rf"{BS}font-style-", r"fst-"), + ReplaceClasses("font-italic", "fst-italic"), + # helpers + RegexReplaceClass(rf"{BS}embed-responsive-(\d+)by(\d+)", r"ratio-\1x\2"), + RegexReplaceClass(rf"{BS}ratio-(\d+)by(\d+)", r"ratio-\1x\2"), + RegexReplaceClass(rf"{BS}embed-responsive(?!-)", r"ratio"), + RegexReplaceClass(rf"{BS}sr-only(-focusable)?", r"visually-hidden\1"), + # media + ReplaceClasses("media-body", "flex-grow-1"), + ReplaceClasses("media", "d-flex"), + ], + } + + def __init__(self, src_version, dst_version, *, is_html=False, is_qweb=False): + self.src_version = src_version + self.dst_version = dst_version + conversions = self._get_conversions(src_version, dst_version) + super().__init__(conversions, is_html=is_html, is_qweb=is_qweb) + + @classmethod + def _get_sorted_conversions(cls): + """ + Return the conversions dict sorted by version, from oldest to newest. + + :meta private: exclude from online docs + """ + return sorted(cls.CONVERSIONS.items(), key=lambda kv: Version(kv[0])) + + @classmethod + @lru_cache(maxsize=8) + def _get_conversions(cls, src_version, dst_version): + """ + Return the list of conversions to convert Bootstrap from ``src_version`` to ``dst_version``. + + :param str src_version: the source Bootstrap version. + :param str dst_version: the destination Bootstrap version. + :rtype: list[ElementOperation | (str, ElementOperation | list[ElementOperation])] + + :meta private: exclude from online docs + """ + if Version(dst_version) < Version(src_version): + raise NotImplementedError("Downgrading Bootstrap versions is not supported.") + if Version(src_version) < Version(cls.MIN_VERSION): + raise NotImplementedError(f"Conversion from Bootstrap version {src_version} is not supported") + + result = [] + for version, conversions in BootstrapConverter._get_sorted_conversions(): + if Version(src_version) < Version(version) <= Version(dst_version): + result.extend(conversions) + + if not result: + if Version(src_version) == Version(dst_version): + _logger.info("Source and destination versions are the same, no conversion needed.") + else: + raise NotImplementedError(f"Conversion from {src_version} to {dst_version} is not supported") + + return result + + +# TODO abt: remove this / usages -> replace with refactored converter classes +class BootstrapHTMLConverter: + def __init__(self, src, dst): + self.src = src + self.dst = dst + + def __call__(self, content): + if not content: + return False, content + converted_content = BootstrapConverter.convert_arch(content, self.src, self.dst, is_html=True, is_qweb=True) + return content != converted_content, converted_content diff --git a/src/util/views/convert.py b/src/util/views/convert.py new file mode 100644 index 000000000..c496c4e71 --- /dev/null +++ b/src/util/views/convert.py @@ -0,0 +1,1253 @@ +"""Helpers to manipulate views/templates.""" + +import logging +import os.path +import re +from abc import ABC, abstractmethod +from functools import lru_cache +from typing import Iterable + +from lxml import etree + +from odoo.modules.module import get_modules + +from .. import misc, pg, snippets +from . import records + +_logger = logging.getLogger(__name__) + + +# Regex boundary patterns for class names within attributes (`\b` is not good enough +# because it matches `-` etc.). It's probably enough to detect if the class name +# is surrounded by either whitespace or beginning/end of string. +# Includes boundary chars that can appear in t-att(f)-* attributes. +BS = r"(?:^|(?<=[\s}'\"]))" +BE = r"(?:$|(?=[\s{#'\"]))" +B = rf"(?:{BS}|{BE})" + + +def _xpath_has_class(context, *cls): + """Extension function for xpath to check if the context node has all the classes passed as arguments.""" + node_classes = set(context.context_node.attrib.get("class", "").split()) + return node_classes.issuperset(cls) + + +@lru_cache(maxsize=128) # does >99% hits over ~10mln calls +def _xpath_has_t_class_inner(attrs_values, classes): + """ + Inner implementation of :func:`_xpath_has_t_class`, suitable for caching. + + :param tuple[str] attrs_values: the values of the ``t-att-class`` and ``t-attf-class`` attributes + :param tuple[str] classes: the classes to check + """ + return any( + all(re.search(rf"{BS}{escaped_cls}{BE}", attr_value or "") for escaped_cls in map(re.escape, classes)) + for attr_value in attrs_values + ) + + +def _xpath_has_t_class(context, *cls): + """Extension function for xpath to check if the context node has all the classes passed as arguments in one of ``class`` or ``t-att(f)-class`` attributes.""" + return _xpath_has_class(context, *cls) or _xpath_has_t_class_inner( + tuple(map(context.context_node.attrib.get, ("t-att-class", "t-attf-class"))), cls + ) + + +@lru_cache(maxsize=1024) # does >88% hits over ~1.5mln calls +def _xpath_regex_inner(pattern, item): + """Inner implementation of :func:`_xpath_regex`, suitable for caching.""" + return bool(re.search(pattern, item)) + + +def _xpath_regex(context, item, pattern): + """Extension function for xpath to check if the passed item (attribute or text) matches the passed regex pattern.""" + if not item: + return False + if isinstance(item, list): + item = item[0] # only first attribute is valid + return _xpath_regex_inner(pattern, item) + + +xpath_utils = etree.FunctionNamespace(None) +xpath_utils["hasclass"] = _xpath_has_class +xpath_utils["has-class"] = _xpath_has_class +xpath_utils["has-t-class"] = _xpath_has_t_class +xpath_utils["regex"] = _xpath_regex + +html_utf8_parser = etree.HTMLParser(encoding="utf-8") + + +def parse_arch(arch, is_html=False): + """ + Parse a string of XML or HTML into a lxml :class:`etree.ElementTree`. + + :param str arch: the XML or HTML code to convert. + :param bool is_html: whether the code is HTML or XML. + :return: the parsed etree and the original document header (if any, removed from the tree). + :rtype: (etree.ElementTree, str) + + :meta private: exclude from online docs + """ + stripped_arch = arch.strip() + doc_header_match = re.search(r"^<\?xml .+\?>\s*", stripped_arch) + doc_header = doc_header_match.group(0) if doc_header_match else "" + stripped_arch = stripped_arch[doc_header_match.end() :] if doc_header_match else stripped_arch + + return etree.fromstring(f"{stripped_arch}", parser=html_utf8_parser if is_html else None), doc_header + + +def unparse_arch(tree, doc_header="", is_html=False): + """ + Convert an etree into a string of XML or HTML. + + :param etree.ElementTree tree: the etree to convert. + :param str doc_header: the document header (if any). + :param bool is_html: whether the code is HTML or XML. + :return: the XML or HTML code. + :rtype: str + + :meta private: exclude from online docs + """ + wrap_node = tree.xpath("//wrap")[0] + return doc_header + "\n".join( + etree.tostring(child, encoding="unicode", with_tail=True, method="html" if is_html else None) + for child in wrap_node + ) + + +class ArchEditor: + """ + Context manager to edit an XML or HTML string. + + It will parse an XML or HTML string into an etree, and return it to its original + string representation when exiting the context. + + The etree is available as the ``tree`` attribute of the context manager. + The arch is available as the ``arch`` attribute of the context manager, + and is updated when exiting the context. + + :param str arch: the XML or HTML code to convert. + :param bool is_html: whether the code is HTML or XML. + + :meta private: exclude from online docs + """ + + def __init__(self, arch, is_html=False): + self.arch = arch + self.is_html = is_html + self.doc_header = "" + self.tree = None + + def __enter__(self): + self.tree, self.doc_header = parse_arch(self.arch, is_html=self.is_html) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type: + return + self.arch = unparse_arch(self.tree, self.doc_header, is_html=self.is_html) + + def __str__(self): + return self.arch + + +def innerxml(element, is_html=False): + """ + Return the inner XML of an element as a string. + + :param etree.ElementBase element: the element to convert. + :param bool is_html: whether to use HTML for serialization, XML otherwise. Defaults to False. + :rtype: str + + :meta private: exclude from online docs + """ + return (element.text or "") + "".join( + etree.tostring(child, encoding=str, method="html" if is_html else None) for child in element + ) + + +def split_classes(*joined_classes): + """ + Return a list of classes given one or more strings of joined classes separated by spaces. + + :meta private: exclude from online docs + """ + return [c for classes in joined_classes for c in (classes or "").split(" ") if c] + + +def get_classes(element): + """ + Return the list of classes from the ``class`` attribute of an element. + + :meta private: exclude from online docs + """ + return split_classes(element.get("class", "")) + + +def join_classes(classes): + """ + Return a string of classes joined by space given a list of classes. + + :meta private: exclude from online docs + """ + return " ".join(classes) + + +def set_classes(element, classes): + """ + Set the ``class`` attribute of an element from a list of classes. + + If the list is empty, the attribute is removed. + + :meta private: exclude from online docs + """ + if classes: + element.attrib["class"] = join_classes(classes) + else: + element.attrib.pop("class", None) + + +def edit_classlist(classes, add, remove): + """ + Edit a class list, adding and removing classes. + + :param str | typing.List[str] classes: the original classes list or str to edit. + :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the list. + :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) + from the list. The `ALL` sentinel value can be specified to remove all classes. + :rtype: typing.List[str] + :return: the new class list. + + :meta private: exclude from online docs + """ + if remove is ALL: + classes = [] + remove = None + else: + if isinstance(classes, str): + classes = [classes] + classes = split_classes(*classes) + + remove_index = None + if isinstance(remove, str): + remove = [remove] + for classname in remove or []: + while classname in classes: + remove_index = max(remove_index or 0, classes.index(classname)) + classes.remove(classname) + + insert_index = remove_index if remove_index is not None else len(classes) + if isinstance(add, str): + add = [add] + for classname in add or []: + if classname not in classes: + classes.insert(insert_index, classname) + insert_index += 1 + + return classes + + +def edit_element_t_classes(element, add, remove): + """ + Edit inplace qweb ``t-att-class`` and ``t-attf-class`` attributes of an element, adding and removing the specified classes. + + N.B. adding new classes will not work if neither ``t-att-class`` nor ``t-attf-class`` are present. + + :param etree.ElementBase element: the element to edit. + :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the element. + :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) + from the element. The `ALL` sentinel value can be specified to remove all classes. + + :meta private: exclude from online docs + """ + if isinstance(add, str): + add = [add] + if add: + add = split_classes(*add) + if isinstance(remove, str): + remove = [remove] + if remove and remove is not ALL: + remove = split_classes(*remove) + + if not add and not remove: + return # nothing to do + + for attr in ("t-att-class", "t-attf-class"): + if attr in element.attrib: + value = element.attrib.pop(attr) + + # if we need to remove all classes, just remove or replace the attribute + # with the literal string of classes to add, removing all logic + if remove is ALL: + if not add: + value = None + else: + value = " ".join(add or []) + if attr.startswith("t-att-"): + value = f"'{value}'" + + else: + joined_adds = join_classes(add or []) + # if there's no classes to remove, try to append the new classes in a sane way + if not remove: + if add: + if attr.startswith("t-att-"): + value = f"{value} + ' {joined_adds}'" if value else f"'{joined_adds}'" + else: + value = f"{value} {joined_adds}" + # otherwise use regexes to replace the classes in the attribute + # if there's just one replace, do a string replacement with the new classes + elif len(remove) == 1: + value = re.sub(rf"{BS}{re.escape(remove[0])}{BE}", joined_adds, value) + # else there's more than one class to remove and split at matches, + # then rejoin with new ones at the position of the last removed one. + else: + value = re.split(rf"{BS}(?:{'|'.join(map(re.escape, remove))}){BE}", value) + value = "".join(value[:-1] + [joined_adds] + value[-1:]) + + if value is not None: + element.attrib[attr] = value + + +def edit_element_classes(element, add, remove, is_qweb=False): + """ + Edit inplace the "class" attribute of an element, adding and removing classes. + + :param etree.ElementBase element: the element to edit. + :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the element. + :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) + from the element. The `ALL` sentinel value can be specified to remove all classes. + :param bool is_qweb: if True also edit classes in ``t-att-class`` and ``t-attf-class`` attributes. + Defaults to False. + + :meta private: exclude from online docs + """ + if not is_qweb or not set(element.attrib) & {"t-att-class", "t-attf-class"}: + set_classes(element, edit_classlist(get_classes(element), add, remove)) + if is_qweb: + edit_element_t_classes(element, add, remove) + + +ALL = object() +"""Sentinel object to indicate "all items" in a collection""" + + +def simple_css_selector_to_xpath(selector, prefix="//"): + """ + Convert a basic CSS selector cases to an XPath expression. + + Supports node names, classes, ``>`` and ``,`` combinators. + + :param str selector: the CSS selector to convert. + :param str prefix: the prefix to add to the XPath expression. Defaults to ``//``. + :return: the resulting XPath expression. + :rtype: str + + :meta private: exclude from online docs + """ + separator = prefix + xpath_parts = [] + combinators = "+>,~ " + for selector_part in map(str.strip, re.split(rf"(\s*[{combinators}]\s*)", selector)): + if not selector_part: + separator = "//" + elif selector_part == ">": + separator = "/" + elif selector_part == ",": + separator = "|" + prefix + elif re.search(r"^(?:[a-z](-?\w+)*|[*.])", selector_part, flags=re.I): + element, *classes = selector_part.split(".") + if not element: + element = "*" + class_predicates = [f"[hasclass('{classname}')]" for classname in classes if classname] + xpath_parts += [separator, element + "".join(class_predicates)] + else: + raise NotImplementedError(f"Unsupported CSS selector syntax: {selector}") + + return "".join(xpath_parts) + + +CSS = simple_css_selector_to_xpath + + +def regex_xpath(pattern, attr=None, xpath=None): + """ + Return an XPath expression that matches elements with an attribute value matching the given regex pattern. + + :param str pattern: a regex pattern to match the attribute value against. + :param str | None attr: the attribute to match the pattern against. + If not given, the pattern is matched against the element's text. + :param str | None xpath: an optional XPath expression to further filter the elements to match. + :rtype: str + + :meta private: exclude from online docs + """ + # TODO abt: investigate lxml xpath variables interpolation (xpath.setcontext? registerVariables?) + if "'" in pattern and '"' in pattern: + quoted_pattern = "concat('" + "', \"'\", '".join(pattern.split("'")) + "')" + elif "'" in pattern: + quoted_pattern = '"' + pattern + '"' + else: + quoted_pattern = "'" + pattern + "'" + xpath_pre = xpath or "//*" + attr_or_text = f"@{attr}" if attr is not None else "text()" + return xpath_pre + f"[regex({attr_or_text}, {quoted_pattern})]" + + +def adapt_xpath_for_qweb(xpath): + """ + Adapts a xpath to enable matching on qweb ``t-att(f)-`` attributes. + + :meta private: exclude from online docs + """ + xpath = re.sub(r"\bhas-?class(?=\()", "has-t-class", xpath) + # supposing that there's only one of `class`, `t-att-class`, `t-attf-class`, + # joining all of them with a space and removing trailing whitespace should behave + # similarly to COALESCE, and result in ORing the values for matching + xpath = re.sub( + r"(?<=\()\s*@(? + ... +
+ + after:: + .. code-block:: html + + ... + + :meta private: exclude from online docs + """ + + def __call__(self, element, converter): + parent = element.getparent() + if parent is None: + raise ValueError(f"Cannot pull up contents of xml element with no parent: {element}") + + prev_sibling = element.getprevious() + if prev_sibling is not None: + prev_sibling.tail = ((prev_sibling.tail or "") + (element.text or "")) or None + else: + parent.text = ((parent.text or "") + (element.text or "")) or None + + for child in element: + element.addprevious(child) + + parent.remove(element) + + +class RenameAttribute(ElementOperation): + """ + Rename an attribute. Silently ignores elements that do not have the attribute. + + :param str old_name: the name of the attribute to rename. + :param str new_name: the new name of the attribute. + :param str | None xpath: see :class:`ElementOperation` ``xpath`` parameter. + + :meta private: exclude from online docs + """ + + def __init__(self, old_name, new_name, *, xpath=None): + super().__init__(xpath=xpath) + self.old_name = old_name + self.new_name = new_name + + def __call__(self, element, converter): + rename_map = {self.old_name: self.new_name} + if converter.is_qweb: + rename_map = { + f"{prefix}{old}": f"{prefix}{new}" + for old, new in rename_map.items() + for prefix in ("",) + (("t-att-", "t-attf-") if not old.startswith("t-") else ()) + if f"{prefix}{old}" in element.attrib + } + if rename_map: + # to preserve attributes order, iterate+rename on a copy and reassign (clear+update, bc readonly property) + attrib_before = dict(element.attrib) + element.attrib.clear() + element.attrib.update({rename_map.get(k, k): v for k, v in attrib_before.items()}) + return element + + @property + def xpath(self): + """ + Return an XPath expression that matches elements with the old attribute name. + + :rtype: str + + :meta private: exclude from online docs + """ + return (self._xpath or "//*") + f"[@{self.old_name}]" + + +class RegexReplace(ElementOperation): + """ + Uses `re.sub` to modify an attribute or the text of an element. + + N.B. no checks are made to ensure the attribute to replace is actually present on the elements. + + :param str pattern: the regex pattern to match. + :param str sub: the replacement string. + :param str | None attr: the attribute to replace. If not specified, the text of the element is replaced. + :param str | None xpath: see :class:`ElementOperation` ``xpath`` parameter. + + :meta private: exclude from online docs + """ + + def __init__(self, pattern, sub, attr=None, *, xpath=None): + super().__init__(xpath=xpath) + self.pattern = pattern + self.repl = sub + self.attr = attr + + def __call__(self, element, converter): + if self.attr is None: + # TODO abt: what about tail? + element.text = re.sub(self.pattern, self.repl, element.text or "") + else: + for attr in (self.attr,) + ((f"t-att-{self.attr}", f"t-attf-{self.attr}") if converter.is_qweb else ()): + if attr in element.attrib: + element.attrib[attr] = re.sub(self.pattern, self.repl, element.attrib[attr]) + return element + + @property + def xpath(self): + """ + Return an XPath expression that matches elements with the old attribute name. + + :rtype: str + + :meta private: exclude from online docs + """ + return regex_xpath(self.pattern, self.attr, self._xpath) + + +class RegexReplaceClass(RegexReplace): + """ + Uses `re.sub` to modify the class. + + Basically, same as `RegexReplace`, but with `attr="class"`. + + :meta private: exclude from online docs + """ + + def __init__(self, pattern, sub, attr="class", *, xpath=None): + super().__init__(pattern, sub, attr, xpath=xpath) + + +class EtreeConverter: + """ + Class for converting lxml etree documents, applying a bunch of operations on them. + + :param list[ElementOperation | (str, ElementOperation | list[ElementOperation])] conversions: + the operations to apply to the tree. + Each item in the conversions list must either be an :class:`ElementOperation` that can provide its own XPath, + or a tuple of ``(xpath, operation)`` or ``(xpath, operations)`` with the XPath and an operation + or a list of operations to apply to the nodes matching the XPath. + :param bool is_html: whether the tree is an HTML document. + :para bool is_qweb: whether the tree contains QWeb directives. + If this is enabled, XPaths will be auto-transformed to try to also match ``t-att*`` attributes. + + :meta private: exclude from online docs + """ + + def __init__(self, conversions, *, is_html=False, is_qweb=False): + self._hashable_conversions = self._make_conversions_hashable(conversions) + conversions_hash = hash(self._hashable_conversions) + self._is_html = is_html + self._is_qweb = is_qweb + self.conversions = self._compile_conversions(self._hashable_conversions, self.is_qweb) + self._cache_hash = hash((self.__class__, conversions_hash, is_html, is_qweb)) + + def __hash__(self): + return self._cache_hash + + def __getstate__(self): + state = self.__dict__.copy() + del state["conversions"] + return state + + def __setstate__(self, state): + self.__dict__.update(state) + self.conversions = self._compile_conversions(self._hashable_conversions, self.is_qweb) + + @property + def is_html(self): + """ + Whether the conversions are for HTML documents. + + :meta private: exclude from online docs + """ + return self._is_html + + @property + def is_qweb(self): + """ + Whether the conversions are for QWeb documents. + + :meta private: exclude from online docs + """ + return self._is_qweb + + @classmethod + @lru_cache(maxsize=32) + def _compile_conversions(cls, conversions, is_qweb): + """ + Compile the given conversions to a list of ``(xpath, operations)`` tuples, with pre-compiled XPaths. + + The conversions must be provided as tuples instead of lists to allow for caching. + + :param tuple[ElementOperation | (str, ElementOperation | tuple[ElementOperation, ...]), ...] conversions: + the conversions to compile. + :param bool is_qweb: whether the conversions are for QWeb. + :rtype: list[(etree.XPath, list[ElementOperation])] + """ + + def process_spec(spec): + xpath, ops = None, None + if isinstance(spec, ElementOperation): # single operation with its own XPath + xpath, ops = spec.xpath, [spec] + elif isinstance(spec, tuple) and len(spec) == 2: # (xpath, operation | operations) tuple + xpath, ops_spec = spec + if isinstance(ops_spec, ElementOperation): # single operation + ops = [ops_spec] + elif isinstance(ops_spec, tuple): # multiple operations + ops = list(ops_spec) + + if xpath is None or ops is None: + raise ValueError(f"Invalid conversion specification: {spec!r}") + + if is_qweb: + xpath = adapt_xpath_for_qweb(xpath) + + return etree.XPath(xpath), ops + + return [process_spec(spec) for spec in conversions] + + @classmethod + def _make_conversions_hashable(cls, conversions): + """ + Normalize the given conversions into tuples, so they can be hashed. + + :param list[ElementOperation | (str, ElementOperation | list[ElementOperation])] conversions: + the conversions to make hashable. + :rtype: tuple[ElementOperation | (str, ElementOperation | tuple[ElementOperation, ...]), ...] + """ + return tuple( + (spec[0], tuple(spec[1])) + if isinstance(spec, tuple) and len(spec) == 2 and isinstance(spec[1], list) + else spec + for spec in conversions + ) + + def get_conversions_keywords(self): + """ + Build and return keywords extracted from the compiled conversions. + + Will return a tuple of sets for class names and other keywords (e.g. tags, attrs). + + :rtype: (set[str], set[str]) + + :meta private: exclude from online docs + """ + return extract_xpaths_keywords(xpath for xpath, _ in self.conversions) + + def build_where_clause(self, cr, column): + """ + Build and return an XPath expression that matches all the conversions keywords. + + :param psycopg2.cursor cr: the database cursor. + :param str column: the column in the query to match the keywords against. + :rtype: str + + :meta private: exclude from online docs + """ + classes, other_kwds = self.get_conversions_keywords() + return build_keywords_where_clause(cr, tuple(classes), tuple(other_kwds), column) + + def convert_tree(self, tree): + """ + Convert an etree document inplace with the prepared conversions. + + Returns the converted document and the number of conversion operations applied. + + :param etree.ElementTree tree: the parsed XML or HTML tree to convert. + :rtype: etree.ElementTree, int + + :meta private: exclude from online docs + """ + applied_operations_count = 0 + for xpath, operations in self.conversions: + for element in xpath(tree): + for operation in operations: + if element is None: # previous operations that returned None (i.e. deleted element) + raise ValueError("Matched xml element is not available anymore! Check operations.") + element = operation(element, self) # noqa: PLW2901 + applied_operations_count += 1 + return tree, applied_operations_count + + convert = convert_tree # alias for backward compatibility + + @lru_cache(maxsize=128) # noqa: B019 + def convert_callback(self, content): + """ + A converter method that can be used with ``util.snippets`` ``convert_html_columns`` or ``convert_html_content``. + + Accepts a single argument, the html/xml content to convert, and returns a tuple of (has_changed, converted_content). + + :param str content: the html/xml content to convert. + :rtype: (bool, str) + + :meta private: exclude from online docs + """ # noqa: D401 + if not content: + return False, content + + with ArchEditor(content, self.is_html) as arch_editor: + tree, ops_count = self.convert_tree(arch_editor.tree) + if not ops_count: + return False, content + return True, arch_editor.arch + + def convert_arch(self, arch): + """ + Convert an XML or HTML arch string with the prepared conversions. + + :param str arch: the arch to convert. + :rtype: str + + :meta private: exclude from online docs + """ + return self.convert_callback(arch)[1] + + def convert_file(self, path): + """ + Convert an XML or HTML file inplace. + + :param str path: the path to the XML or HTML file to convert. + :rtype: None + + :meta private: exclude from online docs + """ + file_is_html = os.path.splitext(path)[1].startswith("htm") + if self.is_html != file_is_html: + raise ValueError(f"File {path!r} is not a {'HTML' if self.is_html else 'XML'} file!") + + tree = etree.parse(path, parser=html_utf8_parser if self.is_html else None) + + tree, ops_count = self.convert_tree(tree) + if not ops_count: + logging.info("No conversion operations applied, skipping file: %s", path) + return + + tree.write(path, encoding="utf-8", method="html" if self.is_html else None, xml_declaration=not self.is_html) + + # -- Operations helper methods, useful where operations need some converter-specific info or logic (e.g. is_html) -- + + def element_factory(self, *args, **kwargs): + """ + Create new elements using the correct document type. + + Basically a wrapper for either etree.XML or etree.HTML depending on the type of document loaded. + + :param args: positional arguments to pass to the etree.XML or etree.HTML function. + :param kwargs: keyword arguments to pass to the etree.XML or etree.HTML function. + :return: the created element. + + :meta private: exclude from online docs + """ + return etree.HTML(*args, **kwargs) if self.is_html else etree.XML(*args, **kwargs) + + def build_element(self, tag, classes=None, contents=None, **attributes): + """ + Create a new element with the given tag, classes, contents and attributes. + + Like :meth:`~.element_factory`, can be used by operations to create elements abstracting away the document type. + + :param str tag: the tag of the element to create. + :param typing.Iterable[str] | None classes: the classes to set on the new element. + :param str | None contents: the contents of the new element (i.e. inner text/HTML/XML). + :param dict[str, str] attributes: attributes to set on the new element, provided as keyword arguments. + :return: the created element. + :rtype: etree.ElementBase + + :meta private: exclude from online docs + """ + element = self.element_factory(f"<{tag}>{contents or ''}") + for name, value in attributes.items(): + element.attrib[name] = value + if classes: + set_classes(element, classes) + return element + + def copy_element( + self, + element, + tag=None, + add_classes=None, + remove_classes=None, + copy_attrs=True, + copy_contents=True, + **attributes, + ): + """ + Create a copy of an element, optionally changing the tag, classes, contents and attributes. + + Like :meth:`~.element_factory`, can be used by operations to copy elements abstracting away the document type. + + :param etree.ElementBase element: the element to copy. + :param str | None tag: if specified, overrides the tag of the new element. + :param str | typing.Iterable[str] | None add_classes: if specified, adds the given class(es) to the new element. + :param str | typing.Iterable[str] | ALL | None remove_classes: if specified, removes the given class(es) + from the new element. The `ALL` sentinel value can be specified to remove all classes. + :param bool copy_attrs: if True, copies the attributes of the source element to the new one. Defaults to True. + :param bool copy_contents: if True, copies the contents of the source element to the new one. Defaults to True. + :param dict[str, str] attributes: attributes to set on the new element, provided as keyword arguments. + Will be str merged with the attributes of the source element, overriding the latter. + :return: the new copied element. + :rtype: etree.ElementBase + + :meta private: exclude from online docs + """ + tag = tag or element.tag + contents = innerxml(element, is_html=self.is_html) if copy_contents else None + if copy_attrs: + attributes = {**element.attrib, **attributes} + new_element = self.build_element(tag, contents=contents, **attributes) + edit_element_classes(new_element, add_classes, remove_classes, is_qweb=self.is_qweb) + return new_element + + def adapt_xpath(self, xpath): + """ + Adapts an XPath to match qweb ``t-att(f)-*`` attributes, if ``is_qweb`` is True. + + :meta private: exclude from online docs + """ + return adapt_xpath_for_qweb(xpath) if self.is_qweb else xpath + + +if misc.version_gte("13.0"): + + def convert_views(cr, views_ids, converter): + """ + Convert the specified views xml arch using the provided converter. + + :param psycopg2.cursor cr: the database cursor. + :param typing.Collection[int] views_ids: the ids of the views to convert. + :param EtreeConverter converter: the converter to use. + :rtype: None + + :meta private: exclude from online docs + """ + if converter.is_html: + raise TypeError("Cannot convert xml views with provided ``is_html`` converter %s" % (repr(converter),)) + + _logger.info("Converting %d views/templates using %s", len(views_ids), repr(converter)) + for view_id in views_ids: + with records.edit_view(cr, view_id=view_id, active=None) as tree: + converter.convert_tree(tree) + # TODO abt: maybe notify in the log or report that custom views with noupdate=False were converted? + + def convert_qweb_views(cr, converter): + """ + Convert QWeb views / templates using the provided converter. + + :param psycopg2.cursor cr: the database cursor. + :param EtreeConverter converter: the converter to use. + :rtype: None + + :meta private: exclude from online docs + """ + if not converter.is_qweb: + raise TypeError("Converter for xml views must be ``is_qweb``, got %s" % (repr(converter),)) + + # views to convert must have `website_id` set and not come from standard modules + standard_modules = set(get_modules()) - {"studio_customization", "__export__", "__cloc_exclude__"} + converter_where = converter.build_where_clause(cr, "v.arch_db") + + # Search for custom/cow'ed views (they have no external ID)... but also + # search for views with external ID that have a related COW'ed view. Indeed, + # when updating a generic view after this script, the archs are compared to + # know if the related COW'ed views must be updated too or not: if we only + # convert COW'ed views they won't get the generic view update as they will be + # judged different from them (user customization) because of the changes + # that were made. + # E.g. + # - In 15.0, install website_sale + # - Enable eCommerce categories: a COW'ed view is created to enable the + # feature (it leaves the generic disabled and creates an exact copy but + # enabled) + # - Migrate to 16.0: you expect your enabled COW'ed view to get the new 16.0 + # version of eCommerce categories... but if the COW'ed view was converted + # while the generic was not, they won't be considered the same + # anymore and only the generic view will get the 16.0 update. + cr.execute( + """ + WITH keys AS ( + SELECT key + FROM ir_ui_view + GROUP BY key + HAVING COUNT(*) > 1 + ) + SELECT v.id + FROM ir_ui_view v + LEFT JOIN ir_model_data imd + ON imd.model = 'ir.ui.view' + AND imd.module IN %%s + AND imd.res_id = v.id + LEFT JOIN keys + ON v.key = keys.key + WHERE v.type = 'qweb' + AND (%s) + AND ( + imd.id IS NULL + OR ( + keys.key IS NOT NULL + AND imd.noupdate = FALSE + ) + ) + """ + % converter_where, + [tuple(standard_modules)], + ) + views_ids = [view_id for (view_id,) in cr.fetchall()] + if views_ids: + convert_views(cr, views_ids, converter) + + def convert_html_fields(cr, converter, verbose=False): + """ + Convert all html fields data in the database using the provided converter. + + :param psycopg2.cursor cr: the database cursor. + :param EtreeConverter converter: the converter to use. + :param bool verbose: whether to print stats about the conversion. + :rtype: None + + :meta private: exclude from online docs + """ + if verbose: + _logger.info("Converting html fields data using %s", repr(converter)) + + matched_count = 0 + converted_count = 0 + + html_fields = list(snippets.html_fields(cr)) + for table, columns in misc.log_progress(html_fields, _logger, "tables", log_hundred_percent=True): + if table not in ("mail_message", "mail_activity"): + extra_where = " OR ".join( + "(%s)" % converter.build_where_clause(cr, pg.get_value_or_en_translation(cr, table, column)) + for column in columns + ) + # TODO abt: adapt to refactor, maybe make snippets compat w/ converter, instead of adapting? + matched, converted = snippets.convert_html_columns( + cr, table, columns, converter.convert_callback, extra_where=extra_where + ) + matched_count += matched + converted_count += converted + + if verbose: + if matched_count: + _logger.info("Converted %d/%d matched html fields values", converted_count, matched_count) + else: + _logger.info("Did not match any html fields values to convert") + +else: + + def convert_views(*args, **kwargs): + raise NotImplementedError( + "This helper function is only available for Odoo 13.0 and above: %s", convert_views.__qualname__ + ) + + def convert_qweb_views(*args, **kwargs): + raise NotImplementedError( + "This helper function is only available for Odoo 13.0 and above: %s", convert_qweb_views.__qualname__ + ) + + def convert_html_fields(*args, **kwargs): + raise NotImplementedError( + "This helper function is only available for Odoo 13.0 and above: %s", convert_html_fields.__qualname__ + ) diff --git a/src/util/views/records.py b/src/util/views/records.py new file mode 100644 index 000000000..70869a2f1 --- /dev/null +++ b/src/util/views/records.py @@ -0,0 +1,276 @@ +# -*- coding: utf-8 -*- +import logging +from contextlib import contextmanager + +# python3 shims +try: + unicode # noqa: B018 +except NameError: + unicode = str + +import lxml +from psycopg2.extras import Json + +try: + from odoo.tools.translate import xml_translate +except ImportError: + xml_translate = lambda callback, value: value + +from ..pg import column_exists, column_type +from ..report import add_to_migration_reports + +_logger = logging.getLogger(__name__) + + +__all__ = [ + "remove_view", + "edit_view", + "add_view", +] + + +def remove_view(cr, xml_id=None, view_id=None, silent=False, key=None): + """ + Remove a view and all its descendants. + + This function recursively deletes the given view and its inherited views, as long as + they are part of a module. It will fail as soon as a custom view exists anywhere in + the hierarchy. It also removes multi-website COWed views. + + :param str xml_id: optional, the xml_id of the view to remove + :param int view_id: optional, the ID of the view to remove + :param bool silent: whether to show in the logs disabled custom views + :param str or None key: key used to detect multi-website COWed views, if `None` then + set to `xml_id` if provided, otherwise set to the xml_id + referencing the view with ID `view_id` if any + + .. warning:: + Either `xml_id` or `view_id` must be set. Specifying both will raise an error. + """ + from ..records import ref, remove_records + + assert bool(xml_id) ^ bool(view_id) + if xml_id: + view_id = ref(cr, xml_id) + if view_id: + module, _, name = xml_id.partition(".") + cr.execute("SELECT model FROM ir_model_data WHERE module=%s AND name=%s", [module, name]) + + [model] = cr.fetchone() + if model != "ir.ui.view": + raise ValueError("%r should point to a 'ir.ui.view', not a %r" % (xml_id, model)) + else: + # search matching xmlid for logging or renaming of custom views + xml_id = "?" + if not key: + cr.execute("SELECT module, name FROM ir_model_data WHERE model='ir.ui.view' AND res_id=%s", [view_id]) + if cr.rowcount: + xml_id = "%s.%s" % cr.fetchone() + + # From given or determined xml_id, the views duplicated in a multi-website + # context are to be found and removed. + if xml_id != "?" and column_exists(cr, "ir_ui_view", "key"): + cr.execute("SELECT id FROM ir_ui_view WHERE key = %s AND id != %s", [xml_id, view_id]) + for [v_id] in cr.fetchall(): + remove_view(cr, view_id=v_id, silent=silent, key=xml_id) + + if not view_id: + return + + cr.execute( + """ + SELECT v.id, x.module || '.' || x.name, v.name + FROM ir_ui_view v LEFT JOIN + ir_model_data x ON (v.id = x.res_id AND x.model = 'ir.ui.view' AND x.module !~ '^_') + WHERE v.inherit_id = %s; + """, + [view_id], + ) + for child_id, child_xml_id, child_name in cr.fetchall(): + if child_xml_id: + if not silent: + _logger.info( + "remove deprecated built-in view %s (ID %s) as parent view %s (ID %s) is going to be removed", + child_xml_id, + child_id, + xml_id, + view_id, + ) + remove_view(cr, child_xml_id, silent=True) + else: + if not silent: + _logger.warning( + "deactivate deprecated custom view with ID %s as parent view %s (ID %s) is going to be removed", + child_id, + xml_id, + view_id, + ) + disable_view_query = """ + UPDATE ir_ui_view + SET name = (name || ' - old view, inherited from ' || %%s), + inherit_id = NULL + %s + WHERE id = %%s + """ + # In 8.0, disabling requires setting mode to 'primary' + extra_set_sql = "" + if column_exists(cr, "ir_ui_view", "mode"): + extra_set_sql = ", mode = 'primary' " + + # Column was not present in v7 and it's older version + if column_exists(cr, "ir_ui_view", "active"): + extra_set_sql += ", active = false " + + disable_view_query = disable_view_query % extra_set_sql + cr.execute(disable_view_query, (key or xml_id, child_id)) + add_to_migration_reports( + {"id": child_id, "name": child_name}, + "Disabled views", + ) + if not silent: + _logger.info("remove deprecated %s view %s (ID %s)", key and "COWed" or "built-in", key or xml_id, view_id) + + remove_records(cr, "ir.ui.view", [view_id]) + + +@contextmanager +def edit_view(cr, xmlid=None, view_id=None, skip_if_not_noupdate=True, active=True): + """ + Context manager to edit a view's arch. + + This function returns a context manager that may yield a parsed arch of a view as an + `etree Element `_. Any changes done + in the returned object will be written back to the database upon exit of the context + manager, updating also the translated versions of the arch. Since the function may not + yield, use :func:`~odoo.upgrade.util.misc.skippable_cm` to avoid errors. + + .. code-block:: python + + with util.skippable_cm(), util.edit_view(cr, "xml.id") as arch: + arch.attrib["string"] = "My Form" + + To select the target view to edit use either `xmlid` or `view_id`, not both. + + When the view is identified by `view_id`, the arch is always yielded if the view + exists, with disregard to any `noupdate` flag it may have associated. When `xmlid` is + set, if the view `noupdate` flag is `True` then the arch will not be yielded *unless* + `skip_if_not_noupdate` is set to `False`. If `noupdate` is `False`, the view will be + yielded for edit. + + If the `active` argument is not `None`, the `active` flag of the view will be set + accordingly. + + .. warning:: + The default value of `active` is `True`, therefore views are always *activated* by + default. To avoid inadvertently activating views, pass `None` as `active` parameter. + + :param str xmlid: optional, xml_id of the view edit + :param int view_id: optional, ID of the view to edit + :param bool skip_if_not_noupdate: whether to force the edit of views requested via + `xmlid` parameter even if they are flagged as + `noupdate=True`, ignored if `view_id` is set + :param bool or None active: active flag value to set, nothing is set when `None` + :return: a context manager that yields the parsed arch, upon exit the context manager + writes back the changes. + """ + assert bool(xmlid) ^ bool(view_id), "You Must specify either xmlid or view_id" + noupdate = True + if xmlid: + if "." not in xmlid: + raise ValueError("Please use fully qualified name .") + + module, _, name = xmlid.partition(".") + cr.execute( + """ + SELECT res_id, noupdate + FROM ir_model_data + WHERE module = %s + AND name = %s + """, + [module, name], + ) + data = cr.fetchone() + if data: + view_id, noupdate = data + + if view_id and not (skip_if_not_noupdate and not noupdate): + arch_col = "arch_db" if column_exists(cr, "ir_ui_view", "arch_db") else "arch" + jsonb_column = column_type(cr, "ir_ui_view", arch_col) == "jsonb" + cr.execute( + """ + SELECT {arch} + FROM ir_ui_view + WHERE id=%s + """.format( + arch=arch_col, + ), + [view_id], + ) + [arch] = cr.fetchone() or [None] + if arch: + + def parse(arch): + arch = arch.encode("utf-8") if isinstance(arch, unicode) else arch + return lxml.etree.fromstring(arch.replace(b" \n", b"\n").strip()) + + if jsonb_column: + + def get_trans_terms(value): + terms = [] + xml_translate(terms.append, value) + return terms + + translation_terms = {lang: get_trans_terms(value) for lang, value in arch.items()} + arch_etree = parse(arch["en_US"]) + yield arch_etree + new_arch = lxml.etree.tostring(arch_etree, encoding="unicode") + terms_en = translation_terms["en_US"] + arch_column_value = Json( + { + lang: xml_translate(dict(zip(terms_en, terms)).get, new_arch) + for lang, terms in translation_terms.items() + } + ) + else: + arch_etree = parse(arch) + yield arch_etree + arch_column_value = lxml.etree.tostring(arch_etree, encoding="unicode") + + set_active = ", active={}".format(bool(active)) if active is not None else "" + cr.execute( + "UPDATE ir_ui_view SET {arch}=%s{set_active} WHERE id=%s".format(arch=arch_col, set_active=set_active), + [arch_column_value, view_id], + ) + + +def add_view(cr, name, model, view_type, arch_db, inherit_xml_id=None, priority=16): + from ..records import ref + + inherit_id = None + if inherit_xml_id: + inherit_id = ref(cr, inherit_xml_id) + if not inherit_id: + raise ValueError( + "Unable to add view '%s' because its inherited view '%s' cannot be found!" % (name, inherit_xml_id) + ) + arch_col = "arch_db" if column_exists(cr, "ir_ui_view", "arch_db") else "arch" + jsonb_column = column_type(cr, "ir_ui_view", arch_col) == "jsonb" + arch_column_value = Json({"en_US": arch_db}) if jsonb_column else arch_db + cr.execute( + """ + INSERT INTO ir_ui_view(name, "type", model, inherit_id, mode, active, priority, %s) + VALUES(%%(name)s, %%(view_type)s, %%(model)s, %%(inherit_id)s, %%(mode)s, 't', %%(priority)s, %%(arch_db)s) + RETURNING id + """ + % arch_col, + { + "name": name, + "view_type": view_type, + "model": model, + "inherit_id": inherit_id, + "mode": "extension" if inherit_id else "primary", + "priority": priority, + "arch_db": arch_column_value, + }, + ) + return cr.fetchone()[0] diff --git a/tools/compile23.py b/tools/compile23.py index b5bc369e4..9594b4a1b 100755 --- a/tools/compile23.py +++ b/tools/compile23.py @@ -13,7 +13,8 @@ "src/testing.py", "src/util/jinja_to_qweb.py", "src/util/snippets.py", - "src/util/convert_bootstrap.py", + "src/util/views/convert.py", + "src/util/views/bootstrap.py", "src/*/tests/*.py", "src/*/17.0.*/*.py", ]