|
| 1 | +""" methods for generating SQL WHERE clauses from datajoint restriction conditions """ |
| 2 | + |
| 3 | +import inspect |
| 4 | +import collections |
| 5 | +import re |
| 6 | +import uuid |
| 7 | +import datetime |
| 8 | +import decimal |
| 9 | +import numpy |
| 10 | +import pandas |
| 11 | +from .errors import DataJointError |
| 12 | + |
| 13 | + |
| 14 | +class PromiscuousOperand: |
| 15 | + """ |
| 16 | + A container for an operand to ignore join compatibility |
| 17 | + """ |
| 18 | + def __init__(self, operand): |
| 19 | + self.operand = operand |
| 20 | + |
| 21 | + |
| 22 | +class AndList(list): |
| 23 | + """ |
| 24 | + A list of conditions to by applied to a query expression by logical conjunction: the conditions are AND-ed. |
| 25 | + All other collections (lists, sets, other entity sets, etc) are applied by logical disjunction (OR). |
| 26 | +
|
| 27 | + Example: |
| 28 | + expr2 = expr & dj.AndList((cond1, cond2, cond3)) |
| 29 | + is equivalent to |
| 30 | + expr2 = expr & cond1 & cond2 & cond3 |
| 31 | + """ |
| 32 | + def append(self, restriction): |
| 33 | + if isinstance(restriction, AndList): |
| 34 | + # extend to reduce nesting |
| 35 | + self.extend(restriction) |
| 36 | + else: |
| 37 | + super().append(restriction) |
| 38 | + |
| 39 | + |
| 40 | +class Not: |
| 41 | + """ invert restriction """ |
| 42 | + def __init__(self, restriction): |
| 43 | + self.restriction = restriction |
| 44 | + |
| 45 | + |
| 46 | +def assert_join_compatibility(expr1, expr2): |
| 47 | + """ |
| 48 | + Determine if expressions expr1 and expr2 are join-compatible. To be join-compatible, the matching attributes |
| 49 | + in the two expressions must be in the primary key of one or the other expression. |
| 50 | + Raises an exception if not compatible. |
| 51 | + :param expr1: A QueryExpression object |
| 52 | + :param expr2: A QueryExpression object |
| 53 | + """ |
| 54 | + from .expression import QueryExpression, U |
| 55 | + |
| 56 | + for rel in (expr1, expr2): |
| 57 | + if not isinstance(rel, (U, QueryExpression)): |
| 58 | + raise DataJointError('Object %r is not a QueryExpression and cannot be joined.' % rel) |
| 59 | + if not isinstance(expr1, U) and not isinstance(expr2, U): # dj.U is always compatible |
| 60 | + try: |
| 61 | + raise DataJointError("Cannot join query expressions on dependent attribute `%s`" % next(r for r in set( |
| 62 | + expr1.heading.secondary_attributes).intersection(expr2.heading.secondary_attributes))) |
| 63 | + except StopIteration: |
| 64 | + pass |
| 65 | + |
| 66 | + |
| 67 | +def make_condition(query_expression, condition, columns): |
| 68 | + """ |
| 69 | + Translate the input condition into the equivalent SQL condition (a string) |
| 70 | + :param query_expression: a dj.QueryExpression object to apply condition |
| 71 | + :param condition: any valid restriction object. |
| 72 | + :param columns: a set passed by reference to collect all column names used in the condition. |
| 73 | + :return: an SQL condition string or a boolean value. |
| 74 | + """ |
| 75 | + from .expression import QueryExpression, Aggregation, U |
| 76 | + |
| 77 | + def prep_value(k, v): |
| 78 | + """prepare value v for inclusion as a string in an SQL condition""" |
| 79 | + if query_expression.heading[k].uuid: |
| 80 | + if not isinstance(v, uuid.UUID): |
| 81 | + try: |
| 82 | + v = uuid.UUID(v) |
| 83 | + except (AttributeError, ValueError): |
| 84 | + raise DataJointError('Badly formed UUID {v} in restriction by `{k}`'.format(k=k, v=v)) from None |
| 85 | + return "X'%s'" % v.bytes.hex() |
| 86 | + if isinstance(v, (datetime.date, datetime.datetime, datetime.time, decimal.Decimal)): |
| 87 | + return '"%s"' % v |
| 88 | + if isinstance(v, str): |
| 89 | + return '"%s"' % v.replace('%', '%%') |
| 90 | + return '%r' % v |
| 91 | + |
| 92 | + negate = False |
| 93 | + while isinstance(condition, Not): |
| 94 | + negate = not negate |
| 95 | + condition = condition.restriction |
| 96 | + template = "NOT (%s)" if negate else "%s" |
| 97 | + |
| 98 | + # restrict by string |
| 99 | + if isinstance(condition, str): |
| 100 | + columns.update(extract_column_names(condition)) |
| 101 | + return template % condition.strip().replace("%", "%%") # escape % in strings, see issue #376 |
| 102 | + |
| 103 | + # restrict by AndList |
| 104 | + if isinstance(condition, AndList): |
| 105 | + # omit all conditions that evaluate to True |
| 106 | + items = [item for item in (make_condition(query_expression, cond, columns) for cond in condition) |
| 107 | + if item is not True] |
| 108 | + if any(item is False for item in items): |
| 109 | + return negate # if any item is False, the whole thing is False |
| 110 | + if not items: |
| 111 | + return not negate # and empty AndList is True |
| 112 | + return template % ('(' + ') AND ('.join(items) + ')') |
| 113 | + |
| 114 | + # restriction by dj.U evaluates to True |
| 115 | + if isinstance(condition, U): |
| 116 | + return not negate |
| 117 | + |
| 118 | + # restrict by boolean |
| 119 | + if isinstance(condition, bool): |
| 120 | + return negate != condition |
| 121 | + |
| 122 | + # restrict by a mapping such as a dict -- convert to an AndList of string equality conditions |
| 123 | + if isinstance(condition, collections.abc.Mapping): |
| 124 | + common_attributes = set(condition).intersection(query_expression.heading.names) |
| 125 | + if not common_attributes: |
| 126 | + return not negate # no matching attributes -> evaluates to True |
| 127 | + columns.update(common_attributes) |
| 128 | + return template % ('(' + ') AND ('.join( |
| 129 | + '`%s`=%s' % (k, prep_value(k, condition[k])) for k in common_attributes) + ')') |
| 130 | + |
| 131 | + # restrict by a numpy record -- convert to an AndList of string equality conditions |
| 132 | + if isinstance(condition, numpy.void): |
| 133 | + common_attributes = set(condition.dtype.fields).intersection(query_expression.heading.names) |
| 134 | + if not common_attributes: |
| 135 | + return not negate # no matching attributes -> evaluate to True |
| 136 | + columns.update(common_attributes) |
| 137 | + return template % ('(' + ') AND ('.join( |
| 138 | + '`%s`=%s' % (k, prep_value(k, condition[k])) for k in common_attributes) + ')') |
| 139 | + |
| 140 | + # restrict by a QueryExpression subclass -- trigger instantiation and move on |
| 141 | + if inspect.isclass(condition) and issubclass(condition, QueryExpression): |
| 142 | + condition = condition() |
| 143 | + |
| 144 | + # restrict by another expression (aka semijoin and antijoin) |
| 145 | + check_compatibility = True |
| 146 | + if isinstance(condition, PromiscuousOperand): |
| 147 | + condition = condition.operand |
| 148 | + check_compatibility = False |
| 149 | + |
| 150 | + if isinstance(condition, QueryExpression): |
| 151 | + if check_compatibility: |
| 152 | + assert_join_compatibility(query_expression, condition) |
| 153 | + common_attributes = [q for q in condition.heading.names if q in query_expression.heading.names] |
| 154 | + columns.update(common_attributes) |
| 155 | + if isinstance(condition, Aggregation): |
| 156 | + condition = condition.make_subquery() |
| 157 | + return ( |
| 158 | + # without common attributes, any non-empty set matches everything |
| 159 | + (not negate if condition else negate) if not common_attributes |
| 160 | + else '({fields}) {not_}in ({subquery})'.format( |
| 161 | + fields='`' + '`,`'.join(common_attributes) + '`', |
| 162 | + not_="not " if negate else "", |
| 163 | + subquery=condition.make_sql(common_attributes))) |
| 164 | + |
| 165 | + # restrict by pandas.DataFrames |
| 166 | + if isinstance(condition, pandas.DataFrame): |
| 167 | + condition = condition.to_records() # convert to numpy.recarray and move on |
| 168 | + |
| 169 | + # if iterable (but not a string, a QueryExpression, or an AndList), treat as an OrList |
| 170 | + try: |
| 171 | + or_list = [make_condition(query_expression, q, columns) for q in condition] |
| 172 | + except TypeError: |
| 173 | + raise DataJointError('Invalid restriction type %r' % condition) |
| 174 | + else: |
| 175 | + or_list = [item for item in or_list if item is not False] # ignore all False conditions |
| 176 | + if any(item is True for item in or_list): # if any item is True, the whole thing is True |
| 177 | + return not negate |
| 178 | + return template % ('(%s)' % ' OR '.join(or_list)) if or_list else negate # an empty or list is False |
| 179 | + |
| 180 | + |
| 181 | +def extract_column_names(sql_expression): |
| 182 | + """ |
| 183 | + extract all presumed column names from an sql expression such as the WHERE clause, for example. |
| 184 | + :param sql_expression: a string containing an SQL expression |
| 185 | + :return: set of extracted column names |
| 186 | + This may be MySQL-specific for now. |
| 187 | + """ |
| 188 | + assert isinstance(sql_expression, str) |
| 189 | + result = set() |
| 190 | + s = sql_expression # for terseness |
| 191 | + # remove escaped quotes |
| 192 | + s = re.sub(r'(\\\")|(\\\')', '', s) |
| 193 | + # remove quoted text |
| 194 | + s = re.sub(r"'[^']*'", "", s) |
| 195 | + s = re.sub(r'"[^"]*"', '', s) |
| 196 | + # find all tokens in back quotes and remove them |
| 197 | + result.update(re.findall(r"`([a-z][a-z_0-9]*)`", s)) |
| 198 | + s = re.sub(r"`[a-z][a-z_0-9]*`", '', s) |
| 199 | + # remove space before parentheses |
| 200 | + s = re.sub(r"\s*\(", "(", s) |
| 201 | + # remove tokens followed by ( since they must be functions |
| 202 | + s = re.sub(r"(\b[a-z][a-z_0-9]*)\(", "(", s) |
| 203 | + remaining_tokens = set(re.findall(r"\b[a-z][a-z_0-9]*\b", s)) |
| 204 | + # update result removing reserved words |
| 205 | + result.update(remaining_tokens - {"is", "in", "between", "like", "and", "or", "null", "not"}) |
| 206 | + return result |
0 commit comments