Merge branch 'master' of https://github.com/datajoint/datajoint-python into fix-mym-#86

Author: A-Baji

CHANGELOG.md

@@ -1,6 +1,6 @@
 ## Release notes
 
-### 0.13.3 -- TBD
+### 0.13.3 -- Feb 9, 2022
 * Bugfix - Fix error in listing ancestors, descendants with part tables.
 * Bugfix - Fix Python 3.10 compatibility (#983) PR #972
 * Bugfix - Allow renaming non-conforming attributes in proj (#982) PR #972
@@ -16,6 +16,8 @@
 * Bugfix - Fix count for left-joined `QueryExpressions` (#951) PR #966
 * Bugfix - Fix assertion error when performing a union into a join (#930) PR #967
 * Update `~jobs.error_stack` from blob to mediumblob to allow error stacks >64kB in jobs (#984) PR #986
+* Bugfix - Fix error when performing a union on multiple tables (#926) PR #964
+* Add - Allow optional keyword arguments for `make()` in `populate()` PR #971
 
 ### 0.13.2 -- May 7, 2021
 * Update `setuptools_certificate` dependency to new name `otumat`

datajoint/autopopulate.py

@@ -128,10 +128,11 @@ def _jobs_to_do(self, restrictions):
 
     def populate(self, *restrictions, suppress_errors=False, return_exception_objects=False,
                  reserve_jobs=False, order="original", limit=None, max_calls=None,
-                 display_progress=False, processes=1):
+                 display_progress=False, processes=1, make_kwargs=None):
         """
-        table.populate() calls table.make(key) for every primary key in self.key_source
-        for which there is not already a tuple in table.
+        ``table.populate()`` calls ``table.make(key)`` for every primary key in
+        ``self.key_source`` for which there is not already a tuple in table.
+
         :param restrictions: a list of restrictions each restrict
             (table.key_source - target.proj())
         :param suppress_errors: if True, do not terminate execution.
@@ -143,6 +144,10 @@ def populate(self, *restrictions, suppress_errors=False, return_exception_object
         :param display_progress: if True, report progress_bar
         :param processes: number of processes to use. When set to a large number, then
             uses as many as CPU cores
+        :param make_kwargs: Keyword arguments which do not affect the result of computation
+            to be passed down to each ``make()`` call. Computation arguments should be
+            specified within the pipeline e.g. using a `dj.Lookup` table.
+        :type make_kwargs: dict, optional
         """
         if self.connection.in_transaction:
             raise DataJointError('Populate cannot be called during a transaction.')
@@ -176,7 +181,8 @@ def handler(signum, frame):
         error_list = []
         populate_kwargs = dict(
             suppress_errors=suppress_errors,
-            return_exception_objects=return_exception_objects)
+            return_exception_objects=return_exception_objects,
+            make_kwargs=make_kwargs)
 
         if processes == 1:
             for key in tqdm(keys, desc=self.__class__.__name__) if display_progress else keys:
@@ -207,7 +213,7 @@ def handler(signum, frame):
         if suppress_errors:
             return error_list
 
-    def _populate1(self, key, jobs, suppress_errors, return_exception_objects):
+    def _populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs=None):
         """
         populates table for one source key, calling self.make inside a transaction.
         :param jobs: the jobs table or None if not reserve_jobs
@@ -228,7 +234,7 @@ def _populate1(self, key, jobs, suppress_errors, return_exception_objects):
                 logger.info('Populating: ' + str(key))
                 self.__class__._allow_insert = True
                 try:
-                    make(dict(key))
+                    make(dict(key), **(make_kwargs or {}))
                 except (KeyboardInterrupt, SystemExit, Exception) as error:
                     try:
                         self.connection.cancel_transaction()

datajoint/expression.py

@@ -102,7 +102,7 @@ def from_clause(self):
         return clause
 
     def where_clause(self):
-        return '' if not self.restriction else ' WHERE(%s)' % ')AND('.join(
+        return '' if not self.restriction else ' WHERE (%s)' % ')AND('.join(
             str(s) for s in self.restriction)
 
     def make_sql(self, fields=None):
@@ -606,6 +606,8 @@ class Union(QueryExpression):
     """
     Union is the private DataJoint class that implements the union operator.
     """
+    __count = count()
+
     @classmethod
     def create(cls, arg1, arg2):
         if inspect.isclass(arg2) and issubclass(arg2, QueryExpression):
@@ -632,9 +634,11 @@ def make_sql(self):
         if not arg1.heading.secondary_attributes and not arg2.heading.secondary_attributes:
             # no secondary attributes: use UNION DISTINCT
             fields = arg1.primary_key
-            return "({sql1}) UNION ({sql2})".format(
-                sql1=arg1.make_sql(fields),
-                sql2=arg2.make_sql(fields))
+            return ("SELECT * FROM (({sql1}) UNION ({sql2})) as `_u{alias}`".format(
+                sql1=arg1.make_sql() if isinstance(arg1, Union) else arg1.make_sql(fields),
+                sql2=arg2.make_sql() if isinstance(arg2, Union) else arg2.make_sql(fields),
+                alias=next(self.__count)
+                ))
         # with secondary attributes, use union of left join with antijoin
         fields = self.heading.names
         sql1 = arg1.join(arg2, left=True).make_sql(fields)

docs-parts/intro/Releases_lang1.rst

@@ -1,4 +1,4 @@
-0.13.3 -- TBD
+0.13.3 -- Feb 9, 2022
 ----------------------
 * Bugfix - Fix error in listing ancestors, descendants with part tables.
 * Bugfix - Fix Python 3.10 compatibility (#983) PR #972
@@ -15,6 +15,8 @@
 * Bugfix - Fix assertion error when performing a union into a join (#930) PR #967
 * Bugfix - Fix regression issue with `DISTINCT` clause and `GROUP_BY` (#914) PR #963
 * Update `~jobs.error_stack` from blob to mediumblob to allow error stacks >64kB in jobs (#984) PR #986
+* Bugfix - Fix error when performing a union on multiple tables (#926) PR #964
+* Add - Allow optional keyword arguments for `make()` in `populate()` PR #971
 
 0.13.2 -- May 7, 2021
 ----------------------

tests/test_relational_operand.py

@@ -516,3 +516,13 @@ def test_joins_with_aggregation():
         session_dates = ((SessionDateA * (subj_query & 'date_trained<"2020-12-21"')) &
                          'session_date<date_trained')
         assert len(session_dates) == 1
+
+    @staticmethod
+    def test_union_multiple():
+        # https://github.com/datajoint/datajoint-python/issues/926
+        q1 = IJ & dict(j=2)
+        q2 = (IJ & dict(j=2, i=0)) + (IJ & dict(j=2, i=1)) + (IJ & dict(j=2, i=2))
+        x = set(zip(*q1.fetch('i', 'j')))
+        y = set(zip(*q2.fetch('i', 'j')))
+        assert x == y
+        assert q1.fetch(as_dict=True) == q2.fetch(as_dict=True)