Merge branch 'master' into key_populate

Author: dimitri-yatsenko

CHANGELOG.md

@@ -1,6 +1,6 @@
 ## Release notes
 
-### 0.13.3 -- TBD
+### 0.13.3 -- Feb 9, 2022
 * Bugfix - Fix error in listing ancestors, descendants with part tables.
 * Bugfix - Fix Python 3.10 compatibility (#983) PR #972
 * Bugfix - Allow renaming non-conforming attributes in proj (#982) PR #972
@@ -15,6 +15,9 @@
 * Bugfix - Fix sql code generation to comply with sql mode `ONLY_FULL_GROUP_BY` (#916) PR #965
 * Bugfix - Fix count for left-joined `QueryExpressions` (#951) PR #966
 * Bugfix - Fix assertion error when performing a union into a join (#930) PR #967
+* Update `~jobs.error_stack` from blob to mediumblob to allow error stacks >64kB in jobs (#984) PR #986
+* Bugfix - Fix error when performing a union on multiple tables (#926) PR #964
+* Add - Allow optional keyword arguments for `make()` in `populate()` PR #971
 
 ### 0.13.2 -- May 7, 2021
 * Update `setuptools_certificate` dependency to new name `otumat`
@@ -248,7 +251,7 @@ Documentation and tutorials available at https://docs.datajoint.io and https://t
 * ERD() no longer text the context argument.
 * ERD.draw() now takes an optional context argument.  By default uses the caller's locals.
 
-### 0.3.2.   
+### 0.3.2.
 * Fixed issue #223:  `insert` can insert relations without fetching.
 * ERD() now takes the `context` argument, which specifies in which context to look for classes. The default is taken from the argument (schema or relation).
 * ERD.draw() no longer has the `prefix` argument: class names are shown as found in the context.

datajoint/autopopulate.py

@@ -129,10 +129,11 @@ def _jobs_to_do(self, restrictions):
     def populate(self, *restrictions, keys=None, suppress_errors=False,
                  return_exception_objects=False,
                  reserve_jobs=False, order="original", limit=None, max_calls=None,
-                 display_progress=False, processes=1):
+                 display_progress=False, processes=1, make_kwargs=None):
         """
-        table.populate() calls table.make(key) for every primary key in self.key_source
-        for which there is not already a tuple in table.
+        ``table.populate()`` calls ``table.make(key)`` for every primary key in
+        ``self.key_source`` for which there is not already a tuple in table.
+
         :param restrictions: a list of restrictions each restrict
             (table.key_source - target.proj())
         :param keys: The list of keys (dicts) to send to self.make().
@@ -146,6 +147,10 @@ def populate(self, *restrictions, keys=None, suppress_errors=False,
         :param display_progress: if True, report progress_bar
         :param processes: number of processes to use. When set to a large number, then
             uses as many as CPU cores
+        :param make_kwargs: Keyword arguments which do not affect the result of computation
+            to be passed down to each ``make()`` call. Computation arguments should be
+            specified within the pipeline e.g. using a `dj.Lookup` table.
+        :type make_kwargs: dict, optional
         """
         if self.connection.in_transaction:
             raise DataJointError('Populate cannot be called during a transaction.')
@@ -181,7 +186,8 @@ def handler(signum, frame):
         error_list = []
         populate_kwargs = dict(
             suppress_errors=suppress_errors,
-            return_exception_objects=return_exception_objects)
+            return_exception_objects=return_exception_objects,
+            make_kwargs=make_kwargs)
 
         if processes == 1:
             for key in tqdm(keys, desc=self.__class__.__name__) if display_progress else keys:
@@ -212,7 +218,7 @@ def handler(signum, frame):
         if suppress_errors:
             return error_list
 
-    def _populate1(self, key, jobs, suppress_errors, return_exception_objects):
+    def _populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs=None):
         """
         populates table for one source key, calling self.make inside a transaction.
         :param jobs: the jobs table or None if not reserve_jobs
@@ -234,7 +240,7 @@ def _populate1(self, key, jobs, suppress_errors, return_exception_objects):
                 logger.info('Populating: ' + str(key))
                 self.__class__._allow_insert = True
                 try:
-                    make(dict(key))
+                    make(dict(key), **(make_kwargs or {}))
                 except (KeyboardInterrupt, SystemExit, Exception) as error:
                     try:
                         self.connection.cancel_transaction()

datajoint/dependencies.py

@@ -26,8 +26,6 @@ def unite_master_parts(lst):
                     # move from the ith position to the (j+1)th position
                     lst[j+1:i+1] = [name] + lst[j+1:i]
                     break
-            else:
-                raise DataJointError("Found a part table {name} without its master table.".format(name=name))
     return lst
 
 

datajoint/expression.py

@@ -102,7 +102,7 @@ def from_clause(self):
         return clause
 
     def where_clause(self):
-        return '' if not self.restriction else ' WHERE(%s)' % ')AND('.join(
+        return '' if not self.restriction else ' WHERE (%s)' % ')AND('.join(
             str(s) for s in self.restriction)
 
     def make_sql(self, fields=None):
@@ -606,6 +606,8 @@ class Union(QueryExpression):
     """
     Union is the private DataJoint class that implements the union operator.
     """
+    __count = count()
+
     @classmethod
     def create(cls, arg1, arg2):
         if inspect.isclass(arg2) and issubclass(arg2, QueryExpression):
@@ -632,9 +634,11 @@ def make_sql(self):
         if not arg1.heading.secondary_attributes and not arg2.heading.secondary_attributes:
             # no secondary attributes: use UNION DISTINCT
             fields = arg1.primary_key
-            return "({sql1}) UNION ({sql2})".format(
-                sql1=arg1.make_sql(fields),
-                sql2=arg2.make_sql(fields))
+            return ("SELECT * FROM (({sql1}) UNION ({sql2})) as `_u{alias}`".format(
+                sql1=arg1.make_sql() if isinstance(arg1, Union) else arg1.make_sql(fields),
+                sql2=arg2.make_sql() if isinstance(arg2, Union) else arg2.make_sql(fields),
+                alias=next(self.__count)
+                ))
         # with secondary attributes, use union of left join with antijoin
         fields = self.heading.names
         sql1 = arg1.join(arg2, left=True).make_sql(fields)

datajoint/jobs.py

@@ -32,7 +32,7 @@ def __init__(self, conn, database):
         status  :enum('reserved','error','ignore')  # if tuple is missing, the job is available
         key=null  :blob  # structure containing the key
         error_message=""  :varchar({error_message_length})  # error message returned if failed
-        error_stack=null  :blob  # error stack if failed
+        error_stack=null  :mediumblob  # error stack if failed
         user="" :varchar(255) # database user
         host=""  :varchar(255)  # system hostname
         pid=0  :int unsigned  # system process id

docs-parts/intro/Releases_lang1.rst

@@ -1,4 +1,4 @@
-0.13.3 -- TBD
+0.13.3 -- Feb 9, 2022
 ----------------------
 * Bugfix - Fix error in listing ancestors, descendants with part tables.
 * Bugfix - Fix Python 3.10 compatibility (#983) PR #972
@@ -14,6 +14,9 @@
 * Bugfix - Fix count for left-joined ``QueryExpressions`` (#951) PR #966
 * Bugfix - Fix assertion error when performing a union into a join (#930) PR #967
 * Bugfix - Fix regression issue with `DISTINCT` clause and `GROUP_BY` (#914) PR #963
+* Update `~jobs.error_stack` from blob to mediumblob to allow error stacks >64kB in jobs (#984) PR #986
+* Bugfix - Fix error when performing a union on multiple tables (#926) PR #964
+* Add - Allow optional keyword arguments for `make()` in `populate()` PR #971
 
 0.13.2 -- May 7, 2021
 ----------------------

tests/test_declare.py

@@ -160,6 +160,37 @@ def test_dependencies():
         assert_set_equal(set(s.full_table_name for s in channel.parents(primary=True, as_objects=True)),
                          {ephys.full_table_name})
 
+    @staticmethod
+    def test_descendants_only_contain_part_table():
+        """issue #927"""
+
+        @schema
+        class A(dj.Manual):
+            definition = """
+            a: int
+            """
+
+        @schema
+        class B(dj.Manual):
+            definition = """
+            -> A
+            b: int
+            """
+
+        @schema
+        class Master(dj.Manual):
+            definition = """
+            table_master: int
+            """
+
+            class Part(dj.Part):
+                definition = """
+                -> master
+                -> B
+                """
+
+        assert A.descendants() == ['`djtest_test1`.`a`', '`djtest_test1`.`b`', '`djtest_test1`.`master__part`']
+
     @staticmethod
     @raises(dj.DataJointError)
     def test_bad_attribute_name():

tests/test_jobs.py

@@ -122,3 +122,23 @@ def test_long_error_message():
     assert_true(error_message == short_error_message, 'error messages do not agree')
     assert_false(error_message.endswith(TRUNCATION_APPENDIX), 'error message should not be truncated')
     schema.schema.jobs.delete()
+
+
+def test_long_error_stack():
+    # clear out jobs table
+    schema.schema.jobs.delete()
+
+    # create long error stack
+    STACK_SIZE = 89942  # Does not fit into small blob (should be 64k, but found to be higher)
+    long_error_stack = ''.join(random.choice(string.ascii_letters) for _ in range(STACK_SIZE))
+    assert subjects
+    table_name = 'fake_table'
+
+    key = subjects.fetch('KEY')[0]
+
+    # test long error stack
+    schema.schema.jobs.reserve(table_name, key)
+    schema.schema.jobs.error(table_name, key, 'error message', long_error_stack)
+    error_stack = schema.schema.jobs.fetch1('error_stack')
+    assert error_stack == long_error_stack, 'error stacks do not agree'
+    schema.schema.jobs.delete()

tests/test_relational_operand.py

@@ -516,3 +516,13 @@ def test_joins_with_aggregation():
         session_dates = ((SessionDateA * (subj_query & 'date_trained<"2020-12-21"')) &
                          'session_date<date_trained')
         assert len(session_dates) == 1
+
+    @staticmethod
+    def test_union_multiple():
+        # https://github.com/datajoint/datajoint-python/issues/926
+        q1 = IJ & dict(j=2)
+        q2 = (IJ & dict(j=2, i=0)) + (IJ & dict(j=2, i=1)) + (IJ & dict(j=2, i=2))
+        x = set(zip(*q1.fetch('i', 'j')))
+        y = set(zip(*q2.fetch('i', 'j')))
+        assert x == y
+        assert q1.fetch(as_dict=True) == q2.fetch(as_dict=True)