Skip to content

Commit 2b451f7

Browse files
2 parents 6fbed92 + 8115393 commit 2b451f7

File tree

9 files changed

+91
-16
lines changed

9 files changed

+91
-16
lines changed

CHANGELOG.md

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
## Release notes
22

3-
### 0.13.3 -- TBD
3+
### 0.13.3 -- Feb 9, 2022
44
* Bugfix - Fix error in listing ancestors, descendants with part tables.
55
* Bugfix - Fix Python 3.10 compatibility (#983) PR #972
66
* Bugfix - Allow renaming non-conforming attributes in proj (#982) PR #972
@@ -15,6 +15,9 @@
1515
* Bugfix - Fix sql code generation to comply with sql mode `ONLY_FULL_GROUP_BY` (#916) PR #965
1616
* Bugfix - Fix count for left-joined `QueryExpressions` (#951) PR #966
1717
* Bugfix - Fix assertion error when performing a union into a join (#930) PR #967
18+
* Update `~jobs.error_stack` from blob to mediumblob to allow error stacks >64kB in jobs (#984) PR #986
19+
* Bugfix - Fix error when performing a union on multiple tables (#926) PR #964
20+
* Add - Allow optional keyword arguments for `make()` in `populate()` PR #971
1821

1922
### 0.13.2 -- May 7, 2021
2023
* Update `setuptools_certificate` dependency to new name `otumat`
@@ -248,7 +251,7 @@ Documentation and tutorials available at https://docs.datajoint.io and https://t
248251
* ERD() no longer text the context argument.
249252
* ERD.draw() now takes an optional context argument. By default uses the caller's locals.
250253

251-
### 0.3.2.
254+
### 0.3.2.
252255
* Fixed issue #223: `insert` can insert relations without fetching.
253256
* ERD() now takes the `context` argument, which specifies in which context to look for classes. The default is taken from the argument (schema or relation).
254257
* ERD.draw() no longer has the `prefix` argument: class names are shown as found in the context.

datajoint/autopopulate.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -128,10 +128,11 @@ def _jobs_to_do(self, restrictions):
128128

129129
def populate(self, *restrictions, suppress_errors=False, return_exception_objects=False,
130130
reserve_jobs=False, order="original", limit=None, max_calls=None,
131-
display_progress=False, processes=1):
131+
display_progress=False, processes=1, make_kwargs=None):
132132
"""
133-
table.populate() calls table.make(key) for every primary key in self.key_source
134-
for which there is not already a tuple in table.
133+
``table.populate()`` calls ``table.make(key)`` for every primary key in
134+
``self.key_source`` for which there is not already a tuple in table.
135+
135136
:param restrictions: a list of restrictions each restrict
136137
(table.key_source - target.proj())
137138
:param suppress_errors: if True, do not terminate execution.
@@ -143,6 +144,10 @@ def populate(self, *restrictions, suppress_errors=False, return_exception_object
143144
:param display_progress: if True, report progress_bar
144145
:param processes: number of processes to use. When set to a large number, then
145146
uses as many as CPU cores
147+
:param make_kwargs: Keyword arguments which do not affect the result of computation
148+
to be passed down to each ``make()`` call. Computation arguments should be
149+
specified within the pipeline e.g. using a `dj.Lookup` table.
150+
:type make_kwargs: dict, optional
146151
"""
147152
if self.connection.in_transaction:
148153
raise DataJointError('Populate cannot be called during a transaction.')
@@ -176,7 +181,8 @@ def handler(signum, frame):
176181
error_list = []
177182
populate_kwargs = dict(
178183
suppress_errors=suppress_errors,
179-
return_exception_objects=return_exception_objects)
184+
return_exception_objects=return_exception_objects,
185+
make_kwargs=make_kwargs)
180186

181187
if processes == 1:
182188
for key in tqdm(keys, desc=self.__class__.__name__) if display_progress else keys:
@@ -207,7 +213,7 @@ def handler(signum, frame):
207213
if suppress_errors:
208214
return error_list
209215

210-
def _populate1(self, key, jobs, suppress_errors, return_exception_objects):
216+
def _populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs=None):
211217
"""
212218
populates table for one source key, calling self.make inside a transaction.
213219
:param jobs: the jobs table or None if not reserve_jobs
@@ -228,7 +234,7 @@ def _populate1(self, key, jobs, suppress_errors, return_exception_objects):
228234
logger.info('Populating: ' + str(key))
229235
self.__class__._allow_insert = True
230236
try:
231-
make(dict(key))
237+
make(dict(key), **(make_kwargs or {}))
232238
except (KeyboardInterrupt, SystemExit, Exception) as error:
233239
try:
234240
self.connection.cancel_transaction()

datajoint/dependencies.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ def unite_master_parts(lst):
2626
# move from the ith position to the (j+1)th position
2727
lst[j+1:i+1] = [name] + lst[j+1:i]
2828
break
29-
else:
30-
raise DataJointError("Found a part table {name} without its master table.".format(name=name))
3129
return lst
3230

3331

datajoint/expression.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ def from_clause(self):
102102
return clause
103103

104104
def where_clause(self):
105-
return '' if not self.restriction else ' WHERE(%s)' % ')AND('.join(
105+
return '' if not self.restriction else ' WHERE (%s)' % ')AND('.join(
106106
str(s) for s in self.restriction)
107107

108108
def make_sql(self, fields=None):
@@ -606,6 +606,8 @@ class Union(QueryExpression):
606606
"""
607607
Union is the private DataJoint class that implements the union operator.
608608
"""
609+
__count = count()
610+
609611
@classmethod
610612
def create(cls, arg1, arg2):
611613
if inspect.isclass(arg2) and issubclass(arg2, QueryExpression):
@@ -632,9 +634,11 @@ def make_sql(self):
632634
if not arg1.heading.secondary_attributes and not arg2.heading.secondary_attributes:
633635
# no secondary attributes: use UNION DISTINCT
634636
fields = arg1.primary_key
635-
return "({sql1}) UNION ({sql2})".format(
636-
sql1=arg1.make_sql(fields),
637-
sql2=arg2.make_sql(fields))
637+
return ("SELECT * FROM (({sql1}) UNION ({sql2})) as `_u{alias}`".format(
638+
sql1=arg1.make_sql() if isinstance(arg1, Union) else arg1.make_sql(fields),
639+
sql2=arg2.make_sql() if isinstance(arg2, Union) else arg2.make_sql(fields),
640+
alias=next(self.__count)
641+
))
638642
# with secondary attributes, use union of left join with antijoin
639643
fields = self.heading.names
640644
sql1 = arg1.join(arg2, left=True).make_sql(fields)

datajoint/jobs.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ def __init__(self, conn, database):
3232
status :enum('reserved','error','ignore') # if tuple is missing, the job is available
3333
key=null :blob # structure containing the key
3434
error_message="" :varchar({error_message_length}) # error message returned if failed
35-
error_stack=null :blob # error stack if failed
35+
error_stack=null :mediumblob # error stack if failed
3636
user="" :varchar(255) # database user
3737
host="" :varchar(255) # system hostname
3838
pid=0 :int unsigned # system process id

docs-parts/intro/Releases_lang1.rst

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
0.13.3 -- TBD
1+
0.13.3 -- Feb 9, 2022
22
----------------------
33
* Bugfix - Fix error in listing ancestors, descendants with part tables.
44
* Bugfix - Fix Python 3.10 compatibility (#983) PR #972
@@ -14,6 +14,9 @@
1414
* Bugfix - Fix count for left-joined ``QueryExpressions`` (#951) PR #966
1515
* Bugfix - Fix assertion error when performing a union into a join (#930) PR #967
1616
* Bugfix - Fix regression issue with `DISTINCT` clause and `GROUP_BY` (#914) PR #963
17+
* Update `~jobs.error_stack` from blob to mediumblob to allow error stacks >64kB in jobs (#984) PR #986
18+
* Bugfix - Fix error when performing a union on multiple tables (#926) PR #964
19+
* Add - Allow optional keyword arguments for `make()` in `populate()` PR #971
1720

1821
0.13.2 -- May 7, 2021
1922
----------------------

tests/test_declare.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,37 @@ def test_dependencies():
160160
assert_set_equal(set(s.full_table_name for s in channel.parents(primary=True, as_objects=True)),
161161
{ephys.full_table_name})
162162

163+
@staticmethod
164+
def test_descendants_only_contain_part_table():
165+
"""issue #927"""
166+
167+
@schema
168+
class A(dj.Manual):
169+
definition = """
170+
a: int
171+
"""
172+
173+
@schema
174+
class B(dj.Manual):
175+
definition = """
176+
-> A
177+
b: int
178+
"""
179+
180+
@schema
181+
class Master(dj.Manual):
182+
definition = """
183+
table_master: int
184+
"""
185+
186+
class Part(dj.Part):
187+
definition = """
188+
-> master
189+
-> B
190+
"""
191+
192+
assert A.descendants() == ['`djtest_test1`.`a`', '`djtest_test1`.`b`', '`djtest_test1`.`master__part`']
193+
163194
@staticmethod
164195
@raises(dj.DataJointError)
165196
def test_bad_attribute_name():

tests/test_jobs.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,3 +122,23 @@ def test_long_error_message():
122122
assert_true(error_message == short_error_message, 'error messages do not agree')
123123
assert_false(error_message.endswith(TRUNCATION_APPENDIX), 'error message should not be truncated')
124124
schema.schema.jobs.delete()
125+
126+
127+
def test_long_error_stack():
128+
# clear out jobs table
129+
schema.schema.jobs.delete()
130+
131+
# create long error stack
132+
STACK_SIZE = 89942 # Does not fit into small blob (should be 64k, but found to be higher)
133+
long_error_stack = ''.join(random.choice(string.ascii_letters) for _ in range(STACK_SIZE))
134+
assert subjects
135+
table_name = 'fake_table'
136+
137+
key = subjects.fetch('KEY')[0]
138+
139+
# test long error stack
140+
schema.schema.jobs.reserve(table_name, key)
141+
schema.schema.jobs.error(table_name, key, 'error message', long_error_stack)
142+
error_stack = schema.schema.jobs.fetch1('error_stack')
143+
assert error_stack == long_error_stack, 'error stacks do not agree'
144+
schema.schema.jobs.delete()

tests/test_relational_operand.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -516,3 +516,13 @@ def test_joins_with_aggregation():
516516
session_dates = ((SessionDateA * (subj_query & 'date_trained<"2020-12-21"')) &
517517
'session_date<date_trained')
518518
assert len(session_dates) == 1
519+
520+
@staticmethod
521+
def test_union_multiple():
522+
# https://github.com/datajoint/datajoint-python/issues/926
523+
q1 = IJ & dict(j=2)
524+
q2 = (IJ & dict(j=2, i=0)) + (IJ & dict(j=2, i=1)) + (IJ & dict(j=2, i=2))
525+
x = set(zip(*q1.fetch('i', 'j')))
526+
y = set(zip(*q2.fetch('i', 'j')))
527+
assert x == y
528+
assert q1.fetch(as_dict=True) == q2.fetch(as_dict=True)

0 commit comments

Comments
 (0)