Fix merge conflicts.

Author: jverswijver

CHANGELOG.md

@@ -1,11 +1,13 @@
 ## Release notes
 
 ### 0.13.3 -- TBD
+* Add - Expose proxy feature for S3 external stores (#961) PR #962
 * Bugfix - Dependencies not properly loaded on populate. (#902) PR #919
 * Bugfix - Replace use of numpy aliases of built-in types with built-in type. (#938) PR #939
 * Bugfix - `ExternalTable.delete` should not remove row on error (#953) PR #956
 * Bugfix - Fix error handling of remove_object function in `s3.py` (#952) PR #955
 * Bugfix - Fix regression issue with DISTINCT clause and GROUP_BY (#914) PR #963
+* Bugfix - Fix assertion error when performing a union into a join (#930) PR #967
 
 ### 0.13.2 -- May 7, 2021
 * Update `setuptools_certificate` dependency to new name `otumat`

datajoint/expression.py

@@ -268,9 +268,11 @@ def join(self, other, semantic_check=True, left=False):
             - join_attributes)
         # need subquery if any of the join attributes are derived
         need_subquery1 = (need_subquery1 or isinstance(self, Aggregation) or
-                          any(n in self.heading.new_attributes for n in join_attributes))
+                          any(n in self.heading.new_attributes for n in join_attributes)
+                          or isinstance(self, Union))
         need_subquery2 = (need_subquery2 or isinstance(other, Aggregation) or
-                          any(n in other.heading.new_attributes for n in join_attributes))
+                          any(n in other.heading.new_attributes for n in join_attributes)
+                          or isinstance(self, Union))
         if need_subquery1:
             self = self.make_subquery()
         if need_subquery2:

datajoint/s3.py

@@ -3,6 +3,7 @@
 """
 from io import BytesIO
 import minio   # https://docs.minio.io/docs/python-client-api-reference
+import urllib3
 import warnings
 import uuid
 import logging
@@ -16,9 +17,24 @@ class Folder:
     """
     A Folder instance manipulates a flat folder of objects within an S3-compatible object store
     """
-    def __init__(self, endpoint, bucket, access_key, secret_key, *, secure=False, **_):
-        self.client = minio.Minio(endpoint, access_key=access_key, secret_key=secret_key,
-                                  secure=secure)
+    def __init__(self, endpoint, bucket, access_key, secret_key, *, secure=False,
+                 proxy_server=None, **_):
+        # from https://docs.min.io/docs/python-client-api-reference
+        self.client = minio.Minio(
+            endpoint,
+            access_key=access_key,
+            secret_key=secret_key,
+            secure=secure,
+            http_client=(
+                urllib3.ProxyManager(proxy_server,
+                                     timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
+                                     cert_reqs="CERT_REQUIRED",
+                                     retries=urllib3.Retry(total=5,
+                                                           backoff_factor=0.2,
+                                                           status_forcelist=[500, 502, 503,
+                                                                             504]))
+                if proxy_server else None),
+            )
         self.bucket = bucket
         if not self.client.bucket_exists(bucket):
             raise errors.BucketInaccessible('Inaccessible s3 bucket %s' % bucket)

datajoint/settings.py

@@ -137,7 +137,8 @@ def get_store_spec(self, store):
         spec['subfolding'] = spec.get('subfolding', DEFAULT_SUBFOLDING)
         spec_keys = {  # REQUIRED in uppercase and allowed in lowercase
             'file': ('PROTOCOL', 'LOCATION', 'subfolding', 'stage'),
-            's3': ('PROTOCOL', 'ENDPOINT', 'BUCKET', 'ACCESS_KEY', 'SECRET_KEY', 'LOCATION', 'secure', 'subfolding', 'stage')}
+            's3': ('PROTOCOL', 'ENDPOINT', 'BUCKET', 'ACCESS_KEY', 'SECRET_KEY', 'LOCATION',
+                   'secure', 'subfolding', 'stage', 'proxy_server')}
 
         try:
             spec_keys = spec_keys[spec.get('protocol', '').lower()]

docs-parts/intro/Releases_lang1.rst

@@ -1,10 +1,12 @@
 0.13.3 -- TBD
 ----------------------
+* Add - Expose proxy feature for S3 external stores (#961) PR #962
 * Bugfix - Dependencies not properly loaded on populate. (#902) PR #919
 * Bugfix - Replace use of numpy aliases of built-in types with built-in type. (#938) PR #939
 * Bugfix - `ExternalTable.delete` should not remove row on error (#953) PR #956
 * Bugfix - Fix error handling of remove_object function in `s3.py` (#952) PR #955
 * Bugfix - Fix regression issue with DISTINCT clause and GROUP_BY (#914) PR #963
+* Bugfix - Fix assertion error when performing a union into a join (#930) PR #967
 
 0.13.2 -- May 7, 2021
 ----------------------

requirements.txt

@@ -10,3 +10,4 @@ minio>=7.0.0
 matplotlib
 cryptography
 otumat
+urllib3

tests/test_aggr_regressions.py

@@ -103,3 +103,16 @@ def test_issue558_part2():
     d = dict(id=3, id2=5)
     assert_equal(len(X & d), len((X & d).proj(id2='3')))
 
+
+def test_union_join():
+    # https://github.com/datajoint/datajoint-python/issues/930
+    A.insert(zip([100, 200, 300, 400, 500, 600]))
+    B.insert([(100, 11), (200, 22), (300, 33), (400, 44)])
+    q1 = B & 'id < 300'
+    q2 = B & 'id > 300'
+
+    expected_data = [{'id': 0, 'id2': 5}, {'id': 1, 'id2': 6}, {'id': 2, 'id2': 7},
+                     {'id': 3, 'id2': 8}, {'id': 4, 'id2': 9}, {'id': 100, 'id2': 11},
+                     {'id': 200, 'id2': 22}, {'id': 400, 'id2': 44}]
+
+    assert ((q1 + q2) * A).fetch(as_dict=True) == expected_data