Merge branch 'master' of https://github.com/datajoint/datajoint-python into black-formatting

Author: A-Baji

CHANGELOG.md

@@ -1,7 +1,10 @@
 ## Release notes
 
-### 0.13.4 -- TBA
+### 0.13.4 -- March, 25 2022
 * Add - Allow reading blobs produced by legacy 32-bit compiled mYm library for matlab. PR #995
+* Bugfix - Add missing `jobs` argument for multiprocessing PR #997
+* Add - Test for multiprocessing PR #1008
+* Bugfix - Fix external store key name doesn't allow '-' (#1005) PR #1006
 
 ### 0.13.3 -- Feb 9, 2022
 * Bugfix - Fix error in listing ancestors, descendants with part tables.

LNX-docker-compose.yml

@@ -32,7 +32,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.0.19
+    image: datajoint/nginx:v0.1.1
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306

README.md

@@ -8,10 +8,8 @@
 # Welcome to DataJoint for Python!
 DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
 
-DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
-
-Vathes LLC supports DataJoint for Python as an open-source project and everyone is welcome to contribute.
-Its DataJoint Neuro (https://djneuro.io) business provides support to neuroscience labs for developing and executing custom data pipelines.
+DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab at Baylor College of Medicine for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
+Presently, the primary developer of DataJoint open-source software is the company DataJoint (https://datajoint.com). Related resources are listed at https://datajoint.org
 
 ## Installation
 ```
@@ -22,7 +20,18 @@ If you already have an older version of DataJoint installed using `pip`, upgrade
 ```bash
 pip3 install --upgrade datajoint
 ```
+
+## Documentation and Tutorials
+
+* https://datajoint.org  -- start page
+* https://docs.datajoint.org -- up-to-date documentation
+* https://tutorials.datajoint.io -- step-by-step tutorials
+* https://elements.datajoint.org -- catalog of example pipelines
+* https://codebook.datajoint.io -- interactive online tutorials
+
 ## Python Native Blobs
+<details>
+<summary>Click to expand details</summary>
 
 DataJoint 0.12 adds full support for all native python data types in blobs: tuples, lists, sets, dicts, strings, bytes, `None`, and all their recursive combinations.
 The new blobs are a superset of the old functionality and are fully backward compatible.
@@ -92,16 +101,11 @@ the situation, but generally the following strategies may apply:
 As always, be sure that your data is safely backed up before modifying any
 important DataJoint schema or records.
 
-## Documentation and Tutorials
-A number of labs are currently adopting DataJoint and we are quickly getting the documentation in shape in February 2017.
-
-* https://datajoint.io  -- start page
-* https://docs.datajoint.io -- up-to-date documentation
-* https://tutorials.datajoint.io -- step-by-step tutorials
-* https://catalog.datajoint.io -- catalog of example pipelines
+</details>
 
 ## Running Tests Locally
-
+<details>
+<summary>Click to expand details</summary>
 
 * Create an `.env` with desired development environment values e.g.
 ``` sh
@@ -136,9 +140,9 @@ GID=1000
 * Add entry in `/etc/hosts` for `127.0.0.1 fakeservices.datajoint.io`
 
 
-
-
 ### Launch Jupyter Notebook for Interactive Use
 * Navigate to `localhost:8888`
 * Input Jupyter password
 * Launch a notebook i.e. `New > Python 3`
+
+</details>

datajoint/autopopulate.py

@@ -233,9 +233,7 @@ def handler(signum, frame):
             # spawn multiple processes
             self.connection.close()  # disconnect parent process from MySQL server
             del self.connection._conn.ctx  # SSLContext is not pickleable
-            with mp.Pool(
-                processes, _initialize_populate, (self, populate_kwargs)
-            ) as pool:
+            with mp.Pool(processes, _initialize_populate, (self, jobs, populate_kwargs)) as pool:
                 if display_progress:
                     with tqdm(desc="Processes: ", total=nkeys) as pbar:
                         for error in pool.imap(_call_populate1, keys, chunksize=1):

datajoint/declare.py

@@ -11,31 +11,25 @@
 
 UUID_DATA_TYPE = "binary(16)"
 MAX_TABLE_NAME_LENGTH = 64
-CONSTANT_LITERALS = {
-    "CURRENT_TIMESTAMP",
-    "NULL",
-}  # SQL literals to be used without quotes (case insensitive)
-EXTERNAL_TABLE_ROOT = "~external"
-
-TYPE_PATTERN = {
-    k: re.compile(v, re.I)
-    for k, v in dict(
-        INTEGER=r"((tiny|small|medium|big|)int|integer)(\s*\(.+\))?(\s+unsigned)?(\s+auto_increment)?|serial$",
-        DECIMAL=r"(decimal|numeric)(\s*\(.+\))?(\s+unsigned)?$",
-        FLOAT=r"(double|float|real)(\s*\(.+\))?(\s+unsigned)?$",
-        STRING=r"(var)?char\s*\(.+\)$",
-        ENUM=r"enum\s*\(.+\)$",
-        BOOL=r"bool(ean)?$",  # aliased to tinyint(1)
-        TEMPORAL=r"(date|datetime|time|timestamp|year)(\s*\(.+\))?$",
-        INTERNAL_BLOB=r"(tiny|small|medium|long|)blob$",
-        EXTERNAL_BLOB=r"blob@(?P<store>[a-z]\w*)$",
-        INTERNAL_ATTACH=r"attach$",
-        EXTERNAL_ATTACH=r"attach@(?P<store>[a-z]\w*)$",
-        FILEPATH=r"filepath@(?P<store>[a-z]\w*)$",
-        UUID=r"uuid$",
-        ADAPTED=r"<.+>$",
-    ).items()
-}
+CONSTANT_LITERALS = {'CURRENT_TIMESTAMP', 'NULL'}  # SQL literals to be used without quotes (case insensitive)
+EXTERNAL_TABLE_ROOT = '~external'
+
+TYPE_PATTERN = {k: re.compile(v, re.I) for k, v in dict(
+    INTEGER=r'((tiny|small|medium|big|)int|integer)(\s*\(.+\))?(\s+unsigned)?(\s+auto_increment)?|serial$',
+    DECIMAL=r'(decimal|numeric)(\s*\(.+\))?(\s+unsigned)?$',
+    FLOAT=r'(double|float|real)(\s*\(.+\))?(\s+unsigned)?$',
+    STRING=r'(var)?char\s*\(.+\)$',
+    ENUM=r'enum\s*\(.+\)$',
+    BOOL=r'bool(ean)?$',   # aliased to tinyint(1)
+    TEMPORAL=r'(date|datetime|time|timestamp|year)(\s*\(.+\))?$',
+    INTERNAL_BLOB=r'(tiny|small|medium|long|)blob$',
+    EXTERNAL_BLOB=r'blob@(?P<store>[a-z][\-\w]*)$',
+    INTERNAL_ATTACH=r'attach$',
+    EXTERNAL_ATTACH=r'attach@(?P<store>[a-z][\-\w]*)$',
+    FILEPATH=r'filepath@(?P<store>[a-z][\-\w]*)$',
+    UUID=r'uuid$',
+    ADAPTED=r'<.+>$'
+).items()}
 
 # custom types are stored in attribute comment
 SPECIAL_TYPES = {

docs-parts/intro/Releases_lang1.rst

@@ -1,6 +1,9 @@
-0.13.4 -- TBA
+0.13.4 -- March 25, 2022
 ----------------------
 * Add - Allow reading blobs produced by legacy 32-bit compiled mYm library for matlab. PR #995
+* Bugfix - Add missing ``jobs`` argument for multiprocessing PR #997
+* Add - Test for multiprocessing PR #1008
+* Bugfix - Fix external store key name doesn't allow '-' (#1005) PR #1006
 
 0.13.3 -- Feb 9, 2022
 ----------------------

local-docker-compose.yml

@@ -34,7 +34,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.0.19
+    image: datajoint/nginx:v0.1.1
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306

tests/schema_adapted.py

@@ -8,7 +8,7 @@
 from . import PREFIX, CONN_INFO, S3_CONN_INFO
 
 stores_config = {
-    'repo_s3': dict(
+    'repo-s3': dict(
         S3_CONN_INFO,
         protocol='s3',
         location='adapted/repo',
@@ -59,7 +59,7 @@ class LayoutToFilepath(dj.AttributeAdapter):
     An adapted data type that saves a graph layout into fixed filepath
     """
 
-    attribute_type = 'filepath@repo_s3'
+    attribute_type = 'filepath@repo-s3'
 
     @staticmethod
     def get(path):
@@ -68,7 +68,7 @@ def get(path):
 
     @staticmethod
     def put(layout):
-        path = Path(dj.config['stores']['repo_s3']['stage'], 'layout.json')
+        path = Path(dj.config['stores']['repo-s3']['stage'], 'layout.json')
         with open(str(path), "w") as f:
             json.dump(layout, f)
         return path

tests/schema_external.py

@@ -22,7 +22,7 @@
         protocol='file',
         location=tempfile.mkdtemp()),
 
-    'repo_s3': dict(
+    'repo-s3': dict(
         S3_CONN_INFO,
         protocol='s3',
         location='dj/repo',
@@ -130,7 +130,7 @@ class FilepathS3(dj.Manual):
     # table for file management 
     fnum : int 
     ---
-    img : filepath@repo_s3  # managed files 
+    img : filepath@repo-s3  # managed files 
     """
 
 dj.errors._switch_filepath_types(False)

tests/test_autopopulate.py

@@ -57,6 +57,12 @@ def test_allow_direct_insert(self):
         key['experiment_date'] = '2018-10-30'
         self.experiment.insert1(key, allow_direct_insert=True)
 
+    def test_multi_processing(self):
+        assert self.subject, 'root tables are empty'
+        assert not self.experiment, 'table already filled?'
+        self.experiment.populate(processes=2)
+        assert len(self.experiment) == len(self.subject)*self.experiment.fake_experiments_per_subject
+
     @raises(DataJointError)
     def test_allow_insert(self):
         assert_true(self.subject, 'root tables are empty')