Merge pull request #856 from guzman-raphael/plugin_0_12_8

-> Plugin: Sync with release 0.12.8

Author: dimitri-yatsenko

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,37 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: 'bug, awaiting-triage'
+assignees: ''
+
+---
+
+## Bug Report
+
+### Description
+A clear and concise description of what is the overall operation that is intended to be performed that resulted in an error.
+
+### Reproducibility
+Include:
+- OS (WIN | MACOS | Linux)
+- Python Version OR MATLAB Version
+- MySQL Version
+- MySQL Deployment Strategy (local-native | local-docker | remote)
+- DataJoint Version
+- Minimum number of steps to reliably reproduce the issue
+- Complete error stack as a result of evaluating the above steps
+
+### Expected Behavior
+A clear and concise description of what you expected to happen.
+
+### Screenshots
+If applicable, add screenshots to help explain your problem.
+
+### Additional Research and Context
+Add any additional research or context that was conducted in creating this report.
+
+For example:
+- Related GitHub issues and PR's either within this repository or in other relevant repositories.
+- Specific links to specific lines or a focus within source code.
+- Relevant summary of Maintainers development meetings, milestones, projects, etc.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,46 @@
+---
+name: Feature request
+about: Suggest an idea for a new feature
+title: ''
+labels: 'enhancement, awaiting-triage'
+assignees: ''
+
+---
+
+## Feature Request
+
+### Problem
+A clear and concise description how this idea has manifested and the context. Elaborate on the need for this feature and/or what could be improved. Ex. I'm always frustrated when [...]
+
+### Requirements
+A clear and concise description of the requirements to satisfy the new feature. Detail what you expect from a successful implementation of the feature. Ex. When using this feature, it should [...]
+
+### Justification
+Provide the key benefits in making this a supported feature. Ex. Adding support for this feature would ensure [...]
+
+### Alternative Considerations
+Do you currently have a work-around for this? Provide any alternative solutions or features you've considered.
+
+### Related Errors
+Add any errors as a direct result of not exposing this feature.
+
+Please include steps to reproduce provided errors as follows:
+- OS (WIN | MACOS | Linux)
+- Python Version OR MATLAB Version
+- MySQL Version
+- MySQL Deployment Strategy (local-native | local-docker | remote)
+- DataJoint Version
+- Minimum number of steps to reliably reproduce the issue
+- Complete error stack as a result of evaluating the above steps
+
+### Screenshots
+If applicable, add screenshots to help explain your feature.
+
+### Additional Research and Context
+Add any additional research or context that was conducted in creating this feature request.
+
+For example:
+- Related GitHub issues and PR's either within this repository or in other relevant repositories.
+- Specific links to specific line or focus within source code.
+- Relevant summary of Maintainers development meetings, milestones, projects, etc.
+- Any additional supplemental web references or links that would further justify this feature request.

.github/workflows/development.yaml

@@ -0,0 +1,53 @@
+name: Development
+on:
+  push:
+    branches:
+      - '**' # every branch
+      - '!stage*' # exclude branches beginning with stage
+  pull_request:
+    branches:
+      - '**' # every branch
+      - '!stage*' # exclude branches beginning with stage
+jobs:
+  test:
+    if: github.event_name == 'push' || github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        py_ver: ["3.8"]
+        mysql_ver: ["8.0", "5.7", "5.6"]
+        include:
+          - py_ver: "3.7"
+            mysql_ver: "5.7"
+          - py_ver: "3.6"
+            mysql_ver: "5.7"
+          - py_ver: "3.5"
+            mysql_ver: "5.7"
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{matrix.py_ver}}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{matrix.py_ver}}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8
+      - name: Run syntax tests
+        run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
+      - name: Run primary tests
+        env:
+          UID: "1001"
+          GID: "116"
+          PY_VER: ${{matrix.py_ver}}
+          MYSQL_VER: ${{matrix.mysql_ver}}
+          ALPINE_VER: "3.10"
+          MINIO_VER: RELEASE.2019-09-26T19-42-35Z
+          COMPOSE_HTTP_TIMEOUT: "120"
+          COVERALLS_SERVICE_NAME: travis-ci
+          COVERALLS_REPO_TOKEN: fd0BoXG46TPReEem0uMy7BJO5j0w1MQiY
+        run: docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
+      - name: Run style tests
+        run: |
+          flake8 --ignore=E121,E123,E126,E226,E24,E704,W503,W504,E722,F401,W605 datajoint \
+                 --count --max-complexity=62 --max-line-length=127 --statistics

.travis.yml

@@ -1,5 +1,14 @@
 ## Release notes
 
+### 0.12.8b1 -- Jan 12, 2021
+* table.children, .parents, .descendents, and ancestors can return queryable objects. PR #833
+* Load dependencies before querying dependencies. (#179) PR #833
+* Fix display of part tables in `schema.save`. (#821) PR #833
+* Add `schema.list_tables`. (#838) PR #844
+* Fix minio new version regression.  PR #847
+* Add more S3 logging for debugging. (#831) PR #832 
+* Convert testing framework from TravisCI to GitHub Actions (#841) PR #840
+ 
 ### 0.12.7b1 -- Oct 27, 2020
 * Fix case sensitivity issues to adapt to MySQL 8+.  PR #819
 * Fix pymysql regression bug (#814) PR #816

CHANGELOG.md

@@ -32,7 +32,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: raphaelguzman/nginx:v0.0.10
+    image: raphaelguzman/nginx:v0.0.13
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306

LNX-docker-compose.yml

@@ -32,25 +32,19 @@ def key_source(self):
                 The default value is the join of the parent relations.
                 Users may override to change the granularity or the scope of populate() calls.
         """
-        def parent_gen(self):
-            if self.target.full_table_name not in self.connection.dependencies:
-                self.connection.dependencies.load()
-            for parent_name, fk_props in self.target.parents(primary=True).items():
-                if not parent_name.isdigit():  # simple foreign key
-                    yield FreeTable(self.connection, parent_name).proj()
-                else:
-                    grandparent = list(self.connection.dependencies.in_edges(parent_name))[0][0]
-                    yield FreeTable(self.connection, grandparent).proj(**{
-                        attr: ref for attr, ref in fk_props['attr_map'].items() if ref != attr})
+        def _rename_attributes(table, props):
+            return (table.proj(
+                **{attr: ref for attr, ref in props['attr_map'].items() if attr != ref})
+                if props['aliased'] else table)
 
         if self._key_source is None:
-            parents = parent_gen(self)
-            try:
-                self._key_source = next(parents)
-            except StopIteration:
-                raise DataJointError('A table must have primary dependencies for auto-populate to work')
-            for q in parents:
-                self._key_source *= q
+            parents = self.target.parents(primary=True, as_objects=True, foreign_key_info=True)
+            if not parents:
+                raise DataJointError(
+                    'A relation must have primary dependencies for auto-populate to work')
+            self._key_source = _rename_attributes(*parents[0])
+            for q in parents[1:]:
+                self._key_source *= _rename_attributes(*q)
         return self._key_source
 
     def make(self, key):

datajoint/autopopulate.py

@@ -195,6 +195,7 @@ def close(self):
 
     def register(self, schema):
         self.schemas[schema.database] = schema
+        self.dependencies.clear()
 
     def ping(self):
         """

datajoint/connection.py

@@ -1,9 +1,36 @@
 import networkx as nx
 import itertools
+import re
 from collections import defaultdict, OrderedDict
 from .errors import DataJointError
 
 
+def unite_master_parts(lst):
+    """
+    re-order a list of table names so that part tables immediately follow their master tables without breaking
+    the topological order.
+    Without this correction, a simple topological sort may insert other descendants between master and parts.
+    The input list must be topologically sorted.
+    :example:
+    unite_master_parts(
+        ['`s`.`a`', '`s`.`a__q`', '`s`.`b`', '`s`.`c`', '`s`.`c__q`', '`s`.`b__q`', '`s`.`d`', '`s`.`a__r`']) ->
+        ['`s`.`a`', '`s`.`a__q`', '`s`.`a__r`', '`s`.`b`', '`s`.`b__q`', '`s`.`c`', '`s`.`c__q`', '`s`.`d`']
+    """
+    for i in range(2, len(lst)):
+        name = lst[i]
+        match = re.match(r'(?P<master>`\w+`.`\w+)__\w+`', name)
+        if match:  # name is a part table
+            master = match.group('master')
+            for j in range(i-1, -1, -1):
+                if lst[j] == master + '`' or lst[j].startswith(master + '__'):
+                    # move from the ith position to the (j+1)th position
+                    lst[j+1:i+1] = [name] + lst[j+1:i]
+                    break
+            else:
+                raise DataJointError("Found a part table {name} without its master table.".format(name=name))
+    return lst
+
+
 class Dependencies(nx.DiGraph):
     """
     The graph of dependencies (foreign keys) between loaded tables.
@@ -16,15 +43,22 @@ class Dependencies(nx.DiGraph):
     def __init__(self, connection=None):
         self._conn = connection
         self._node_alias_count = itertools.count()
+        self._loaded = False
         super().__init__(self)
 
-    def load(self):
+    def clear(self):
+        self._loaded = False
+        super().clear()
+
+    def load(self, force=True):
         """
         Load dependencies for all loaded schemas.
         This method gets called before any operation that requires dependencies: delete, drop, populate, progress.
         """
-
         # reload from scratch to prevent duplication of renamed edges
+        if self._loaded and not force:
+            return
+
         self.clear()
 
         # load primary key info
@@ -77,6 +111,7 @@ def load(self):
 
         if not nx.is_directed_acyclic_graph(self):  # pragma: no cover
             raise DataJointError('DataJoint can only work with acyclic dependencies')
+        self._loaded = True
 
     def parents(self, table_name, primary=None):
         """
@@ -86,6 +121,7 @@ def parents(self, table_name, primary=None):
             attribute are considered.
         :return: dict of tables referenced by the foreign keys of table
         """
+        self.load(force=False)
         return {p[0]: p[2] for p in self.in_edges(table_name, data=True)
                 if primary is None or p[2]['primary'] == primary}
 
@@ -97,6 +133,7 @@ def children(self, table_name, primary=None):
             attribute are considered.
         :return: dict of tables referencing the table through foreign keys
         """
+        self.load(force=False)
         return {p[1]: p[2] for p in self.out_edges(table_name, data=True)
                 if primary is None or p[2]['primary'] == primary}
 
@@ -105,17 +142,19 @@ def descendants(self, full_table_name):
         :param full_table_name:  In form `schema`.`table_name`
         :return: all dependent tables sorted in topological order.  Self is included.
         """
+        self.load(force=False)
         nodes = self.subgraph(
             nx.algorithms.dag.descendants(self, full_table_name))
-        return [full_table_name] + list(
-            nx.algorithms.dag.topological_sort(nodes))
+        return unite_master_parts([full_table_name] + list(
+            nx.algorithms.dag.topological_sort(nodes)))
 
     def ancestors(self, full_table_name):
         """
         :param full_table_name:  In form `schema`.`table_name`
         :return: all dependent tables sorted in topological order.  Self is included.
         """
+        self.load(force=False)
         nodes = self.subgraph(
             nx.algorithms.dag.ancestors(self, full_table_name))
-        return [full_table_name] + list(reversed(list(
-            nx.algorithms.dag.topological_sort(nodes))))
+        return list(reversed(unite_master_parts(list(
+            nx.algorithms.dag.topological_sort(nodes)) + [full_table_name])))