Merge pull request #1062 from ttngu207/populate_exclude_error_ignore

`.populate(reserve_jobs=True)` to exclude `error` and `ignore` keys

Author: dimitri-yatsenko

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### 0.14.0 -- TBA
 * Bugfix - Activating a schema requires all tables to exist even if `create_tables=False` PR [#1058](https://github.com/datajoint/datajoint-python/pull/1058)
+* Update - Populate call with `reserve_jobs=True` to exclude `error` and `ignore` keys - PR [#1062](https://github.com/datajoint/datajoint-python/pull/1062)
 
 ### 0.13.8 -- Sep 21, 2022
 * Add - New documentation structure based on markdown PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052)

datajoint/autopopulate.py

@@ -5,6 +5,7 @@
 import random
 import inspect
 from tqdm import tqdm
+from .hash import key_hash
 from .expression import QueryExpression, AndList
 from .errors import DataJointError, LostConnectionError
 import signal
@@ -202,6 +203,16 @@ def handler(signum, frame):
             old_handler = signal.signal(signal.SIGTERM, handler)
 
         keys = (self._jobs_to_do(restrictions) - self.target).fetch("KEY", limit=limit)
+
+        # exclude "error" or "ignore" jobs
+        if reserve_jobs:
+            exclude_key_hashes = (
+                jobs
+                & {"table_name": self.target.table_name}
+                & 'status in ("error", "ignore")'
+            ).fetch("key_hash")
+            keys = [key for key in keys if key_hash(key) not in exclude_key_hashes]
+
         if order == "reverse":
             keys.reverse()
         elif order == "random":

tests/test_autopopulate.py

@@ -53,6 +53,28 @@ def test_populate(self):
         assert_true(self.ephys)
         assert_true(self.channel)
 
+    def test_populate_exclude_error_and_ignore_jobs(self):
+        # test simple populate
+        assert_true(self.subject, "root tables are empty")
+        assert_false(self.experiment, "table already filled?")
+
+        keys = self.experiment.key_source.fetch("KEY", limit=2)
+        for idx, key in enumerate(keys):
+            schema.schema.jobs.insert1(
+                {
+                    "table_name": self.experiment.table_name,
+                    "key_hash": dj.hash.key_hash(key),
+                    "status": "error" if idx == 0 else "ignore",
+                    "key": key,
+                }
+            )
+
+        self.experiment.populate(reserve_jobs=True)
+        assert_equal(
+            len(self.experiment.key_source & self.experiment),
+            len(self.experiment.key_source) - 2,
+        )
+
     def test_allow_direct_insert(self):
         assert_true(self.subject, "root tables are empty")
         key = self.subject.fetch("KEY", limit=1)[0]