Support non-blocking (de)registration in the latest SewerRat API.

Author: LTLA

src/sewerrat/deregister.py

@@ -5,15 +5,15 @@
 from . import _utils as ut
 
 
-def deregister(path: str, url: str, retry: int = 3, wait: float = 1):
+def deregister(path: str, url: str, retry: int = 3, wait: float = 1, block: bool = True):
     """
-    Deregister a directory from the SewerRat search index. It is assumed that
-    this directory is world-readable and that the caller has write access to
-    it; or, the directory does not exist.
+    Deregister a directory from the SewerRat search index.
 
     Args:
         path: 
-            Path to the directory to be registered.
+            Path to the directory to be deregistered.
+            The directory should either be readable by the SewerRat API and the caller should have write access;
+            or the directory should not exist.
 
         url:
             URL to the SewerRat REST API. 
@@ -23,9 +23,18 @@ def deregister(path: str, url: str, retry: int = 3, wait: float = 1):
 
         wait:
             Deprecated, ignored.
+
+        block:
+            Whether to block on successful deregistration.
+
+    Returns:
+        On success, the directory is deregistered. 
+
+        If ``block = False``, the function returns before confirmation of successful deregistration from the SewerRat API.
+        This can be useful for asynchronous processing of directories with many files.
     """
     path = ut.clean_path(path)
-    res = requests.post(url + "/deregister/start", json = { "path": path }, allow_redirects=True)
+    res = requests.post(url + "/deregister/start", json = { "path": path, "block": block }, allow_redirects=True)
     if res.status_code >= 300:
         raise ut.format_error(res)
 
@@ -40,7 +49,7 @@ def deregister(path: str, url: str, retry: int = 3, wait: float = 1):
         pass
 
     try:
-        res = requests.post(url + "/deregister/finish", json = { "path": path }, allow_redirects=True)
+        res = requests.post(url + "/deregister/finish", json = { "path": path, "block": block }, allow_redirects=True)
         if res.status_code >= 300:
             raise ut.format_error(res)
     finally:

src/sewerrat/register.py

@@ -7,22 +7,18 @@
 from . import _utils as ut
 
 
-def register(path: str, names: Union[str, List[str]], url: str, retry: int = 3, wait: int = 1):
+def register(path: str, names: Union[str, List[str]], url: str, retry: int = 3, wait: int = 1, block: bool = True):
     """
-    Register a directory into the SewerRat search index. It is assumed that
-    that the directory is world-readable and that the caller has write access.
-    If a metadata file cannot be indexed (e.g., due to incorrect formatting,
-    insufficient permissions), a warning will be printed but the function will
-    not throw an error.
+    Register a directory into the SewerRat search index.
 
     Args:
         path: 
             Path to the directory to be registered.
+            The directory should be readable by the SewerRat API and the caller should have write access.
 
         names: 
-            List of strings containing the base names of metadata files inside
-            ``path`` to be indexed. Alternatively, a single string containing
-            the base name for a single metadata file.
+            List of strings containing the base names of metadata files inside ``path`` to be indexed.
+            Alternatively, a single string containing the base name for a single metadata file.
 
         url:
             URL to the SewerRat REST API.
@@ -32,6 +28,16 @@ def register(path: str, names: Union[str, List[str]], url: str, retry: int = 3,
 
         wait:
             Deprecated, ignored.
+
+        block:
+            Whether to block on successful registration.
+
+    Returns:
+        On success, the directory is registered. 
+        If a metadata file cannot be indexed (e.g., due to incorrect formatting, insufficient permissions), a warning will be printed but the function will not throw an error.
+
+        If ``block = False``, the function returns before confirmation of successful registration from the SewerRat API.
+        This can be useful for asynchronous processing of directories with many files. 
     """
     if isinstance(names, str):
         names = [names]
@@ -50,13 +56,14 @@ def register(path: str, names: Union[str, List[str]], url: str, retry: int = 3,
         handle.write("")
 
     try:
-        res = requests.post(url + "/register/finish", json = { "path": path, "base": names }, allow_redirects=True)
+        res = requests.post(url + "/register/finish", json = { "path": path, "base": names, "block": block }, allow_redirects=True)
         if res.status_code >= 300:
             raise ut.format_error(res)
         body = res.json()
     finally:
         os.unlink(target)
 
-    for comment in body["comments"]:
-        warnings.warn(comment)
+    if block:
+        for comment in body["comments"]:
+            warnings.warn(comment)
     return

src/sewerrat/start_sewerrat.py

@@ -9,7 +9,7 @@
 test_api_port = None
 
 
-def start_sewerrat(db: Optional[str] = None, port: Optional[int] = None, wait: float = 1, version: str = "1.1.0", overwrite: bool = False) -> Tuple[bool, int]:
+def start_sewerrat(db: Optional[str] = None, port: Optional[int] = None, wait: float = 1, version: str = "1.1.1", overwrite: bool = False) -> Tuple[bool, int]:
     """
     Start a test SewerRat service.
 

tests/test_register.py

@@ -33,3 +33,36 @@ def test_basic():
 
     finally:
         sewerrat.deregister(mydir, url=url)
+
+
+def test_unblocked():
+    mydir = tempfile.mkdtemp()
+    with open(os.path.join(mydir, "metadata.json"), "w") as handle:
+        handle.write('{ "first": "Aaron", "last": "Lun" }')
+
+    os.mkdir(os.path.join(mydir, "diet"))
+    with open(os.path.join(mydir, "diet", "metadata.json"), "w") as handle:
+        handle.write('{ "meal": "lunch", "ingredients": "water" }')
+
+    _, url = sewerrat.start_sewerrat()
+    res = sewerrat.query(url, "aaron")
+    assert len(res) == 0
+
+    try:
+        sewerrat.register(mydir, ["metadata.json"], url=url, block=False)
+        for i in range(10):
+            time.sleep(0.1)
+            res = sewerrat.query(url, "aaron")
+            if len(res) == 1:
+                break
+        assert len(res) == 1
+
+        sewerrat.deregister(mydir, url=url, block=False)
+        for i in range(10):
+            time.sleep(0.1)
+            res = sewerrat.query(url, "aaron")
+            if len(res) == 0:
+                break
+        assert len(res) == 0 
+    finally:
+        sewerrat.deregister(mydir, url=url)