Hoist cyclic dependency tracking and support merging with known tasks

Author: PeterJCLaw

scripts/element_tracking.py

@@ -0,0 +1,37 @@
+import contextlib
+from typing import Callable, Iterator, MutableMapping, Optional, Set
+
+
+class ElementsInProgress:
+    """
+    Keep track of the tasks as they're being imported.
+
+    This checks for cyclic dependencies and provides a way to consume existing
+    mappings of tasks that are known to be equivalent to those in this repo.
+    """
+
+    def __init__(self, known_elements: Optional[MutableMapping[str, int]]) -> None:
+        self.known_elements = known_elements or {}
+        self._current: Set[str] = set()
+
+    def get(self, key: str) -> Optional[int]:
+        if key in self._current:
+            raise RuntimeError(f"Cyclic dependency on {key}")
+
+        return self.known_elements.get(key)
+
+    @contextlib.contextmanager
+    def process(self, key: str) -> Iterator[Callable[[int], None]]:
+        if key in self._current:
+            raise RuntimeError(
+                f"Re-entrant processing not supported (attempted {key})",
+            )
+
+        def done(x: int) -> None:
+            self.known_elements[key] = x
+
+        self._current.add(key)
+        try:
+            yield done
+        finally:
+            self._current.remove(key)

scripts/import.py

@@ -2,9 +2,10 @@
 
 import argparse
 from pathlib import Path
-from typing import TYPE_CHECKING, Callable, Dict, Union
+from typing import TYPE_CHECKING, Callable, MutableMapping, Optional
 
 import yaml
+from element_tracking import ElementsInProgress
 from import_backends import FakeTracBackend, GitHubBackend, RealTracBackend
 from ticket_type import Ticket
 
@@ -79,7 +80,13 @@ def process(element_name: str, *, year: str, handle_dep: Callable[[str], int]) -
     return ticket
 
 
-def add(element: str, backend: 'Backend', year: str) -> int:
+def add(
+    element: str,
+    backend: 'Backend',
+    year: str,
+    *,
+    known_elements: Optional[MutableMapping[str, int]] = None,
+) -> int:
     """
     Add 'element' into the task tracker, along with all its dependencies.
 
@@ -92,22 +99,19 @@ def add(element: str, backend: 'Backend', year: str) -> int:
     dependencies which have already been imported at the point they are
     depended upon by a new parent.
     """
-    CYCLE = object()
-    elements: Dict[str, Union[int, object]] = {}
+    elements = ElementsInProgress(known_elements)
 
     def _add(element: str) -> int:
-        if element in elements:
-            previous = elements[element]
-            if previous is CYCLE:
-                raise RuntimeError(f"cyclic dependency on {element}")
-            assert isinstance(previous, int)
+        previous = elements.get(element)
+        if previous:
             return previous
-        else:
-            elements[element] = CYCLE
+
+        with elements.process(element) as record_id:
             generated = process(element, year=year, handle_dep=_add)
             ticket_id = backend.submit(generated)
-            elements[element] = ticket_id
-            return ticket_id
+            record_id(ticket_id)
+
+        return ticket_id
 
     return _add(element)