feat: Add migration utilities and fix flaky job tests (#1332)

* feat: add Phase 2 migration functions for column type and blob markers

Add migrate_columns() and supporting functions for Phase 2 of the
0.14.6 → 2.0 migration:

- analyze_columns(): Identify columns needing type labels
- migrate_columns(): Add core type markers to column comments
- NATIVE_TO_CORE_TYPE mapping for type conversion
- Support for bool/datetime special cases
- Dry-run mode for previewing changes

Also adds placeholder stubs for Phase 3-4 migration functions:
- migrate_external(): For external storage migration
- migrate_filepath(): For filepath attribute migration
- finalize_migration(): For Phase 4 finalization

These functions implement the migration guide documented in
datajoint-docs/src/how-to/migrate-from-0x.md.

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* fix: resolve flaky tests by using delay=-1 for immediate job scheduling

The tests test_sigint, test_sigterm, test_suppress_dj_errors, and
test_populate_exclude_error_and_ignore_jobs were flaky due to a race
condition: jobs created with scheduled_time=NOW(3) might not pass the
scheduled_time <= NOW(3) check if checked in the same millisecond.

Fix by using delay=-1 in auto-refresh during populate(), ensuring jobs
are scheduled 1 second in the past and immediately schedulable.

Also update test_populate_exclude_error_and_ignore_jobs to use delay=-1
in its explicit refresh() call.

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* fix: address PR #1311 review comments

- Replace bare except: with except ImportError: in diagram.py
- Replace assert statements with explicit raises in blob.py
- Replace assert False with explicit raises in expression.py and declare.py
- Implement hash verification in objectref.py

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* fix: remove deprecated dj.key from __all__

dj.key was removed in 2.0 but was still listed in __all__ without
being imported or defined.

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* fix: remove deprecated dj.schema and dj.Di aliases

- Remove `schema` alias for `Schema` (use `dj.Schema` instead of `dj.schema`)
- Remove `Di` alias for `Diagram` (use `dj.Diagram` or `dj.ERD`)
- Update all examples, tests, and docstrings

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* docs: fix broken documentation links

Update outdated docs.datajoint.com URLs to new paths:
- diagram.py: /how-to/installation/
- heading.py: /how-to/migrate-from-0x/
- expression.py: /how-to/migrate-from-0x/

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* chore: bump version to 2.0.0a19

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* fix: remove deprecated dj.key_hash and dj.Di

- Remove key_hash function (legacy job table debugging)
- Remove hash.py module
- Fix test_erd.py to use dj.Diagram instead of dj.Di
- Bump version to 2.0.0a20

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* fix: remove deprecated dj.ERD alias

- Remove ERD alias (use dj.Diagram)
- Rename test_erd_algebra to test_diagram_algebra
- Remove test_diagram_aliases test
- Bump version to 2.0.0a21

Co-Authored-By: Claude Opus 4.5 <[email protected]>

---------

Co-authored-by: Claude Opus 4.5 <[email protected]>

Author: dimitri-yatsenko

src/datajoint/__init__.py

@@ -24,7 +24,6 @@
     "conn",
     "Connection",
     "Schema",
-    "schema",
     "VirtualModule",
     "virtual_schema",
     "list_schemas",
@@ -40,8 +39,6 @@
     "Top",
     "U",
     "Diagram",
-    "Di",
-    "ERD",
     "kill",
     "MatCell",
     "MatStruct",
@@ -56,8 +53,6 @@
     "errors",
     "migrate",
     "DataJointError",
-    "key",
-    "key_hash",
     "logger",
     "cli",
     "ValidationResult",
@@ -81,7 +76,6 @@
 from .connection import Connection, conn
 from .errors import DataJointError
 from .expression import AndList, Not, Top, U
-from .hash import key_hash
 from .logging import logger
 from .objectref import ObjectRef
 from .schemas import Schema, VirtualModule, list_schemas, virtual_schema
@@ -90,8 +84,6 @@
 from .user_tables import Computed, Imported, Lookup, Manual, Part
 from .version import __version__
 
-schema = Schema  # Alias for Schema
-
 # =============================================================================
 # Lazy imports — heavy dependencies loaded on first access
 # =============================================================================
@@ -101,8 +93,6 @@
 _lazy_modules = {
     # Diagram imports networkx and matplotlib
     "Diagram": (".diagram", "Diagram"),
-    "Di": (".diagram", "Diagram"),
-    "ERD": (".diagram", "Diagram"),
     "diagram": (".diagram", None),  # Return the module itself
     # kill imports pymysql via connection
     "kill": (".admin", "kill"),

src/datajoint/autopopulate.py

@@ -485,7 +485,9 @@ def handler(signum, frame):
             if refresh is None:
                 refresh = config.jobs.auto_refresh
             if refresh:
-                self.jobs.refresh(*restrictions, priority=priority)
+                # Use delay=-1 to ensure jobs are immediately schedulable
+                # (avoids race condition with scheduled_time <= NOW(3) check)
+                self.jobs.refresh(*restrictions, priority=priority, delay=-1)
 
             # Fetch pending jobs ordered by priority (use NOW(3) to match CURRENT_TIMESTAMP(3) precision)
             pending_query = self.jobs.pending & "scheduled_time <= NOW(3)"

src/datajoint/blob.py

@@ -159,7 +159,8 @@ def unpack(self, blob):
             self._pos += len(prefix)
             blob_size = self.read_value()
             blob = compression[prefix](self._blob[self._pos :])
-            assert len(blob) == blob_size
+            if len(blob) != blob_size:
+                raise DataJointError(f"Blob size mismatch: expected {blob_size}, got {len(blob)}")
             self._blob = blob
             self._pos = 0
         blob_format = self.read_zero_terminated_string()
@@ -363,7 +364,8 @@ def read_int(self):
     @staticmethod
     def pack_int(v):
         n_bytes = v.bit_length() // 8 + 1
-        assert 0 < n_bytes <= 0xFFFF, "Integers are limited to 65535 bytes"
+        if not (0 < n_bytes <= 0xFFFF):
+            raise DataJointError("Integers are limited to 65535 bytes")
         return b"\x0a" + np.uint16(n_bytes).tobytes() + v.to_bytes(n_bytes, byteorder="little", signed=True)
 
     def read_bool(self):

src/datajoint/connection.py

@@ -299,7 +299,7 @@ def __enter__(self) -> "Connection":
         Examples
         --------
         >>> with dj.Connection(host, user, password) as conn:
-        ...     schema = dj.schema('my_schema', connection=conn)
+        ...     schema = dj.Schema('my_schema', connection=conn)
         ...     # perform operations
         ... # connection automatically closed
         """

src/datajoint/declare.py

@@ -673,7 +673,7 @@ def substitute_special_type(match: dict, category: str, foreign_key_sql: list[st
             match["type"] = sql_type
         # else: type passes through as-is (json, date, datetime, char, varchar, enum)
     else:
-        assert False, f"Unknown special type: {category}"
+        raise DataJointError(f"Unknown special type: {category}")
 
 
 def compile_attribute(line: str, in_key: bool, foreign_key_sql: list[str], context: dict) -> tuple[str, str, str | None]:

src/datajoint/diagram.py

@@ -23,14 +23,14 @@
     from matplotlib import pyplot as plt
 
     plot_active = True
-except:
+except ImportError:
     plot_active = False
 
 try:
     from networkx.drawing.nx_pydot import pydot_layout
 
     diagram_active = True
-except:
+except ImportError:
     diagram_active = False
 
 
@@ -48,7 +48,7 @@ class Diagram:
 
         See Also
         --------
-        https://docs.datajoint.com/core/datajoint-python/0.14/client/install/
+        https://docs.datajoint.com/how-to/installation/
         """
 
         def __init__(self, *args, **kwargs) -> None:

src/datajoint/expression.py

@@ -596,7 +596,7 @@ def fetch(self):
 
         For single-row fetch, use fetch1() which is unchanged.
 
-        See migration guide: https://docs.datajoint.com/migration/fetch-api
+        See migration guide: https://docs.datajoint.com/how-to/migrate-from-0x/
         """
         raise AttributeError(
             "fetch() has been removed in DataJoint 2.0. "
@@ -1085,12 +1085,12 @@ def make_sql(self):
         return "({sql1})  UNION ({sql2})".format(sql1=sql1, sql2=sql2)
 
     def from_clause(self):
-        """The union does not use a FROM clause"""
-        assert False
+        """The union does not use a FROM clause."""
+        raise NotImplementedError("Union does not use a FROM clause")
 
     def where_clause(self):
-        """The union does not use a WHERE clause"""
-        assert False
+        """The union does not use a WHERE clause."""
+        raise NotImplementedError("Union does not use a WHERE clause")
 
     def __len__(self):
         return self.connection.query(

src/datajoint/hash.py

@@ -466,8 +466,8 @@ def _init_from_database(self) -> None:
                 except StopIteration:
                     if original_type.startswith("external"):
                         raise DataJointError(
-                            f"Legacy datatype `{original_type}`. Migrate your external stores to datajoint 0.12: "
-                            "https://docs.datajoint.io/python/admin/5-blob-config.html#migration-between-datajoint-v0-11-and-v0-12"
+                            f"Legacy datatype `{original_type}`. See migration guide: "
+                            "https://docs.datajoint.com/how-to/migrate-from-0x/"
                         )
                     # Not a special type - that's fine, could be native passthrough
                     category = None