Add caching and filtering to gen_models. (#888)

Adds the following flags to `tools/gen_models.py`:
- `-k` allows filtering for tests, similar to pytest
- `--cache` and `--no-cache` which allows skipping schemas which have
not changed
    - If caching, the schemas are copied to `.gen_models_cache`

Author: dnwpark

.gitignore

@@ -36,3 +36,4 @@ docs/_build
 /uv.lock
 .dir-locals.el
 .envrc
+.gen_models_cache

gel/_testbase.py

@@ -35,7 +35,6 @@
 import pickle
 import re
 import shutil
-import site
 import subprocess
 import sys
 import tempfile
@@ -65,12 +64,6 @@
 _unset = object()
 
 
-SITE_PACKAGES: typing.Final[pathlib.Path] = pathlib.Path(
-    site.getsitepackages()[0]
-)
-MODELS_DEST: typing.Final[pathlib.Path] = SITE_PACKAGES / "models"
-
-
 @contextlib.contextmanager
 def silence_asyncio_long_exec_warning():
     def flt(log_record):
@@ -635,7 +628,7 @@ def get_database_name(cls):
         return dbname.lower()
 
     @classmethod
-    def get_schema_texts(cls) -> list[str]:
+    def get_combined_schemas(cls) -> str:
         schema_texts: list[str] = []
 
         # Look at all SCHEMA entries and potentially create multiple
@@ -647,7 +640,11 @@ def get_schema_texts(cls) -> list[str]:
             if schema_text := cls.get_schema_text(name):
                 schema_texts.append(schema_text)
 
-        return schema_texts
+        return "\n\n".join(st for st in schema_texts)
+
+    @classmethod
+    def is_schema_field(cls, field: str) -> bool:
+        return bool(re.match(r"^SCHEMA(?:_(\w+))?", field))
 
     @classmethod
     def get_schema_text(cls, field: str) -> str | None:
@@ -678,12 +675,11 @@ def get_schema_text(cls, field: str) -> str | None:
     @classmethod
     def get_setup_script(cls):
         script = ""
-        schema = "\n\n".join(st for st in cls.get_schema_texts())
 
         # Don't wrap the script into a transaction here, so that
         # potentially it's easier to stitch multiple such scripts
         # together in a fashion similar to what `edb inittestdb` does.
-        script += f"\nSTART MIGRATION TO {{ {schema} }};"
+        script += f"\nSTART MIGRATION TO {{ {cls.get_combined_schemas()} }};"
         script += f"\nPOPULATE MIGRATION; \nCOMMIT MIGRATION;"
 
         if cls.SETUP:
@@ -763,6 +759,7 @@ def adapt_call(cls, result):
 
 
 class BaseModelTestCase(DatabaseTestCase):
+    SCHEMA: str
     DEFAULT_MODULE = "default"
 
     client: typing.ClassVar[gel.Client]
@@ -790,7 +787,7 @@ def setUpClass(cls):
 
         cls.tmp_model_dir = tempfile.TemporaryDirectory(**td_kwargs)
 
-        model_from_file, model_name = cls._model_info()
+        _, model_name = cls._model_info()
 
         if cls.orm_debug:
             print(cls.tmp_model_dir.name)
@@ -830,15 +827,6 @@ def setUpClass(cls):
         finally:
             gen_client.terminate()
 
-        if not model_from_file:
-            # This is a direct schema, let's copy it to the site paths
-            site_model_dir = MODELS_DEST / model_name
-            if site_model_dir.exists():
-                shutil.rmtree(site_model_dir)
-            shutil.copytree(
-                model_output_dir, site_model_dir, dirs_exist_ok=True
-            )
-
         sys.path.insert(0, cls.tmp_model_dir.name)
 
         import models

tests/test_model_generator.py

@@ -6583,6 +6583,8 @@ def test_model_gen_direct_01(self):
     def test_model_gen_direct_02(self):
         from models.TestModelGenDirect import default
 
+        # This still works because the actual test runs with a model generated
+        # after SETUP.
         bar = default.Bar(x=12)
         self.client.sync(bar)