feat(models): improve models API (#268)

Reimplement programmatic access to models for looser coupling and better developer/user experience. Instead of baking a model registry into the package, let repositories publish their own registries, discoverable by devtools. Allows devtools and model repositories to move independently, and allows explicit model versioning. See the design document for more detail.

Author: wpbonelli

.github/workflows/ci.yml

@@ -105,7 +105,7 @@ jobs:
         env:
           REPOS_PATH: ${{ github.workspace }}
         # use --dist loadfile to so tests requiring pytest-virtualenv run on the same worker
-        run: uv run pytest -v -n auto --dist loadfile --durations 0 --ignore test_download.py
+        run: uv run pytest -v -n auto --dist loadfile --durations 0 --ignore test_download.py --ignore test_models.py
 
       - name: Run network-dependent tests
         # only invoke the GH API on one OS and Python version
@@ -116,7 +116,11 @@ jobs:
         env:
           REPOS_PATH: ${{ github.workspace }}
           GITHUB_TOKEN: ${{ github.token }}
-        run: uv run pytest -v -n auto --durations 0 test_download.py
+          TEST_REPO: wpbonelli/modflow6-testmodels
+          TEST_REF: registry
+          TEST_SOURCE: modflow6-testmodels
+          TEST_SOURCE_NAME: mf6/test
+        run: uv run pytest -v -n auto --dist loadgroup --durations 0 test_download.py test_models.py
 
   rtd:
     name: Docs

autotest/mf6examples.zip.lock

@@ -15,15 +15,11 @@
 _modflow6_repo_path = _repos_path / "modflow6"
 _system = platform.system()
 _exe_ext = ".exe" if _system == "Windows" else ""
-_lib_ext = (
-    ".so" if _system == "Linux" else (".dylib" if _system == "Darwin" else ".dll")
-)
+_lib_ext = ".so" if _system == "Linux" else (".dylib" if _system == "Darwin" else ".dll")
 
 
 @requires_exe("meson", "ninja")
-@pytest.mark.skipif(
-    not _modflow6_repo_path.is_dir(), reason="modflow6 repository not found"
-)
+@pytest.mark.skipif(not _modflow6_repo_path.is_dir(), reason="modflow6 repository not found")
 def test_meson_build(tmp_path):
     build_path = tmp_path / "builddir"
     bin_path = tmp_path / "bin"

autotest/test_build.py

@@ -20,9 +20,7 @@ def pytest_generate_tests(metafunc):
         if not any(DFN_DIR.glob("*.dfn")):
             get_dfns(MF6_OWNER, MF6_REPO, MF6_REF, DFN_DIR, verbose=True)
         dfn_names = [
-            dfn.stem
-            for dfn in DFN_DIR.glob("*.dfn")
-            if dfn.stem not in ["common", "flopy"]
+            dfn.stem for dfn in DFN_DIR.glob("*.dfn") if dfn.stem not in ["common", "flopy"]
         ]
         metafunc.parametrize("dfn_name", dfn_names, ids=dfn_names)
 

autotest/test_dfn.py

@@ -45,7 +45,7 @@ def test_get_release(repo):
     tag = "latest"
     release = get_release(repo, tag, verbose=True)
     assets = release["assets"]
-    expected_names = ["linux.zip", "mac.zip", "win64.zip"]
+    expected_names = ["linux.zip", "macarm.zip", "win64.zip"]
     actual_names = [asset["name"] for asset in assets]
 
     if repo == "MODFLOW-ORG/modflow6":

autotest/test_download.py

@@ -55,8 +55,7 @@ def test_function_scoped_tmpdir_slash_in_name(function_tmpdir, name):
     )
     assert (
         f"{inspect.currentframe().f_code.co_name}_{replaced1}_" in function_tmpdir.stem
-        or f"{inspect.currentframe().f_code.co_name}_{replaced2}_"
-        in function_tmpdir.stem
+        or f"{inspect.currentframe().f_code.co_name}_{replaced2}_" in function_tmpdir.stem
     )
 
 
@@ -157,9 +156,7 @@ def test_keep_class_scoped_tmpdir(tmp_path, arg):
         tmp_path,
     ]
     assert pytest.main(args) == ExitCode.OK
-    assert (
-        tmp_path / f"{TestKeepClassScopedTmpdirInner.__name__}0" / test_keep_fname
-    ).is_file()
+    assert (tmp_path / f"{TestKeepClassScopedTmpdirInner.__name__}0" / test_keep_fname).is_file()
 
 
 @pytest.mark.parametrize("arg", ["--keep", "-K"])

autotest/test_fixtures.py

@@ -43,10 +43,11 @@ def test_set_env():
             assert environ.get(key) is None
 
 
-_repos_path = environ.get("REPOS_PATH")
-if _repos_path is None:
-    _repos_path = Path(__file__).parent.parent.parent.parent
-_repos_path = Path(_repos_path).expanduser().absolute()
+_repos_path_str = environ.get("REPOS_PATH")
+if _repos_path_str is None:
+    _repos_path: Path = Path(__file__).parent.parent.parent.parent
+else:
+    _repos_path = Path(_repos_path_str).expanduser().absolute()
 _testmodels_repo_path = _repos_path / "modflow6-testmodels"
 _testmodels_repo_paths_mf6 = sorted((_testmodels_repo_path / "mf6").glob("test*"))
 _testmodels_repo_paths_mf5to6 = sorted((_testmodels_repo_path / "mf5to6").glob("test*"))
@@ -57,9 +58,7 @@ def test_set_env():
 _example_paths = sorted(_examples_path.glob("ex-*")) if _examples_path.is_dir() else []
 
 
-@pytest.mark.skipif(
-    not any(_testmodels_repo_paths_mf6), reason="mf6 test models not found"
-)
+@pytest.mark.skipif(not any(_testmodels_repo_paths_mf6), reason="mf6 test models not found")
 def test_get_packages():
     model_path = _testmodels_repo_paths_mf6[0]
 
@@ -79,9 +78,7 @@ def test_get_packages():
     assert "ims" not in packages
 
 
-@pytest.mark.skipif(
-    not any(_testmodels_repo_paths_mf6), reason="mf6 test models not found"
-)
+@pytest.mark.skipif(not any(_testmodels_repo_paths_mf6), reason="mf6 test models not found")
 def test_get_packages_fails_on_invalid_namefile(module_tmpdir):
     model_path = _testmodels_repo_paths_mf6[0]
     new_model_path = module_tmpdir / model_path.name
@@ -132,7 +129,7 @@ def get_expected_model_dirs(path, pattern="mfsim.nam") -> list[Path]:
 
 
 def get_expected_namefiles(path, pattern="mfsim.nam") -> list[Path]:
-    folders = []
+    folders: list[Path] = []
     for root, dirs, _ in os.walk(path):
         for d in dirs:
             p = Path(root) / d
@@ -154,9 +151,7 @@ def test_get_model_paths_examples():
     assert set(expected_paths) == set(paths)
 
 
-@pytest.mark.skipif(
-    not any(_largetestmodel_paths), reason="modflow6-largetestmodels not found"
-)
+@pytest.mark.skipif(not any(_largetestmodel_paths), reason="modflow6-largetestmodels not found")
 def test_get_model_paths_largetestmodels():
     expected_paths = get_expected_model_dirs(_examples_path)
     paths = get_model_paths(_examples_path)
@@ -173,9 +168,7 @@ def test_get_model_paths_largetestmodels():
     not any(_largetestmodel_paths) or not any(_example_paths),
     reason="repos not found",
 )
-@pytest.mark.parametrize(
-    "models", [(_examples_path, 63), (_largetestmodels_repo_path, 16)]
-)
+@pytest.mark.parametrize("models", [(_examples_path, 63), (_largetestmodels_repo_path, 16)])
 def test_get_model_paths_exclude_patterns(models):
     path, expected_count = models
     paths = get_model_paths(path, excluded=["gwt"])
@@ -195,9 +188,7 @@ def test_get_namefile_paths_examples():
     assert set(expected_paths) == set(paths)
 
 
-@pytest.mark.skipif(
-    not any(_largetestmodel_paths), reason="modflow6-largetestmodels not found"
-)
+@pytest.mark.skipif(not any(_largetestmodel_paths), reason="modflow6-largetestmodels not found")
 def test_get_namefile_paths_largetestmodels():
     expected_paths = get_expected_namefiles(_largetestmodels_repo_path)
     paths = get_namefile_paths(_largetestmodels_repo_path)
@@ -214,9 +205,7 @@ def test_get_namefile_paths_largetestmodels():
     not any(_largetestmodel_paths) or not any(_example_paths),
     reason="repos not found",
 )
-@pytest.mark.parametrize(
-    "models", [(_examples_path, 43), (_largetestmodels_repo_path, 19)]
-)
+@pytest.mark.parametrize("models", [(_examples_path, 43), (_largetestmodels_repo_path, 19)])
 def test_get_namefile_paths_exclude_patterns(models):
     path, expected_count = models
     paths = get_namefile_paths(path, excluded=["gwf"])