Support prompt engineered annotators backed by dynamic SUTs

.github/workflows/tests.yml

@@ -38,10 +38,11 @@ jobs:
         run: |
           poetry env remove python || true
 
+      # TODO: get rid of the POETRY_INSTALLER_MAX_WORKERS env var once we get rid of plugins
       - name: Install dependencies
         run: |
           poetry cache clear --no-interaction --all .
-          poetry install --with test --no-cache
+          POETRY_INSTALLER_MAX_WORKERS=1 poetry install --with test --no-cache
 
       - name: Run all tests
         run: MLFLOW_TRACKING_URI=http://localhost:8080 poetry run pytest

pyproject.toml

@@ -13,7 +13,7 @@ packages = [
 python = ">=3.10,!=3.12.5,<3.13"
 click = "^8"
 dvc = {extras = ["gs"], version = "^3.60"} 
-modelbench = {git = "https://github.com/mlcommons/modelbench.git", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
+modelbench = {git = "https://github.com/mlcommons/modelbench.git", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
 mlflow = "^3.1.1"
 python-dotenv = "^1"
 requests = "^2"
@@ -25,19 +25,17 @@ jupyter = "^1"
 scikit-learn = "^1.5.0"
 pandas = "^2.2.2"
 # plugins (would like to figure out a better way to manage these)
-modelgauge_anthropic = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/anthropic", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
-modelgauge-azure = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/azure", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
-modelgauge_baseten = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/baseten", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
-modelgauge_demo_plugin = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "demo_plugin", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
-modelgauge_nvidia = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/nvidia", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
-modelgauge_openai = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/openai", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
-modelgauge_huggingface = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/huggingface", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
-modelgauge_perspective_api = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/perspective_api", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
-modelgauge_google = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/google", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
-modelgauge_vertexai = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/vertexai", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
-modelgauge_mistral = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/mistral", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
-modelgauge_amazon = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/amazon", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" }
-modelbench-private = { git = "[email protected]:mlcommons/modelbench-private.git", rev = "982973bd51e6c7ebb1bebaaa1eeb6a297d871bb3", optional = true }
+modelgauge_anthropic = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/anthropic", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
+modelgauge-azure = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/azure", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
+modelgauge_baseten = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/baseten", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
+modelgauge_demo_plugin = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "demo_plugin", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
+modelgauge_nvidia = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/nvidia", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
+modelgauge_perspective_api = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/perspective_api", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
+modelgauge_google = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/google", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
+modelgauge_vertexai = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/vertexai", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
+modelgauge_mistral = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/mistral", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
+modelgauge_amazon = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/amazon", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
+modelbench-private = { git = "[email protected]:mlcommons/modelbench-private.git", rev = "ecac4cfd343411c8011c0bba875d638fc22e9478", optional = true }
 
 [tool.poetry.extras]
 modelbench-private = ["modelbench-private"]
@@ -50,7 +48,7 @@ pytest = "^8"
 pytest-cov = "^6"
 
 [tool.poetry.scripts]
-modelplane = "modelplane.runways.run:cli"
+modelplane = "modelplane.cli:cli"
 
 [build-system]
 requires = ["poetry-core"]

src/modelplane/__init__.py

@@ -0,0 +1,3 @@
+from modelgauge.config import write_default_config
+
+write_default_config()

src/modelplane/runways/lister.py

@@ -1,16 +1,15 @@
 from modelgauge.annotator_registry import ANNOTATORS
-from modelgauge.command_line import compact_sut_list
 from modelgauge.ensemble_annotator_set import ENSEMBLE_STRATEGIES
 from modelgauge.sut_registry import SUTS
 
 
 def list_annotators():
-    print(compact_sut_list(ANNOTATORS))
+    print(ANNOTATORS.compact_uid_list())
 
 
 def list_suts():
-    print(compact_sut_list(SUTS))
+    print(SUTS.compact_uid_list())
 
 
 def list_ensemble_strategies():
-    print(compact_sut_list(ENSEMBLE_STRATEGIES))
+    print(sorted(ENSEMBLE_STRATEGIES))

tests/it/test_cli.py

@@ -1,7 +1,7 @@
 import pytest
 from click.testing import CliRunner
 
-from modelplane.runways.run import cli
+from modelplane.cli import cli
 
 
 def test_main_help():