Merge pull request #70 from mlcommons/parameterize-col-names

Parameterize column names

Author: bkorycki

.github/workflows/tests.yml

@@ -38,11 +38,10 @@ jobs:
         run: |
           poetry env remove python || true
 
-      # TODO: get rid of the POETRY_INSTALLER_MAX_WORKERS env var once we get rid of plugins
       - name: Install dependencies
         run: |
           poetry cache clear --no-interaction --all .
-          POETRY_INSTALLER_MAX_WORKERS=1 poetry install --with test --no-cache
+          poetry install --with test --no-cache
 
       - name: Run all tests
         run: MLFLOW_TRACKING_URI=http://localhost:8080 poetry run pytest

flightpaths/Annotator Development Template.ipynb

@@ -147,7 +147,7 @@
    "source": [
     "## Run the model\n",
     "\n",
-    "This step will get responses to the prompts from the given SUT.\n",
+    "This step will get responses to the prompts from the given SUT. You can optionally pass arguments `prompt_uid_col` and `prompt_text_col` if your prompts dataset has different column names than the default ones.\n",
     "\n",
     "Save this run_id to avoid having to re-run the model later. The results are saved as an artifact in mlflow.\n",
     "\n",

poetry.lock

@@ -13,7 +13,7 @@ packages = [
 python = ">=3.10,!=3.12.5,<3.13"
 click = "^8"
 dvc = {extras = ["gs"], version = "^3.60"} 
-modelbench = {git = "https://github.com/mlcommons/modelbench.git", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
+modelbench = {git = "https://github.com/mlcommons/modelbench.git", rev = "a71a902ef2e503569696ea76d56ac40fad9ee97b" }
 mlflow = "^3.1.1"
 python-dotenv = "^1"
 requests = "^2"
@@ -25,18 +25,7 @@ jupyter = "^1"
 jupyterlab-git = "*"
 scikit-learn = "^1.5.0"
 pandas = "^2.2.2"
-# plugins (would like to figure out a better way to manage these)
-modelgauge_anthropic = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/anthropic", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
-modelgauge-azure = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/azure", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
-modelgauge_baseten = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/baseten", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
-modelgauge_demo_plugin = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "demo_plugin", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
-modelgauge_nvidia = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/nvidia", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
-modelgauge_perspective_api = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/perspective_api", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
-modelgauge_google = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/google", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
-modelgauge_vertexai = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/vertexai", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
-modelgauge_mistral = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/mistral", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
-modelgauge_amazon = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/amazon", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" }
-modelbench-private = { git = "[email protected]:mlcommons/modelbench-private.git", rev = "ecac4cfd343411c8011c0bba875d638fc22e9478", optional = true }
+modelbench-private = { git = "[email protected]:mlcommons/modelbench-private.git", rev = "c1e4f214f8a4ac3466a8a428a668e7847c94ee4d", optional = true }
 
 [tool.poetry.extras]
 modelbench-private = ["modelbench-private"]