diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ef0d7b2..b5dbf1f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,10 +38,11 @@ jobs: run: | poetry env remove python || true + # TODO: get rid of the POETRY_INSTALLER_MAX_WORKERS env var once we get rid of plugins - name: Install dependencies run: | poetry cache clear --no-interaction --all . - poetry install --with test --no-cache + POETRY_INSTALLER_MAX_WORKERS=1 poetry install --with test --no-cache - name: Run all tests run: MLFLOW_TRACKING_URI=http://localhost:8080 poetry run pytest diff --git a/poetry.lock b/poetry.lock index 2318718..8501d71 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4269,12 +4269,15 @@ click = "^8.1.7" diskcache = "^5.6.3" fastapi = "^0.115.0" gdown = ">=5.1.0" +huggingface-hub = "^0.30.2" jinja2 = "^3.1.3" jq = "^1.6.0" jsonlines = "^4.0.0" llama-api-client = "^0.1.1" +openai = "^1.8.0" pip = ">=24,<26" prometheus-client = "^0.21.1" +pyarrow = "^20.0" pydantic = "^2.6.0" retry = "^0.9.2" scipy = "^1.12.0" @@ -4291,25 +4294,23 @@ typing-extensions = "^4.10.0" zstandard = {version = "^0.23.0", extras = ["cffi"]} [package.extras] -all-plugins = ["modelgauge_amazon", "modelgauge_anthropic", "modelgauge_azure", "modelgauge_baseten", "modelgauge_demo_plugin", "modelgauge_google", "modelgauge_huggingface", "modelgauge_mistral", "modelgauge_nvidia", "modelgauge_openai", "modelgauge_perspective_api", "modelgauge_vertexai"] -amazon = ["modelgauge_amazon"] -anthropic = ["modelgauge_anthropic"] -azure = ["modelgauge_azure"] -baseten = ["modelgauge_baseten"] -demo = ["modelgauge_demo_plugin"] -google = ["modelgauge_google"] -huggingface = ["modelgauge_huggingface"] -mistral = ["modelgauge_mistral"] -nvidia = ["modelgauge_nvidia"] -openai = ["modelgauge_openai"] -perspective-api = ["modelgauge_perspective_api"] -vertexai = ["modelgauge_vertexai"] +all-plugins = ["modelgauge_amazon @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/amazon", "modelgauge_anthropic @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/anthropic", "modelgauge_azure @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/azure", "modelgauge_baseten @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/baseten", "modelgauge_demo_plugin @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/demo_plugin", "modelgauge_google @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/google", "modelgauge_mistral @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/mistral", "modelgauge_nvidia @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/nvidia", "modelgauge_perspective_api @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/perspective_api", "modelgauge_vertexai @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/vertexai"] +amazon = ["modelgauge_amazon @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/amazon"] +anthropic = ["modelgauge_anthropic @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/anthropic"] +azure = ["modelgauge_azure @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/azure"] +baseten = ["modelgauge_baseten @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/baseten"] +demo = ["modelgauge_demo_plugin @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/demo_plugin"] +google = ["modelgauge_google @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/google"] +mistral = ["modelgauge_mistral @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/mistral"] +nvidia = ["modelgauge_nvidia @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/nvidia"] +perspective-api = ["modelgauge_perspective_api @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/perspective_api"] +vertexai = ["modelgauge_vertexai @ file:///Users/vishal/Library/Caches/pypoetry/virtualenvs/modelplane-8xT__OfZ-py3.12/src/modelbench/plugins/vertexai"] [package.source] type = "git" url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" +reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" +resolved_reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" [[package]] name = "modelbench-private" @@ -4322,13 +4323,12 @@ develop = false [package.extras] all-plugins = [] -modelbench = [] [package.source] type = "git" url = "git@github.com:mlcommons/modelbench-private.git" -reference = "982973bd51e6c7ebb1bebaaa1eeb6a297d871bb3" -resolved_reference = "982973bd51e6c7ebb1bebaaa1eeb6a297d871bb3" +reference = "ecac4cfd343411c8011c0bba875d638fc22e9478" +resolved_reference = "ecac4cfd343411c8011c0bba875d638fc22e9478" [[package]] name = "modelgauge-amazon" @@ -4345,8 +4345,8 @@ boto3 = "^1.36.25" [package.source] type = "git" url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" +reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" +resolved_reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" subdirectory = "plugins/amazon" [[package]] @@ -4360,13 +4360,12 @@ develop = false [package.dependencies] anthropic = "*" -modelgauge_openai = "*" [package.source] type = "git" url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" +reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" +resolved_reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" subdirectory = "plugins/anthropic" [[package]] @@ -4384,8 +4383,8 @@ azure-ai-ml = "^1.22" [package.source] type = "git" url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" +reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" +resolved_reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" subdirectory = "plugins/azure" [[package]] @@ -4400,8 +4399,8 @@ develop = false [package.source] type = "git" url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" +reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" +resolved_reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" subdirectory = "plugins/baseten" [[package]] @@ -4416,8 +4415,8 @@ develop = false [package.source] type = "git" url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" +reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" +resolved_reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" subdirectory = "demo_plugin" [[package]] @@ -4436,29 +4435,10 @@ google-generativeai = "^0.8.0" [package.source] type = "git" url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" +reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" +resolved_reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" subdirectory = "plugins/google" -[[package]] -name = "modelgauge-huggingface" -version = "1.0.0" -description = "" -optional = false -python-versions = "^3.10" -files = [] -develop = false - -[package.dependencies] -huggingface-hub = "^0.30.2" - -[package.source] -type = "git" -url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -subdirectory = "plugins/huggingface" - [[package]] name = "modelgauge-mistral" version = "1.0.0" @@ -4475,8 +4455,8 @@ typing-inspect = "^0.9.0" [package.source] type = "git" url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" +reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" +resolved_reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" subdirectory = "plugins/mistral" [[package]] @@ -4494,29 +4474,10 @@ openai = "^1.8.0" [package.source] type = "git" url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" +reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" +resolved_reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" subdirectory = "plugins/nvidia" -[[package]] -name = "modelgauge-openai" -version = "1.0.0" -description = "" -optional = false -python-versions = "^3.10" -files = [] -develop = false - -[package.dependencies] -openai = "^1.8.0" - -[package.source] -type = "git" -url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -subdirectory = "plugins/openai" - [[package]] name = "modelgauge-perspective-api" version = "1.0.0" @@ -4532,8 +4493,8 @@ google-api-python-client = ">=2.64.0,<2.65.0" [package.source] type = "git" url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" +reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" +resolved_reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" subdirectory = "plugins/perspective_api" [[package]] @@ -4551,8 +4512,8 @@ google-auth = "^2.36.0" [package.source] type = "git" url = "https://github.com/mlcommons/modelbench.git" -reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" -resolved_reference = "afe1cbfa42eebe3363440d1ab5841c007384e244" +reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" +resolved_reference = "ba582ed35a58cf6219786624c83e6915a9bcf98a" subdirectory = "plugins/vertexai" [[package]] @@ -5908,53 +5869,66 @@ files = [ [[package]] name = "pyarrow" -version = "19.0.1" +version = "20.0.0" description = "Python library for Apache Arrow" optional = false python-versions = ">=3.9" files = [ - {file = "pyarrow-19.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:fc28912a2dc924dddc2087679cc8b7263accc71b9ff025a1362b004711661a69"}, - {file = "pyarrow-19.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec"}, - {file = "pyarrow-19.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad76aef7f5f7e4a757fddcdcf010a8290958f09e3470ea458c80d26f4316ae89"}, - {file = "pyarrow-19.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d03c9d6f2a3dffbd62671ca070f13fc527bb1867b4ec2b98c7eeed381d4f389a"}, - {file = "pyarrow-19.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:65cf9feebab489b19cdfcfe4aa82f62147218558d8d3f0fc1e9dea0ab8e7905a"}, - {file = "pyarrow-19.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:41f9706fbe505e0abc10e84bf3a906a1338905cbbcf1177b71486b03e6ea6608"}, - {file = "pyarrow-19.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:c6cb2335a411b713fdf1e82a752162f72d4a7b5dbc588e32aa18383318b05866"}, - {file = "pyarrow-19.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:cc55d71898ea30dc95900297d191377caba257612f384207fe9f8293b5850f90"}, - {file = "pyarrow-19.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:7a544ec12de66769612b2d6988c36adc96fb9767ecc8ee0a4d270b10b1c51e00"}, - {file = "pyarrow-19.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0148bb4fc158bfbc3d6dfe5001d93ebeed253793fff4435167f6ce1dc4bddeae"}, - {file = "pyarrow-19.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f24faab6ed18f216a37870d8c5623f9c044566d75ec586ef884e13a02a9d62c5"}, - {file = "pyarrow-19.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4982f8e2b7afd6dae8608d70ba5bd91699077323f812a0448d8b7abdff6cb5d3"}, - {file = "pyarrow-19.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:49a3aecb62c1be1d822f8bf629226d4a96418228a42f5b40835c1f10d42e4db6"}, - {file = "pyarrow-19.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466"}, - {file = "pyarrow-19.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:80b2ad2b193e7d19e81008a96e313fbd53157945c7be9ac65f44f8937a55427b"}, - {file = "pyarrow-19.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:ee8dec072569f43835932a3b10c55973593abc00936c202707a4ad06af7cb294"}, - {file = "pyarrow-19.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d5d1ec7ec5324b98887bdc006f4d2ce534e10e60f7ad995e7875ffa0ff9cb14"}, - {file = "pyarrow-19.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3ad4c0eb4e2a9aeb990af6c09e6fa0b195c8c0e7b272ecc8d4d2b6574809d34"}, - {file = "pyarrow-19.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d383591f3dcbe545f6cc62daaef9c7cdfe0dff0fb9e1c8121101cabe9098cfa6"}, - {file = "pyarrow-19.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b4c4156a625f1e35d6c0b2132635a237708944eb41df5fbe7d50f20d20c17832"}, - {file = "pyarrow-19.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:5bd1618ae5e5476b7654c7b55a6364ae87686d4724538c24185bbb2952679960"}, - {file = "pyarrow-19.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e45274b20e524ae5c39d7fc1ca2aa923aab494776d2d4b316b49ec7572ca324c"}, - {file = "pyarrow-19.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:d9dedeaf19097a143ed6da37f04f4051aba353c95ef507764d344229b2b740ae"}, - {file = "pyarrow-19.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ebfb5171bb5f4a52319344ebbbecc731af3f021e49318c74f33d520d31ae0c4"}, - {file = "pyarrow-19.0.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a21d39fbdb948857f67eacb5bbaaf36802de044ec36fbef7a1c8f0dd3a4ab2"}, - {file = "pyarrow-19.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:99bc1bec6d234359743b01e70d4310d0ab240c3d6b0da7e2a93663b0158616f6"}, - {file = "pyarrow-19.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1b93ef2c93e77c442c979b0d596af45e4665d8b96da598db145b0fec014b9136"}, - {file = "pyarrow-19.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:d9d46e06846a41ba906ab25302cf0fd522f81aa2a85a71021826f34639ad31ef"}, - {file = "pyarrow-19.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:c0fe3dbbf054a00d1f162fda94ce236a899ca01123a798c561ba307ca38af5f0"}, - {file = "pyarrow-19.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:96606c3ba57944d128e8a8399da4812f56c7f61de8c647e3470b417f795d0ef9"}, - {file = "pyarrow-19.0.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f04d49a6b64cf24719c080b3c2029a3a5b16417fd5fd7c4041f94233af732f3"}, - {file = "pyarrow-19.0.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a9137cf7e1640dce4c190551ee69d478f7121b5c6f323553b319cac936395f6"}, - {file = "pyarrow-19.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7c1bca1897c28013db5e4c83944a2ab53231f541b9e0c3f4791206d0c0de389a"}, - {file = "pyarrow-19.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:58d9397b2e273ef76264b45531e9d552d8ec8a6688b7390b5be44c02a37aade8"}, - {file = "pyarrow-19.0.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:b9766a47a9cb56fefe95cb27f535038b5a195707a08bf61b180e642324963b46"}, - {file = "pyarrow-19.0.1-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:6c5941c1aac89a6c2f2b16cd64fe76bcdb94b2b1e99ca6459de4e6f07638d755"}, - {file = "pyarrow-19.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8"}, - {file = "pyarrow-19.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:335d170e050bcc7da867a1ed8ffb8b44c57aaa6e0843b156a501298657b1e972"}, - {file = "pyarrow-19.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:1c7556165bd38cf0cd992df2636f8bcdd2d4b26916c6b7e646101aff3c16f76f"}, - {file = "pyarrow-19.0.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:699799f9c80bebcf1da0983ba86d7f289c5a2a5c04b945e2f2bcf7e874a91911"}, - {file = "pyarrow-19.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8464c9fbe6d94a7fe1599e7e8965f350fd233532868232ab2596a71586c5a429"}, - {file = "pyarrow-19.0.1.tar.gz", hash = "sha256:3bf266b485df66a400f282ac0b6d1b500b9d2ae73314a153dbe97d6d5cc8a99e"}, + {file = "pyarrow-20.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:c7dd06fd7d7b410ca5dc839cc9d485d2bc4ae5240851bcd45d85105cc90a47d7"}, + {file = "pyarrow-20.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:d5382de8dc34c943249b01c19110783d0d64b207167c728461add1ecc2db88e4"}, + {file = "pyarrow-20.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6415a0d0174487456ddc9beaead703d0ded5966129fa4fd3114d76b5d1c5ceae"}, + {file = "pyarrow-20.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15aa1b3b2587e74328a730457068dc6c89e6dcbf438d4369f572af9d320a25ee"}, + {file = "pyarrow-20.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:5605919fbe67a7948c1f03b9f3727d82846c053cd2ce9303ace791855923fd20"}, + {file = "pyarrow-20.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a5704f29a74b81673d266e5ec1fe376f060627c2e42c5c7651288ed4b0db29e9"}, + {file = "pyarrow-20.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:00138f79ee1b5aca81e2bdedb91e3739b987245e11fa3c826f9e57c5d102fb75"}, + {file = "pyarrow-20.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f2d67ac28f57a362f1a2c1e6fa98bfe2f03230f7e15927aecd067433b1e70ce8"}, + {file = "pyarrow-20.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:4a8b029a07956b8d7bd742ffca25374dd3f634b35e46cc7a7c3fa4c75b297191"}, + {file = "pyarrow-20.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:24ca380585444cb2a31324c546a9a56abbe87e26069189e14bdba19c86c049f0"}, + {file = "pyarrow-20.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:95b330059ddfdc591a3225f2d272123be26c8fa76e8c9ee1a77aad507361cfdb"}, + {file = "pyarrow-20.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f0fb1041267e9968c6d0d2ce3ff92e3928b243e2b6d11eeb84d9ac547308232"}, + {file = "pyarrow-20.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8ff87cc837601532cc8242d2f7e09b4e02404de1b797aee747dd4ba4bd6313f"}, + {file = "pyarrow-20.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:7a3a5dcf54286e6141d5114522cf31dd67a9e7c9133d150799f30ee302a7a1ab"}, + {file = "pyarrow-20.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a6ad3e7758ecf559900261a4df985662df54fb7fdb55e8e3b3aa99b23d526b62"}, + {file = "pyarrow-20.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6bb830757103a6cb300a04610e08d9636f0cd223d32f388418ea893a3e655f1c"}, + {file = "pyarrow-20.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:96e37f0766ecb4514a899d9a3554fadda770fb57ddf42b63d80f14bc20aa7db3"}, + {file = "pyarrow-20.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:3346babb516f4b6fd790da99b98bed9708e3f02e734c84971faccb20736848dc"}, + {file = "pyarrow-20.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:75a51a5b0eef32727a247707d4755322cb970be7e935172b6a3a9f9ae98404ba"}, + {file = "pyarrow-20.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:211d5e84cecc640c7a3ab900f930aaff5cd2702177e0d562d426fb7c4f737781"}, + {file = "pyarrow-20.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ba3cf4182828be7a896cbd232aa8dd6a31bd1f9e32776cc3796c012855e1199"}, + {file = "pyarrow-20.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c3a01f313ffe27ac4126f4c2e5ea0f36a5fc6ab51f8726cf41fee4b256680bd"}, + {file = "pyarrow-20.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:a2791f69ad72addd33510fec7bb14ee06c2a448e06b649e264c094c5b5f7ce28"}, + {file = "pyarrow-20.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4250e28a22302ce8692d3a0e8ec9d9dde54ec00d237cff4dfa9c1fbf79e472a8"}, + {file = "pyarrow-20.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:89e030dc58fc760e4010148e6ff164d2f44441490280ef1e97a542375e41058e"}, + {file = "pyarrow-20.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6102b4864d77102dbbb72965618e204e550135a940c2534711d5ffa787df2a5a"}, + {file = "pyarrow-20.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:96d6a0a37d9c98be08f5ed6a10831d88d52cac7b13f5287f1e0f625a0de8062b"}, + {file = "pyarrow-20.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:a15532e77b94c61efadde86d10957950392999503b3616b2ffcef7621a002893"}, + {file = "pyarrow-20.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:dd43f58037443af715f34f1322c782ec463a3c8a94a85fdb2d987ceb5658e061"}, + {file = "pyarrow-20.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa0d288143a8585806e3cc7c39566407aab646fb9ece164609dac1cfff45f6ae"}, + {file = "pyarrow-20.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6953f0114f8d6f3d905d98e987d0924dabce59c3cda380bdfaa25a6201563b4"}, + {file = "pyarrow-20.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:991f85b48a8a5e839b2128590ce07611fae48a904cae6cab1f089c5955b57eb5"}, + {file = "pyarrow-20.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:97c8dc984ed09cb07d618d57d8d4b67a5100a30c3818c2fb0b04599f0da2de7b"}, + {file = "pyarrow-20.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9b71daf534f4745818f96c214dbc1e6124d7daf059167330b610fc69b6f3d3e3"}, + {file = "pyarrow-20.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8b88758f9303fa5a83d6c90e176714b2fd3852e776fc2d7e42a22dd6c2fb368"}, + {file = "pyarrow-20.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:30b3051b7975801c1e1d387e17c588d8ab05ced9b1e14eec57915f79869b5031"}, + {file = "pyarrow-20.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:ca151afa4f9b7bc45bcc791eb9a89e90a9eb2772767d0b1e5389609c7d03db63"}, + {file = "pyarrow-20.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:4680f01ecd86e0dd63e39eb5cd59ef9ff24a9d166db328679e36c108dc993d4c"}, + {file = "pyarrow-20.0.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f4c8534e2ff059765647aa69b75d6543f9fef59e2cd4c6d18015192565d2b70"}, + {file = "pyarrow-20.0.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e1f8a47f4b4ae4c69c4d702cfbdfe4d41e18e5c7ef6f1bb1c50918c1e81c57b"}, + {file = "pyarrow-20.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:a1f60dc14658efaa927f8214734f6a01a806d7690be4b3232ba526836d216122"}, + {file = "pyarrow-20.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:204a846dca751428991346976b914d6d2a82ae5b8316a6ed99789ebf976551e6"}, + {file = "pyarrow-20.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f3b117b922af5e4c6b9a9115825726cac7d8b1421c37c2b5e24fbacc8930612c"}, + {file = "pyarrow-20.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e724a3fd23ae5b9c010e7be857f4405ed5e679db5c93e66204db1a69f733936a"}, + {file = "pyarrow-20.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:82f1ee5133bd8f49d31be1299dc07f585136679666b502540db854968576faf9"}, + {file = "pyarrow-20.0.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:1bcbe471ef3349be7714261dea28fe280db574f9d0f77eeccc195a2d161fd861"}, + {file = "pyarrow-20.0.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:a18a14baef7d7ae49247e75641fd8bcbb39f44ed49a9fc4ec2f65d5031aa3b96"}, + {file = "pyarrow-20.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb497649e505dc36542d0e68eca1a3c94ecbe9799cb67b578b55f2441a247fbc"}, + {file = "pyarrow-20.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11529a2283cb1f6271d7c23e4a8f9f8b7fd173f7360776b668e509d712a02eec"}, + {file = "pyarrow-20.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:6fc1499ed3b4b57ee4e090e1cea6eb3584793fe3d1b4297bbf53f09b434991a5"}, + {file = "pyarrow-20.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:db53390eaf8a4dab4dbd6d93c85c5cf002db24902dbff0ca7d988beb5c9dd15b"}, + {file = "pyarrow-20.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:851c6a8260ad387caf82d2bbf54759130534723e37083111d4ed481cb253cc0d"}, + {file = "pyarrow-20.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:e22f80b97a271f0a7d9cd07394a7d348f80d3ac63ed7cc38b6d1b696ab3b2619"}, + {file = "pyarrow-20.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:9965a050048ab02409fb7cbbefeedba04d3d67f2cc899eff505cc084345959ca"}, + {file = "pyarrow-20.0.0.tar.gz", hash = "sha256:febc4a913592573c8d5805091a6c2b5064c8bd6e002131f01061797d91c783c1"}, ] [package.extras] @@ -8336,4 +8310,4 @@ modelbench-private = ["modelbench-private"] [metadata] lock-version = "2.0" python-versions = ">=3.10,!=3.12.5,<3.13" -content-hash = "e7dd29826732acdbba3c59daba48f8d981ebaf1552d2b1a86c78ba631dbbbd12" +content-hash = "f28ad6ef1be39cdee175d7f278f6f5a09cfeb6fa2a76a932e1461e4255e8554b" diff --git a/pyproject.toml b/pyproject.toml index cf2f3d5..e11da31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ packages = [ python = ">=3.10,!=3.12.5,<3.13" click = "^8" dvc = {extras = ["gs"], version = "^3.60"} -modelbench = {git = "https://github.com/mlcommons/modelbench.git", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } +modelbench = {git = "https://github.com/mlcommons/modelbench.git", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" } mlflow = "^3.1.1" python-dotenv = "^1" requests = "^2" @@ -25,19 +25,17 @@ jupyter = "^1" scikit-learn = "^1.5.0" pandas = "^2.2.2" # plugins (would like to figure out a better way to manage these) -modelgauge_anthropic = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/anthropic", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } -modelgauge-azure = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/azure", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } -modelgauge_baseten = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/baseten", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } -modelgauge_demo_plugin = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "demo_plugin", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } -modelgauge_nvidia = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/nvidia", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } -modelgauge_openai = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/openai", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } -modelgauge_huggingface = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/huggingface", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } -modelgauge_perspective_api = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/perspective_api", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } -modelgauge_google = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/google", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } -modelgauge_vertexai = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/vertexai", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } -modelgauge_mistral = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/mistral", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } -modelgauge_amazon = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/amazon", rev = "afe1cbfa42eebe3363440d1ab5841c007384e244" } -modelbench-private = { git = "git@github.com:mlcommons/modelbench-private.git", rev = "982973bd51e6c7ebb1bebaaa1eeb6a297d871bb3", optional = true } +modelgauge_anthropic = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/anthropic", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" } +modelgauge-azure = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/azure", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" } +modelgauge_baseten = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/baseten", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" } +modelgauge_demo_plugin = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "demo_plugin", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" } +modelgauge_nvidia = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/nvidia", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" } +modelgauge_perspective_api = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/perspective_api", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" } +modelgauge_google = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/google", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" } +modelgauge_vertexai = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/vertexai", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" } +modelgauge_mistral = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/mistral", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" } +modelgauge_amazon = { git = "https://github.com/mlcommons/modelbench.git", subdirectory = "plugins/amazon", rev = "ba582ed35a58cf6219786624c83e6915a9bcf98a" } +modelbench-private = { git = "git@github.com:mlcommons/modelbench-private.git", rev = "ecac4cfd343411c8011c0bba875d638fc22e9478", optional = true } [tool.poetry.extras] modelbench-private = ["modelbench-private"] @@ -50,7 +48,7 @@ pytest = "^8" pytest-cov = "^6" [tool.poetry.scripts] -modelplane = "modelplane.runways.run:cli" +modelplane = "modelplane.cli:cli" [build-system] requires = ["poetry-core"] diff --git a/src/modelplane/__init__.py b/src/modelplane/__init__.py new file mode 100644 index 0000000..622e444 --- /dev/null +++ b/src/modelplane/__init__.py @@ -0,0 +1,3 @@ +from modelgauge.config import write_default_config + +write_default_config() diff --git a/src/modelplane/runways/run.py b/src/modelplane/cli.py similarity index 100% rename from src/modelplane/runways/run.py rename to src/modelplane/cli.py diff --git a/src/modelplane/runways/lister.py b/src/modelplane/runways/lister.py index 617a693..9fab845 100644 --- a/src/modelplane/runways/lister.py +++ b/src/modelplane/runways/lister.py @@ -1,16 +1,15 @@ from modelgauge.annotator_registry import ANNOTATORS -from modelgauge.command_line import compact_sut_list from modelgauge.ensemble_annotator_set import ENSEMBLE_STRATEGIES from modelgauge.sut_registry import SUTS def list_annotators(): - print(compact_sut_list(ANNOTATORS)) + print(ANNOTATORS.compact_uid_list()) def list_suts(): - print(compact_sut_list(SUTS)) + print(SUTS.compact_uid_list()) def list_ensemble_strategies(): - print(compact_sut_list(ENSEMBLE_STRATEGIES)) + print(sorted(ENSEMBLE_STRATEGIES)) diff --git a/tests/it/test_cli.py b/tests/it/test_cli.py index 2813843..55f9205 100644 --- a/tests/it/test_cli.py +++ b/tests/it/test_cli.py @@ -1,7 +1,7 @@ import pytest from click.testing import CliRunner -from modelplane.runways.run import cli +from modelplane.cli import cli def test_main_help():