pyproject.toml

[build-system]
requires = ["setuptools"]
build-backend = "setuptools.build_meta"

[project]
name = "crfm-helm"
version = "0.5.13"
authors = [
    { name = "Stanford CRFM", email = "contact-crfm@stanford.edu" }
]
description = "Framework for evaluation of foundation models"
readme = { file = "README.md", content-type = "text/markdown" }
keywords = ["language", "models", "benchmarking"]
license = { text = "Apache License 2.0" }
classifiers = [
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3 :: Only",
    "License :: OSI Approved :: Apache Software License",
]
urls = { "Homepage" = "https://github.com/stanford-crfm/helm" }

requires-python = ">=3.10"

dependencies = [
    # Common
    "cattrs~=22.2",
    "colorlog~=6.9",
    "dacite~=1.6",
    "Mako~=1.2",
    "numpy>=1.26,<3",
    "pandas~=2.0",
    "pyhocon~=0.3.59",
    "ubelt~=1.3",
    "retrying~=1.3",
    "spacy~=3.5",
    "tqdm~=4.64",
    "zstandard~=0.18.0",
    # sqlitedict==2.0.0 is slow! https://github.com/RaRe-Technologies/sqlitedict/issues/152
    # Keep sqlitedict version at 1.7.0.
    "sqlitedict>=2.1.0,<3.0",
    "bottle~=0.12.23",
    # Basic Scenarios
    "datasets~=3.1",
    "pyarrow>=11.0.0", # Pinned transitive dependency for datasets; workaround for #1026
    "pyarrow-hotfix~=0.6", # Hotfix for CVE-2023-47248
    # Basic metrics
    "nltk~=3.7,!=3.9.0", # Cannot use 3.9.0 due to https://github.com/nltk/nltk/issues/3308
    "rouge-score~=0.1.2",
    "scipy>=1.10",
    "uncertainty-calibration~=0.1.4",
    "scikit-learn>=1.1",
    # Models and Metrics Extras
    # TODO(#3731): Remove the bounds on transformers version after #3731 is resolved.
    "transformers~=4.53", # For anthropic_client, vision_language.huggingface_vlm_client, huggingface_client, huggingface_tokenizer, test_openai_token_cost_estimator, model_summac (via summarization_metrics)
    # TODO: Upgrade torch - we need > 2.0.0 for newer versions of transformers
    "torch>=1.13.1,<3.0.0", # For huggingface_client, yalm_tokenizer, model_summac (via summarization_metrics)
    "torchvision>=0.14.1,<3.0.0", # For huggingface_client, yalm_tokenizer, model_summac (via summarization_metrics)
]

[project.optional-dependencies]
proxy-server = [
    "gunicorn>=20.1",
]

human-evaluation = [
    "scaleapi~=2.13",
    "surge-api~=1.1",
]

dspy = [
    "dspy~=3.0",
]

scenarios = [
    "gdown~=5.1",  # For disinformation_scenario, med_mcqa_scenario, med_qa_scenario: used by ensure_file_downloaded()
    "xlrd~=2.0",  # For ice_scenario: used by pandas.read_excel()
]

metrics = [
    "google-api-python-client~=2.64",  # For perspective_api_client via toxicity_metrics
    "numba~=0.56",  # For copyright_metrics
    "sacrebleu~=2.2",  # For disinformation_metrics, machine_translation_metrics
    "langdetect~=1.0",  # For ifeval_metrics
    "immutabledict~=4.2",  # For ifeval_metrics
    "gradio_client~=1.3",  # For bigcodebench_metrics
]

ranking = [
    "pytrec_eval==0.5",  # For ranking_metrics
]

summarization = [
    # For summarization_metrics
    "summ-eval~=0.892",
    "bert-score~=0.3",
    "sentencepiece~=0.2.0",
    "protobuf",
]

plots = [
    "colorcet~=3.0",
    "matplotlib>=3.6.0",
    "seaborn>=0.11.0",
]

decodingtrust = [
    "fairlearn~=0.9.0",
]

slurm = [
    "simple-slurm~=0.2.6",
]

cleva = [
    "unidecode~=1.3",
    "pypinyin~=0.49.0",
    "jieba~=0.42.1",
    "opencc~=1.1",
    "langdetect~=1.0",
]

images = [
    "crfm-helm[accelerate]",
    "pillow>=10.2",
]

mongo = [
    "pymongo~=4.2",
]

unitxt = [
    "evaluate~=0.4.1",
]

seahelm = [
    "pythainlp==5.0.0",
    "pyonmttok==1.37.0",
    "sacrebleu~=2.2",
    "python-crfsuite~=0.9.11",
]

# Model extras
accelerate = [
    "accelerate~=0.25",
]

aleph-alpha = [
    "aleph-alpha-client~=2.14",
    "tokenizers>=0.13.3",
]

allenai = [
    "ai2-olmo~=0.2",
]

amazon = [
    "boto3~=1.34",
    "awscli~=1.33",
    "botocore~=1.34",
]

anthropic = [
    "anthropic~=0.41",
    "websocket-client~=1.3",  # For legacy stanford-online-all-v4-s3
]

cohere = [
    "cohere~=5.3",
]

writer = [
    "writerai~=4.0",
]

mistral = [
    "mistralai~=1.1",
]

openai = [
    "openai~=2.8",
    "tiktoken~=0.7",
    "pydantic~=2.0", # For model_dump(mode="json") - openai only requires pydantic>=1.9.0
]

google = [
    "google-cloud-aiplatform~=1.48",
    "google-genai~=1.48",
]

together = [
    "together~=1.1",
]

yandex = [
    "sentencepiece~=0.2.0",
]

models = [
    "crfm-helm[ai21]",
    "crfm-helm[accelerate]",
    "crfm-helm[aleph-alpha]",
    "crfm-helm[allenai]",
    "crfm-helm[amazon]",
    "crfm-helm[anthropic]",
    "crfm-helm[cohere]",
    "crfm-helm[google]",
    "crfm-helm[mistral]",
    "crfm-helm[openai]",
    "crfm-helm[reka]",
    "crfm-helm[together]",
    "crfm-helm[yandex]",
    "crfm-helm[writer]",
]

reka = [
    "reka-api~=2.0",
]

vlm = [
    "crfm-helm[openai]",

    # For OpenFlamingo
    "einops~=0.7.0",
    "einops-exts~=0.0.4",
    "open-clip-torch~=2.24",

    # For IDEFICS
    "torch~=2.1",

    # For Qwen: https://github.com/QwenLM/Qwen-VL/blob/master/requirements.txt
    "transformers_stream_generator~=0.0.4",
    "scipy~=1.10",
    "torchvision>=0.14.1,<3.0.0",

    # For Reka AI
    "crfm-helm[reka]",

    # VLM scenarios
    "crfm-helm[images]",
    "crfm-helm[image2struct]",

    # For metrics
    "pycocoevalcap~=1.2",

    # For Qwen2
    "qwen-vl-utils~=0.0.8",
]

ibm-enterprise-scenarios = [
    "openpyxl~=3.1",
]

ibm = [
    # incompatability issues with pandas library !
    "ibm-watsonx-ai~=1.2",
]

image2struct = [
    "crfm-helm[images]",

    # Latex
    # You will need to install LaTeX separately.
    # You can run `sudo apt-get install texlive-full` on Ubuntu.
    "latex~=0.7.0",
    "pdf2image~=1.16",

    # Webpage
    # You will need install Jekyll separately.
    "selenium~=4.17",
    "html2text~=2024.2.26",

    # Metrics
    "opencv-python-headless>=4.7.0.68,<=4.11.0.86",
    "lpips~=0.1.4",
    "imagehash~=4.3"  # for caching
]

heim = [
    # HEIM scenarios
    "gdown~=5.1",

    # HEIM models
    "diffusers~=0.34.0",
    "icetk~=0.0.4",
    "jax~=0.6.2; python_version >= '3.10'",
    "jax~=0.4.30; python_version < '3.10'",
    "jaxlib~=0.6.2; python_version >= '3.10'",
    "jaxlib~=0.4.30; python_version < '3.10'",
    "crfm-helm[openai]",

    # For model, kakaobrain/mindall-e
    "einops~=0.7.0",
    "omegaconf~=2.3",
    "pytorch-lightning~=2.0",

    # For model, craiyon/dalle-mini and craiyon/dalle-mega
    "flax~=0.10.7; python_version >= '3.10'",
    "flax~=0.8.5; python_version < '3.10'",
    "ftfy~=6.1",
    "Unidecode~=1.3",
    "wandb~=0.16",

    # HEIM perturbations
    "google-cloud-translate~=3.11",

    # HEIM metrics
    "autokeras~=1.0",
    "clip-anytorch~=2.5",
    "google-cloud-storage~=2.9",
    "lpips~=0.1.4",
    "multilingual-clip~=1.0",
    "NudeNet~=2.0",
    "numpy>=1.26",
    "opencv-python>=4.7.0.68,<4.8.2.0; python_version >= '3.10'",
    "opencv-python-headless>=4.7.0.68,<=4.11.0.86; python_version < '3.10'",
    "pytorch-fid~=0.3.0",
    "tensorflow~=2.11",
    "timm~=0.6.12",
    "torch-fidelity~=0.3.0",
    "torchmetrics~=0.11.1",

    # Transitive dependency of NudeNet
    # This needs to be a version that provides wheels for all Python versions
    # supported by crfm-helm i.e. 3.10, 3.11, 3.12
    "scikit-image>=0.22,==0.*",

    # Shared image dependencies
    "crfm-helm[images]",
]

medhelm = [
    "accelerate~=0.25",
    "crfm-helm[openai]",
    "crfm-helm[yandex]",
    "crfm-helm[scenarios]",

    #MedHELM scenarios
    "bert_score~=0.3.13",
    "lxml~=5.3",
    "openpyxl~=3.1",
    "python-docx~=1.1",
]

audiolm = [
    "crfm-helm[openai]",
    "crfm-helm[google]",

    # For clipping and converting audio
    "pydub~=0.25.1",

    # For extracting audio from videos
    "ffmpeg-python~=0.2.0",

    # For HuggingFace audio datasets
    "soundfile~=0.12",
    "librosa~=0.10",
    "einops~=0.7.0",

    # For LLaMA-Omni
    "openai-whisper==20240930",

    # For Qwen2.5-Omni
    "transformers_stream_generator~=0.0.4",
    "av~=14.3",
    "scipy~=1.10",
    "torchvision>=0.14.1,<3.0.0",
    "flash-attn~=2.7",

    # For metrics
    "pycocoevalcap~=1.2",
    "jiwer~=3.0",
    "rapidfuzz~=3.10",
    "jieba~=0.42.1",
]

codeinsights = [
    "clang~=20.1",
    "Levenshtein~=0.27",
]

lmkt = [
    "sentence_transformers~=4.1",
]

all = [
    "crfm-helm[proxy-server]",
    "crfm-helm[scenarios]",
    "crfm-helm[metrics]",
    "crfm-helm[plots]",
    "crfm-helm[decodingtrust]",
    "crfm-helm[slurm]",
    "crfm-helm[cleva]",
    "crfm-helm[images]",
    "crfm-helm[models]",
    "crfm-helm[mongo]",
    "crfm-helm[heim]",
    "crfm-helm[vlm]",
    "crfm-helm[codeinsights]",
    "crfm-helm[lmkt]",
    # crfm-helm[human-evaluation] is excluded because it conflicts with crfm-helm[ibm]
    # crfm-helm[audiolm] is excluded because it requires transformers~=4.48
    # crfm-helm[seahelm] is excluded because pyonmttok does not support Python 3.12
    # crfm-helm[dev] is excluded because end-users don't need it.
    # crfm-helm[summarization] is excluded because it requires torch<2.0 and numpy<2.0
    # crfm-helm[ranking] is excluded because pytrec_eval causes build errors (see #3470)
]

ci = [
  "crfm-helm[metrics]",
  "crfm-helm[openai]",
  "crfm-helm[plots]",
  "crfm-helm[together]",
  "crfm-helm[yandex]",
  "crfm-helm[cohere]",
  "crfm-helm[proxy-server]",
]
litellm = [
    "litellm>=1.80.0",
]

[dependency-groups]

dev = [
    "pytest~=7.2.0",
    "xdoctest~=1.2.0",
    "pre-commit~=2.20.0",
    # Errors produced by type checkers and linters are very version-specific
    # so they are pinned to an exact version.
    "black==24.3.0",
    "mypy==1.16.0",
    "flake8==7.3.0",
    "en_core_web_sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl",
]

types = [
    "types-cachetools",
    "types-pyOpenSSL",
    "types-PyYAML",
    "types-requests",
    "types-simplejson",
    "types-tabulate",
]

[project.scripts]
helm-run = "helm.benchmark.run:main"
helm-summarize = "helm.benchmark.presentation.summarize:main"
helm-server = "helm.benchmark.server:main"
helm-create-plots = "helm.benchmark.presentation.create_plots:main"
crfm-proxy-server = "helm.proxy.server:main"
crfm-proxy-cli = "helm.proxy.cli:main"

[tool.uv]
default-groups = ["dev", "types"]

override-dependencies = [
    # override torch dependency for summ-eval in "summarization" extra:
    # - summ-eval depends on blanc 
    # - blanc depends on torch<2.0
    "torch~=2.1",
    # override torch dependency for summ-eval in "summarization" extra:
    # - summ-eval depends on blanc 
    # - blanc depends on numpy<2.0
    "numpy>=1.26,<3",
]

conflicts = [
    [
      { extra = "ibm" },
      { extra = "human-evaluation" },
    ],
]

[tool.setuptools]
package-dir = { "" = "src" }
zip-safe = false
include-package-data = true

[tool.setuptools.packages.find]
where = ["src"]
exclude = ["tests*"]

# Note: flake8 does not support pyproject.toml
# the flake8 configuration is in .flake8
# it may be a good idea to migrate to ruff

# Settings for Mypy: static type checker for Python 3
[tool.mypy]
ignore_missing_imports = true
check_untyped_defs = true
# TODO: Remove disable_error_code
disable_error_code = "annotation-unchecked"
# TODO: Change disallow_untyped_defs to True
disallow_untyped_defs = false
exclude = "dalle_mini|mindalle|open_flamingo|llama_omni|qwen_omni"

[tool.pytest.ini_options]
# By default:
# - we don't test models because doing so will
#   make real requests and spend real money
# - we don't test scenarios because these will
#   download files, which is slow, consumes disk
#   space, and increases the chance of spurious
#   test failures due to failed downloads.
#
# For more documentation on pytest markers, see:
# - https://docs.pytest.org/en/latest/how-to/mark.html#mark
# - https://docs.pytest.org/en/latest/example/markers.html#mark-examples
addopts = """
-m 'not models and not scenarios' 
--xdoctest-modules 
--xdoctest-style=google
"""

markers = [
    # Marker for model tests that make real model requests
    "models",
    # Marker for scenario tests that download files
    "scenarios",
]

# Settings for Black: The Uncompromising Code Formatter
[tool.black]
line-length = 120