tokenizers/pyproject.toml at main · skrueger/tokenizers · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
[build-system]
requires = [
  "cmake", # For building binary targets in the wheel.
  "pip>=23",  # For building the pip package.
  "setuptools>=63",  # For building the pip package contents.
  "wheel",  # For building the pip package archive.
  "pytest",  # For running tests.
  "pybind11",  # For building the pybind11 C++ extension.
]
build-backend = "setuptools.build_meta"

[project]
name = "pytorch_tokenizers"
dynamic = [
  # setup.py will set the version.
  'version',
  'readme',
]
description = "A package with common tokenizers in Python and C++"
authors = [
  {name="PyTorch Team", email="packages@pytorch.org"},
]
license = {file = "LICENSE"}
keywords = ["pytorch", "machine learning", "llm"]
# PyPI package information.
classifiers = [
    # How mature is this project? Common values are
    #   3 - Alpha
    #   4 - Beta
    #   5 - Production/Stable
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "Intended Audience :: Education",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: BSD License",
    "Topic :: Scientific/Engineering",
    "Topic :: Scientific/Engineering :: Mathematics",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Software Development",
    "Topic :: Software Development :: Libraries",
    "Topic :: Software Development :: Libraries :: Python Modules",
    "Programming Language :: C++",
    "Programming Language :: Python :: 3",
    # Update this as we support more versions of python.
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
]

# Python dependencies required for use.
requires-python = ">=3.10"
dependencies=[
  "tiktoken",
  "tokenizers",
  "sentencepiece",
]

[project.urls]
# The keys are arbitrary but will be visible on PyPI.
Homepage = "https://pytorch.org/executorch/"
Repository = "https://github.com/pytorch/executorch"
Issues = "https://github.com/pytorch/executorch/issues"
Changelog = "https://github.com/pytorch/executorch/releases"


[tool.setuptools.exclude-package-data]
"*" = ["*.pyc"]

[tool.pytest.ini_options]
testpaths = ["test"]
python_files = ["test_*.py", "*_test.py"]
python_classes = ["Test*"]
python_functions = ["test_*"]

[tool.black]
target-version = ['py38', 'py39', 'py310', 'py311', 'py312']
include = '\.pyi?$'
extend-exclude = '''
/(
  # directories
  \.eggs
  | \.git
  | build
  | dist
  | third-party
)/
'''