Skip to content

Commit a87d94b

Browse files
seemethereseemethere
authored
s3_management: Switch PACKAGES_PER_PROJECT to dict (#6908)
Switches PACKAGES_PER_PROJECT to a dict, I will be using this in a later PR to enable our ability to mirror these packages directly from PyPI. Validated locally using: ``` python3 s3_management/update_dependencies.py --dry-run ``` Part of #6907 --------- Signed-off-by: Eli Uriegas <[email protected]>
1 parent 045b7fa commit a87d94b

File tree

1 file changed

+108
-115
lines changed

1 file changed

+108
-115
lines changed

‎s3_management/update_dependencies.py

Lines changed: 108 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -8,158 +8,147 @@
88
CLIENT = boto3.client("s3")
99
BUCKET = S3.Bucket("pytorch")
1010

11-
PACKAGES_PER_PROJECT = [
12-
{"package": "sympy", "version": "latest", "project": "torch"},
13-
{"package": "mpmath", "version": "latest", "project": "torch"},
14-
{"package": "pillow", "version": "latest", "project": "torch"},
15-
{"package": "networkx", "version": "latest", "project": "torch"},
16-
{"package": "numpy", "version": "latest", "project": "torch"},
17-
{"package": "jinja2", "version": "latest", "project": "torch"},
18-
{"package": "filelock", "version": "latest", "project": "torch"},
19-
{"package": "fsspec", "version": "latest", "project": "torch"},
20-
{"package": "nvidia-cudnn-cu11", "version": "latest", "project": "torch"},
21-
{"package": "nvidia-cudnn-cu12", "version": "latest", "project": "torch"},
22-
{"package": "typing-extensions", "version": "latest", "project": "torch"},
23-
{
24-
"package": "nvidia-cuda-nvrtc-cu12",
11+
PACKAGES_PER_PROJECT = {
12+
"sympy": {"version": "latest", "project": "torch"},
13+
"mpmath": {"version": "latest", "project": "torch"},
14+
"pillow": {"version": "latest", "project": "torch"},
15+
"networkx": {"version": "latest", "project": "torch"},
16+
"numpy": {"version": "latest", "project": "torch"},
17+
"jinja2": {"version": "latest", "project": "torch"},
18+
"filelock": {"version": "latest", "project": "torch"},
19+
"fsspec": {"version": "latest", "project": "torch"},
20+
"nvidia-cudnn-cu11": {"version": "latest", "project": "torch"},
21+
"nvidia-cudnn-cu12": {"version": "latest", "project": "torch"},
22+
"typing-extensions": {"version": "latest", "project": "torch"},
23+
"nvidia-cuda-nvrtc-cu12": {
2524
"version": "12.9.86",
2625
"project": "torch",
2726
"target": "cu129",
2827
},
29-
{
30-
"package": "nvidia-cuda-runtime-cu12",
28+
"nvidia-cuda-runtime-cu12": {
3129
"version": "12.9.79",
3230
"project": "torch",
3331
"target": "cu129",
3432
},
35-
{
36-
"package": "nvidia-cuda-cupti-cu12",
33+
"nvidia-cuda-cupti-cu12": {
3734
"version": "12.9.79",
3835
"project": "torch",
3936
"target": "cu129",
4037
},
41-
{
42-
"package": "nvidia-cublas-cu12",
38+
"nvidia-cublas-cu12": {
4339
"version": "12.9.1.4",
4440
"project": "torch",
4541
"target": "cu129",
4642
},
47-
{
48-
"package": "nvidia-cufft-cu12",
43+
"nvidia-cufft-cu12": {
4944
"version": "11.4.1.4",
5045
"project": "torch",
5146
"target": "cu129",
5247
},
53-
{
54-
"package": "nvidia-curand-cu12",
48+
"nvidia-curand-cu12": {
5549
"version": "10.3.10.19",
5650
"project": "torch",
5751
"target": "cu129",
5852
},
59-
{
60-
"package": "nvidia-cusolver-cu12",
53+
"nvidia-cusolver-cu12": {
6154
"version": "11.7.5.82",
6255
"project": "torch",
6356
"target": "cu129",
6457
},
65-
{
66-
"package": "nvidia-cusparse-cu12",
58+
"nvidia-cusparse-cu12": {
6759
"version": "12.5.10.65",
6860
"project": "torch",
6961
"target": "cu129",
7062
},
71-
{
72-
"package": "nvidia-nvtx-cu12",
63+
"nvidia-nvtx-cu12": {
7364
"version": "12.9.79",
7465
"project": "torch",
7566
"target": "cu129",
7667
},
77-
{
78-
"package": "nvidia-nvjitlink-cu12",
68+
"nvidia-nvjitlink-cu12": {
7969
"version": "12.9.86",
8070
"project": "torch",
8171
"target": "cu129",
8272
},
83-
{
84-
"package": "nvidia-cufile-cu12",
73+
"nvidia-cufile-cu12": {
8574
"version": "1.14.1.1",
8675
"project": "torch",
8776
"target": "cu129",
8877
},
89-
{"package": "arpeggio", "version": "latest", "project": "triton"},
90-
{"package": "caliper-reader", "version": "latest", "project": "triton"},
91-
{"package": "contourpy", "version": "latest", "project": "triton"},
92-
{"package": "cycler", "version": "latest", "project": "triton"},
93-
{"package": "dill", "version": "latest", "project": "triton"},
94-
{"package": "fonttools", "version": "latest", "project": "triton"},
95-
{"package": "kiwisolver", "version": "latest", "project": "triton"},
96-
{"package": "llnl-hatchet", "version": "latest", "project": "triton"},
97-
{"package": "matplotlib", "version": "latest", "project": "triton"},
98-
{"package": "pandas", "version": "latest", "project": "triton"},
99-
{"package": "pydot", "version": "latest", "project": "triton"},
100-
{"package": "pyparsing", "version": "latest", "project": "triton"},
101-
{"package": "pytz", "version": "latest", "project": "triton"},
102-
{"package": "textX", "version": "latest", "project": "triton"},
103-
{"package": "tzdata", "version": "latest", "project": "triton"},
104-
{"package": "importlib-metadata", "version": "latest", "project": "triton"},
105-
{"package": "importlib-resources", "version": "latest", "project": "triton"},
106-
{"package": "zipp", "version": "latest", "project": "triton"},
107-
{"package": "aiohttp", "version": "latest", "project": "torchtune"},
108-
{"package": "aiosignal", "version": "latest", "project": "torchtune"},
109-
{"package": "antlr4-python3-runtime", "version": "latest", "project": "torchtune"},
110-
{"package": "attrs", "version": "latest", "project": "torchtune"},
111-
{"package": "blobfile", "version": "latest", "project": "torchtune"},
112-
{"package": "certifi", "version": "latest", "project": "torchtune"},
113-
{"package": "charset-normalizer", "version": "latest", "project": "torchtune"},
114-
{"package": "datasets", "version": "latest", "project": "torchtune"},
115-
{"package": "dill", "version": "latest", "project": "torchtune"},
116-
{"package": "frozenlist", "version": "latest", "project": "torchtune"},
117-
{"package": "huggingface-hub", "version": "latest", "project": "torchtune"},
118-
{"package": "idna", "version": "latest", "project": "torchtune"},
119-
{"package": "lxml", "version": "latest", "project": "torchtune"},
120-
{"package": "markupsafe", "version": "latest", "project": "torchtune"},
121-
{"package": "multidict", "version": "latest", "project": "torchtune"},
122-
{"package": "multiprocess", "version": "latest", "project": "torchtune"},
123-
{"package": "omegaconf", "version": "latest", "project": "torchtune"},
124-
{"package": "pandas", "version": "latest", "project": "torchtune"},
125-
{"package": "pyarrow", "version": "latest", "project": "torchtune"},
126-
{"package": "pyarrow-hotfix", "version": "latest", "project": "torchtune"},
127-
{"package": "pycryptodomex", "version": "latest", "project": "torchtune"},
128-
{"package": "python-dateutil", "version": "latest", "project": "torchtune"},
129-
{"package": "pytz", "version": "latest", "project": "torchtune"},
130-
{"package": "pyyaml", "version": "latest", "project": "torchtune"},
131-
{"package": "regex", "version": "latest", "project": "torchtune"},
132-
{"package": "requests", "version": "latest", "project": "torchtune"},
133-
{"package": "safetensors", "version": "latest", "project": "torchtune"},
134-
{"package": "sentencepiece", "version": "latest", "project": "torchtune"},
135-
{"package": "six", "version": "latest", "project": "torchtune"},
136-
{"package": "tiktoken", "version": "latest", "project": "torchtune"},
137-
{"package": "tqdm", "version": "latest", "project": "torchtune"},
138-
{"package": "tzdata", "version": "latest", "project": "torchtune"},
139-
{"package": "urllib3", "version": "latest", "project": "torchtune"},
140-
{"package": "xxhash", "version": "latest", "project": "torchtune"},
141-
{"package": "yarl", "version": "latest", "project": "torchtune"},
142-
{"package": "dpcpp-cpp-rt", "version": "latest", "project": "torch_xpu"},
143-
{"package": "intel-cmplr-lib-rt", "version": "latest", "project": "torch_xpu"},
144-
{"package": "intel-cmplr-lib-ur", "version": "latest", "project": "torch_xpu"},
145-
{"package": "intel-cmplr-lic-rt", "version": "latest", "project": "torch_xpu"},
146-
{"package": "intel-opencl-rt", "version": "latest", "project": "torch_xpu"},
147-
{"package": "intel-sycl-rt", "version": "latest", "project": "torch_xpu"},
148-
{"package": "intel-openmp", "version": "latest", "project": "torch_xpu"},
149-
{"package": "tcmlib", "version": "latest", "project": "torch_xpu"},
150-
{"package": "umf", "version": "latest", "project": "torch_xpu"},
151-
{"package": "intel-pti", "version": "latest", "project": "torch_xpu"},
152-
{"package": "tbb", "version": "latest", "project": "torch_xpu"},
153-
{"package": "oneccl-devel", "version": "latest", "project": "torch_xpu"},
154-
{"package": "oneccl", "version": "latest", "project": "torch_xpu"},
155-
{"package": "impi-rt", "version": "latest", "project": "torch_xpu"},
156-
{"package": "onemkl-sycl-blas", "version": "latest", "project": "torch_xpu"},
157-
{"package": "onemkl-sycl-dft", "version": "latest", "project": "torch_xpu"},
158-
{"package": "onemkl-sycl-lapack", "version": "latest", "project": "torch_xpu"},
159-
{"package": "onemkl-sycl-sparse", "version": "latest", "project": "torch_xpu"},
160-
{"package": "onemkl-sycl-rng", "version": "latest", "project": "torch_xpu"},
161-
{"package": "mkl", "version": "latest", "project": "torch_xpu"},
162-
]
78+
"arpeggio": {"version": "latest", "project": "triton"},
79+
"caliper-reader": {"version": "latest", "project": "triton"},
80+
"contourpy": {"version": "latest", "project": "triton"},
81+
"cycler": {"version": "latest", "project": "triton"},
82+
"dill": {"version": "latest", "project": "triton"},
83+
"fonttools": {"version": "latest", "project": "triton"},
84+
"kiwisolver": {"version": "latest", "project": "triton"},
85+
"llnl-hatchet": {"version": "latest", "project": "triton"},
86+
"matplotlib": {"version": "latest", "project": "triton"},
87+
"pandas": {"version": "latest", "project": "triton"},
88+
"pydot": {"version": "latest", "project": "triton"},
89+
"pyparsing": {"version": "latest", "project": "triton"},
90+
"pytz": {"version": "latest", "project": "triton"},
91+
"textX": {"version": "latest", "project": "triton"},
92+
"tzdata": {"version": "latest", "project": "triton"},
93+
"importlib-metadata": {"version": "latest", "project": "triton"},
94+
"importlib-resources": {"version": "latest", "project": "triton"},
95+
"zipp": {"version": "latest", "project": "triton"},
96+
"aiohttp": {"version": "latest", "project": "torchtune"},
97+
"aiosignal": {"version": "latest", "project": "torchtune"},
98+
"antlr4-python3-runtime": {"version": "latest", "project": "torchtune"},
99+
"attrs": {"version": "latest", "project": "torchtune"},
100+
"blobfile": {"version": "latest", "project": "torchtune"},
101+
"certifi": {"version": "latest", "project": "torchtune"},
102+
"charset-normalizer": {"version": "latest", "project": "torchtune"},
103+
"datasets": {"version": "latest", "project": "torchtune"},
104+
"dill": {"version": "latest", "project": "torchtune"},
105+
"frozenlist": {"version": "latest", "project": "torchtune"},
106+
"huggingface-hub": {"version": "latest", "project": "torchtune"},
107+
"idna": {"version": "latest", "project": "torchtune"},
108+
"lxml": {"version": "latest", "project": "torchtune"},
109+
"markupsafe": {"version": "latest", "project": "torchtune"},
110+
"multidict": {"version": "latest", "project": "torchtune"},
111+
"multiprocess": {"version": "latest", "project": "torchtune"},
112+
"omegaconf": {"version": "latest", "project": "torchtune"},
113+
"pandas": {"version": "latest", "project": "torchtune"},
114+
"pyarrow": {"version": "latest", "project": "torchtune"},
115+
"pyarrow-hotfix": {"version": "latest", "project": "torchtune"},
116+
"pycryptodomex": {"version": "latest", "project": "torchtune"},
117+
"python-dateutil": {"version": "latest", "project": "torchtune"},
118+
"pytz": {"version": "latest", "project": "torchtune"},
119+
"pyyaml": {"version": "latest", "project": "torchtune"},
120+
"regex": {"version": "latest", "project": "torchtune"},
121+
"requests": {"version": "latest", "project": "torchtune"},
122+
"safetensors": {"version": "latest", "project": "torchtune"},
123+
"sentencepiece": {"version": "latest", "project": "torchtune"},
124+
"six": {"version": "latest", "project": "torchtune"},
125+
"tiktoken": {"version": "latest", "project": "torchtune"},
126+
"tqdm": {"version": "latest", "project": "torchtune"},
127+
"tzdata": {"version": "latest", "project": "torchtune"},
128+
"urllib3": {"version": "latest", "project": "torchtune"},
129+
"xxhash": {"version": "latest", "project": "torchtune"},
130+
"yarl": {"version": "latest", "project": "torchtune"},
131+
"dpcpp-cpp-rt": {"version": "latest", "project": "torch_xpu"},
132+
"intel-cmplr-lib-rt": {"version": "latest", "project": "torch_xpu"},
133+
"intel-cmplr-lib-ur": {"version": "latest", "project": "torch_xpu"},
134+
"intel-cmplr-lic-rt": {"version": "latest", "project": "torch_xpu"},
135+
"intel-opencl-rt": {"version": "latest", "project": "torch_xpu"},
136+
"intel-sycl-rt": {"version": "latest", "project": "torch_xpu"},
137+
"intel-openmp": {"version": "latest", "project": "torch_xpu"},
138+
"tcmlib": {"version": "latest", "project": "torch_xpu"},
139+
"umf": {"version": "latest", "project": "torch_xpu"},
140+
"intel-pti": {"version": "latest", "project": "torch_xpu"},
141+
"tbb": {"version": "latest", "project": "torch_xpu"},
142+
"oneccl-devel": {"version": "latest", "project": "torch_xpu"},
143+
"oneccl": {"version": "latest", "project": "torch_xpu"},
144+
"impi-rt": {"version": "latest", "project": "torch_xpu"},
145+
"onemkl-sycl-blas": {"version": "latest", "project": "torch_xpu"},
146+
"onemkl-sycl-dft": {"version": "latest", "project": "torch_xpu"},
147+
"onemkl-sycl-lapack": {"version": "latest", "project": "torch_xpu"},
148+
"onemkl-sycl-sparse": {"version": "latest", "project": "torch_xpu"},
149+
"onemkl-sycl-rng": {"version": "latest", "project": "torch_xpu"},
150+
"mkl": {"version": "latest", "project": "torch_xpu"},
151+
}
163152

164153

165154
def download(url: str) -> bytes:
@@ -265,7 +254,9 @@ def main() -> None:
265254

266255
parser = ArgumentParser("Upload dependent packages to s3://pytorch")
267256
# Get unique paths from the packages list
268-
project_paths = list(set(pkg["project"] for pkg in PACKAGES_PER_PROJECT))
257+
project_paths = list(
258+
set(pkg_info["project"] for pkg_info in PACKAGES_PER_PROJECT.values())
259+
)
269260
parser.add_argument("--package", choices=project_paths, default="torch")
270261
parser.add_argument("--dry-run", action="store_true")
271262
parser.add_argument("--only-pypi", action="store_true")
@@ -278,17 +269,19 @@ def main() -> None:
278269

279270
for prefix in SUBFOLDERS:
280271
# Filter packages by the selected project path
281-
selected_packages = [
282-
pkg for pkg in PACKAGES_PER_PROJECT if pkg["project"] == args.package
283-
]
284-
for pkg_info in selected_packages:
285-
if hasattr(pkg_info, "target") and pkg_info["target"] != "":
272+
selected_packages = {
273+
pkg_name: pkg_info
274+
for pkg_name, pkg_info in PACKAGES_PER_PROJECT.items()
275+
if pkg_info["project"] == args.package
276+
}
277+
for pkg_name, pkg_info in selected_packages.items():
278+
if "target" in pkg_info and pkg_info["target"] != "":
286279
full_path = f'{prefix}/{pkg_info["target"]}'
287280
else:
288281
full_path = f"{prefix}"
289282

290283
upload_missing_whls(
291-
pkg_info["package"],
284+
pkg_name,
292285
full_path,
293286
dry_run=args.dry_run,
294287
only_pypi=args.only_pypi,

0 commit comments

Comments
 (0)