Skip to content

Commit 9def441

Browse files
wbo4958hcho3
wbo4958
and
hcho3
authored
[CI] add script to generate meta info and upload to s3 (dmlc#10295)
* [CI] add script to generate meta info and upload to s3 * Write Python script to generate meta.json * Update other pipelines * Add wheel_name field * Add description --------- Co-authored-by: Hyunsu Cho <[email protected]>
1 parent 5086dec commit 9def441

File tree

7 files changed

+147
-42
lines changed

7 files changed

+147
-42
lines changed

‎tests/buildkite/build-cpu-arm64.sh

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,17 @@ $command_wrapper bash -c "cd build && ctest --extra-verbose"
1919
echo "--- Build binary wheel"
2020
$command_wrapper bash -c \
2121
"cd python-package && rm -rf dist/* && pip wheel --no-deps -v . --wheel-dir dist/"
22-
$command_wrapper python tests/ci_build/rename_whl.py python-package/dist/*.whl \
23-
${BUILDKITE_COMMIT} ${WHEEL_TAG}
22+
$command_wrapper python tests/ci_build/rename_whl.py \
23+
--wheel-path python-package/dist/*.whl \
24+
--commit-hash ${BUILDKITE_COMMIT} \
25+
--platform-tag ${WHEEL_TAG}
2426

2527
echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
2628
$command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl
27-
$command_wrapper python tests/ci_build/rename_whl.py wheelhouse/*.whl \
28-
${BUILDKITE_COMMIT} ${WHEEL_TAG}
29+
$command_wrapper python tests/ci_build/rename_whl.py \
30+
--wheel-path wheelhouse/*.whl \
31+
--commit-hash ${BUILDKITE_COMMIT} \
32+
--platform-tag ${WHEEL_TAG}
2933
mv -v wheelhouse/*.whl python-package/dist/
3034
# Make sure that libgomp.so is vendored in the wheel
3135
$command_wrapper bash -c \

‎tests/buildkite/build-cuda-with-rmm.sh

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,18 @@ $command_wrapper tests/ci_build/build_via_cmake.sh \
3636
echo "--- Build binary wheel"
3737
$command_wrapper bash -c \
3838
"cd python-package && rm -rf dist/* && pip wheel --no-deps -v . --wheel-dir dist/"
39-
$command_wrapper python tests/ci_build/rename_whl.py python-package/dist/*.whl \
40-
${BUILDKITE_COMMIT} ${WHEEL_TAG}
39+
$command_wrapper python tests/ci_build/rename_whl.py \
40+
--wheel-path python-package/dist/*.whl \
41+
--commit-hash ${BUILDKITE_COMMIT} \
42+
--platform-tag ${WHEEL_TAG}
4143

4244
echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
4345
tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
4446
--plat ${WHEEL_TAG} python-package/dist/*.whl
45-
$command_wrapper python tests/ci_build/rename_whl.py wheelhouse/*.whl \
46-
${BUILDKITE_COMMIT} ${WHEEL_TAG}
47+
$command_wrapper python tests/ci_build/rename_whl.py \
48+
--wheel-path wheelhouse/*.whl \
49+
--commit-hash ${BUILDKITE_COMMIT} \
50+
--platform-tag ${WHEEL_TAG}
4751
mv -v wheelhouse/*.whl python-package/dist/
4852
# Make sure that libgomp.so is vendored in the wheel
4953
tests/ci_build/ci_build.sh auditwheel_x86_64 bash -c \

‎tests/buildkite/build-cuda.sh

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,14 +35,18 @@ $command_wrapper tests/ci_build/build_via_cmake.sh \
3535
echo "--- Build binary wheel"
3636
$command_wrapper bash -c \
3737
"cd python-package && rm -rf dist/* && pip wheel --no-deps -v . --wheel-dir dist/"
38-
$command_wrapper python tests/ci_build/rename_whl.py python-package/dist/*.whl \
39-
${BUILDKITE_COMMIT} ${WHEEL_TAG}
38+
$command_wrapper python tests/ci_build/rename_whl.py \
39+
--wheel-path python-package/dist/*.whl \
40+
--commit-hash ${BUILDKITE_COMMIT} \
41+
--platform-tag ${WHEEL_TAG}
4042

4143
echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
4244
tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
4345
--plat ${WHEEL_TAG} python-package/dist/*.whl
44-
$command_wrapper python tests/ci_build/rename_whl.py wheelhouse/*.whl \
45-
${BUILDKITE_COMMIT} ${WHEEL_TAG}
46+
$command_wrapper python tests/ci_build/rename_whl.py \
47+
--wheel-path wheelhouse/*.whl \
48+
--commit-hash ${BUILDKITE_COMMIT} \
49+
--platform-tag ${WHEEL_TAG}
4650
mv -v wheelhouse/*.whl python-package/dist/
4751
# Make sure that libgomp.so is vendored in the wheel
4852
tests/ci_build/ci_build.sh auditwheel_x86_64 bash -c \
@@ -54,6 +58,15 @@ if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
5458
then
5559
aws s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
5660
--acl public-read --no-progress
61+
62+
# Generate the meta info which includes xgboost version and the commit info
63+
$command_wrapper python tests/ci_build/format_wheel_meta.py \
64+
--wheel-path python-package/dist/*.whl \
65+
--commit-hash ${BUILDKITE_COMMIT} \
66+
--platform-tag ${WHEEL_TAG} \
67+
--meta-path python-package/dist/
68+
aws s3 cp python-package/dist/meta.json s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
69+
--acl public-read --no-progress
5770
fi
5871
echo "-- Stash C++ test executable (testxgboost)"
5972
buildkite-agent artifact upload build/testxgboost

‎tests/buildkite/build-win64-gpu.ps1

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@ conda activate
2929
& pip wheel --no-deps -v . --wheel-dir dist/
3030
Get-ChildItem . -Filter dist/*.whl |
3131
Foreach-Object {
32-
& python ../tests/ci_build/rename_whl.py $_.FullName $Env:BUILDKITE_COMMIT win_amd64
32+
& python ../tests/ci_build/rename_whl.py `
33+
--wheel-path $_.FullName `
34+
--commit-hash $Env:BUILDKITE_COMMIT `
35+
--platform-tag win_amd64
3336
if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
3437
}
3538

@@ -44,7 +47,7 @@ if ( $is_release_branch -eq 1 ) {
4447
Get-ChildItem . -Filter python-package/dist/*.whl |
4548
Foreach-Object {
4649
& aws s3 cp python-package/dist/$_ s3://xgboost-nightly-builds/$Env:BUILDKITE_BRANCH/ `
47-
--acl public-read --no-progress
50+
--acl public-read --no-progress
4851
if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
4952
}
5053
}

‎tests/ci_build/build_python_wheels.sh

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,4 +67,7 @@ fi
6767

6868
python -m pip install cibuildwheel
6969
python -m cibuildwheel python-package --output-dir wheelhouse
70-
python tests/ci_build/rename_whl.py wheelhouse/*.whl ${commit_id} ${wheel_tag}
70+
python tests/ci_build/rename_whl.py \
71+
--wheel-path wheelhouse/*.whl \
72+
--commit-hash ${commit_id} \
73+
--platform-tag ${wheel_tag}

‎tests/ci_build/format_wheel_meta.py

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
"""
2+
Script to generate meta.json to store metadata for a nightly build of
3+
XGBoost Python package.
4+
"""
5+
import json
6+
import pathlib
7+
from argparse import ArgumentParser
8+
9+
10+
def main(args):
11+
wheel_path = pathlib.Path(args.wheel_path).expanduser().resolve()
12+
if not wheel_path.exists():
13+
raise ValueError(f"Wheel cannot be found at path {wheel_path}")
14+
if not wheel_path.is_file():
15+
raise ValueError(f"Path {wheel_path} is not a valid file")
16+
wheel_dir, wheel_name = wheel_path.parent, wheel_path.name
17+
18+
meta_path = pathlib.Path(args.meta_path)
19+
if not meta_path.exists():
20+
raise ValueError(f"Path {meta_path} does not exist")
21+
if not meta_path.is_dir():
22+
raise ValueError(f"Path {meta_path} is not a valid directory")
23+
24+
tokens = wheel_name.split("-")
25+
assert len(tokens) == 5
26+
version = tokens[1].split("+")[0]
27+
28+
meta_info = {
29+
"wheel_name": wheel_name,
30+
"platform_tag": args.platform_tag,
31+
"version": version,
32+
"commit_id": args.commit_hash,
33+
}
34+
with open(meta_path / "meta.json", "w") as f:
35+
json.dump(meta_info, f, indent=4)
36+
37+
38+
if __name__ == "__main__":
39+
parser = ArgumentParser(
40+
description="Format meta.json encoding the latest nightly version of the Python wheel"
41+
)
42+
parser.add_argument(
43+
"--wheel-path", type=str, required=True, help="Path to the wheel"
44+
)
45+
parser.add_argument(
46+
"--commit-hash", type=str, required=True, help="Git commit hash"
47+
)
48+
parser.add_argument(
49+
"--platform-tag",
50+
type=str,
51+
required=True,
52+
help="Platform tag (e.g. manylinux2014_x86_64)",
53+
)
54+
parser.add_argument(
55+
"--meta-path", type=str, required=True, help="Directory to place meta.json"
56+
)
57+
parsed_args = parser.parse_args()
58+
main(parsed_args)

‎tests/ci_build/rename_whl.py

Lines changed: 47 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,59 @@
1-
import os
2-
import sys
1+
import pathlib
2+
from argparse import ArgumentParser
33

4-
from test_utils import DirectoryExcursion
54

6-
if len(sys.argv) != 4:
7-
print("Usage: {} [wheel to rename] [commit id] [platform tag]".format(sys.argv[0]))
8-
sys.exit(1)
5+
def main(args):
6+
wheel_path = pathlib.Path(args.wheel_path).expanduser().resolve()
7+
if not wheel_path.exists():
8+
raise ValueError(f"Wheel cannot be found at path {wheel_path}")
9+
if not wheel_path.is_file():
10+
raise ValueError(f"Path {wheel_path} is not a valid file")
11+
wheel_dir, wheel_name = wheel_path.parent, wheel_path.name
912

10-
11-
whl_path = sys.argv[1]
12-
commit_id = sys.argv[2]
13-
platform_tag = sys.argv[3]
14-
15-
dirname, basename = os.path.dirname(whl_path), os.path.basename(whl_path)
16-
17-
with DirectoryExcursion(dirname):
18-
tokens = basename.split("-")
13+
tokens = wheel_name.split("-")
1914
assert len(tokens) == 5
2015
version = tokens[1].split("+")[0]
2116
keywords = {
2217
"pkg_name": tokens[0],
2318
"version": version,
24-
"commit_id": commit_id,
25-
"platform_tag": platform_tag,
19+
"commit_id": args.commit_hash,
20+
"platform_tag": args.platform_tag,
2621
}
27-
new_name = "{pkg_name}-{version}+{commit_id}-py3-none-{platform_tag}.whl".format(
28-
**keywords
22+
new_wheel_name = (
23+
"{pkg_name}-{version}+{commit_id}-py3-none-{platform_tag}.whl".format(
24+
**keywords
25+
)
2926
)
30-
print("Renaming {} to {}...".format(basename, new_name))
31-
if os.path.isfile(new_name):
32-
os.remove(new_name)
33-
os.rename(basename, new_name)
27+
new_wheel_path = wheel_dir / new_wheel_name
28+
print(f"Renaming {wheel_name} to {new_wheel_name}...")
29+
if new_wheel_path.is_file():
30+
new_wheel_path.unlink()
31+
wheel_path.rename(new_wheel_path)
32+
33+
filesize = new_wheel_path.stat().st_size / 1024 / 1024 # MiB
34+
print(f"Wheel size: {filesize:.2f} MiB")
35+
36+
if filesize > 300:
37+
raise RuntimeError(
38+
f"Limit of wheel size set by PyPI is exceeded. {new_wheel_name}: {filesize:.2f} MiB"
39+
)
3440

35-
filesize = os.path.getsize(new_name) / 1024 / 1024 # MB
36-
print(f"Wheel size: {filesize}")
3741

38-
msg = f"Limit of wheel size set by PyPI is exceeded. {new_name}: {filesize}"
39-
assert filesize <= 300, msg
42+
if __name__ == "__main__":
43+
parser = ArgumentParser(
44+
description="Format a Python wheel's name using the git commit hash and platform tag"
45+
)
46+
parser.add_argument(
47+
"--wheel-path", type=str, required=True, help="Path to the wheel"
48+
)
49+
parser.add_argument(
50+
"--commit-hash", type=str, required=True, help="Git commit hash"
51+
)
52+
parser.add_argument(
53+
"--platform-tag",
54+
type=str,
55+
required=True,
56+
help="Platform tag (e.g. manylinux2014_x86_64)",
57+
)
58+
parsed_args = parser.parse_args()
59+
main(parsed_args)

0 commit comments

Comments
 (0)