Skip to content

Commit 372f050

Browse files
Automate nightly build schedules, add new workflows for adding/removing nightly versions (#650)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1 parent 69222a3 commit 372f050

13 files changed

+632
-295
lines changed
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Workflow to start nightly build(s) upon new image version release.
2+
# Triggered by release (https://github.com/aws/sagemaker-distribution/releases) publication, for `*-cpu` only so we run once per image version.
3+
# Can also be triggered manually.
4+
# When a minor version (X.y.0) is released, we'll start the next minor (X.(y+1).0) and patch (X.y.1).
5+
# When a patch version (X.y.z) is released, we'll start the next patch (X.y.(z+1)).
6+
name: Add New Nightly Build(s)
7+
on:
8+
release:
9+
types: [published]
10+
workflow_dispatch:
11+
inputs:
12+
version:
13+
description: 'Released version to add next versions for (x.y.z)'
14+
required: true
15+
type: string
16+
concurrency:
17+
group: schedule-update
18+
cancel-in-progress: false
19+
defaults:
20+
run:
21+
shell: bash -l {0}
22+
jobs:
23+
update-nightly-schedule:
24+
name: Update NIGHTLY_BUILD_SCHEDULE with new version(s).
25+
runs-on: ubuntu-latest
26+
# Only run for CPU releases or manual triggers
27+
if: |
28+
github.event_name == 'workflow_dispatch' ||
29+
(github.event_name == 'release' && endsWith(github.event.release.tag_name, '-cpu'))
30+
steps:
31+
- uses: actions/checkout@v4
32+
- uses: mamba-org/setup-micromamba@v2
33+
with:
34+
environment-file: ./environment.lock
35+
environment-name: sagemaker-distribution
36+
init-shell: bash
37+
- name: Set version
38+
id: version
39+
run: |
40+
if [ "${{ github.event_name }}" == "release" ]; then
41+
# Extract x.y.z from x.y.z-cpu
42+
VERSION=$(echo "${{ github.event.release.tag_name }}" | sed 's/-cpu$//')
43+
echo "version=$VERSION" >> $GITHUB_OUTPUT
44+
else
45+
# Use manually provided version
46+
echo "version=${{ inputs.version }}" >> $GITHUB_OUTPUT
47+
fi
48+
- name: Update nightly build schedule
49+
id: update-schedule
50+
env:
51+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
52+
run: |
53+
micromamba activate sagemaker-distribution
54+
python ./.github/workflows/utils/nightly_build_helper.py \
55+
add-next-versions ${{ steps.version.outputs.version }}

.github/workflows/check-image-size.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ jobs:
3939
contents: write
4040
steps:
4141
- uses: actions/checkout@v4
42-
- uses: mamba-org/setup-micromamba@v1
42+
- uses: mamba-org/setup-micromamba@v2
4343
with:
4444
environment-file: ./environment.lock
4545
environment-name: sagemaker-distribution

.github/workflows/check_code_quality.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ jobs:
2929
contents: write
3030
steps:
3131
- uses: actions/checkout@v4
32-
- uses: mamba-org/setup-micromamba@v1
32+
- uses: mamba-org/setup-micromamba@v2
3333
with:
3434
environment-file: ./environment.lock
3535
environment-name: sagemaker-distribution
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Kickoff Nightly builds
2+
run-name: Kickoff nightly builds
3+
on:
4+
# Run manually
5+
workflow_dispatch:
6+
# Run on Mon-Fri at 4PM PST / 5PM PDT
7+
schedule:
8+
- cron: '0 0 * * 1-5'
9+
jobs:
10+
generate-version-matrix:
11+
name: Generate version matrix
12+
runs-on: ubuntu-latest
13+
if: github.repository == 'aws/sagemaker-distribution'
14+
outputs:
15+
matrix: ${{ steps.set-matrix.outputs.matrix }}
16+
patch-versions: ${{ steps.set-matrix.outputs.patch-versions }}
17+
minor-versions: ${{ steps.set-matrix.outputs.minor-versions }}
18+
steps:
19+
- uses: actions/checkout@v4
20+
- name: Get versions from schedule
21+
id: set-matrix
22+
env:
23+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
24+
run: |
25+
SCHEDULE='${{ vars.NIGHTLY_BUILD_SCHEDULE }}'
26+
echo "Triggering builds for $(echo $SCHEDULE | jq -c '.active_nightly_builds')"
27+
echo "matrix=$(echo $SCHEDULE | jq -c)" >> $GITHUB_OUTPUT
28+
echo "patch-versions=$(echo $SCHEDULE | jq -c '.patch_base_versions')" >> $GITHUB_OUTPUT
29+
echo "minor-versions=$(echo $SCHEDULE | jq -c '.minor_base_versions')" >> $GITHUB_OUTPUT
30+
start-minor-build:
31+
name: Start nightly minor build
32+
needs: generate-version-matrix
33+
permissions:
34+
pull-requests: write
35+
contents: write
36+
id-token: write
37+
strategy:
38+
matrix:
39+
version: ${{ fromJson(needs.generate-version-matrix.outputs.minor-versions) }}
40+
fail-fast: false
41+
uses: aws/sagemaker-distribution/.github/workflows/build-image.yml@main
42+
secrets: inherit
43+
with:
44+
release-type: "minor"
45+
base-version: ${{ matrix.version }}
46+
start-patch-build:
47+
name: Start nightly patch build
48+
needs: generate-version-matrix
49+
permissions:
50+
pull-requests: write
51+
contents: write
52+
id-token: write
53+
strategy:
54+
matrix:
55+
version: ${{ fromJson(needs.generate-version-matrix.outputs.patch-versions) }}
56+
fail-fast: false
57+
uses: aws/sagemaker-distribution/.github/workflows/build-image.yml@main
58+
secrets: inherit
59+
with:
60+
release-type: "patch"
61+
base-version: ${{ matrix.version }}

.github/workflows/nightly-minor-build.yml

Lines changed: 0 additions & 59 deletions
This file was deleted.
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Workflow manually triggered to stop nightly builds for a given image version.
2+
name: Stop Nightly Build
3+
run-name: Stop Nightly Build (${{ inputs.version }})
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
version:
8+
description: 'Version to stop building (x.y.z, e.g. 3.0.0)'
9+
required: true
10+
type: string
11+
concurrency:
12+
group: schedule-update
13+
cancel-in-progress: false
14+
defaults:
15+
run:
16+
shell: bash -l {0}
17+
jobs:
18+
update-nightly-schedule:
19+
name: Update nightly build schedule
20+
runs-on: ubuntu-latest
21+
steps:
22+
- uses: actions/checkout@v4
23+
- uses: mamba-org/setup-micromamba@v2
24+
with:
25+
environment-file: ./environment.lock
26+
environment-name: sagemaker-distribution
27+
init-shell: bash
28+
- name: Update nightly_build_schedule.json
29+
id: update-schedule
30+
env:
31+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
32+
run: |
33+
micromamba activate sagemaker-distribution
34+
python ./.github/workflows/utils/nightly_build_helper.py remove-version ${{ inputs.version }}

.github/workflows/update-environment-lock.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
contents: write
1919
steps:
2020
- uses: actions/checkout@v4
21-
- uses: mamba-org/setup-micromamba@v1
21+
- uses: mamba-org/setup-micromamba@v2
2222
with:
2323
environment-file: ./environment.yml
2424
environment-name: sagemaker-distribution
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
import argparse
2+
import json
3+
import os
4+
import semver
5+
import sys
6+
from github import Github
7+
from packaging.version import Version
8+
9+
class NightlyBuildHelper:
10+
def __init__(self):
11+
"""Initialize with GitHub credentials from environment variables."""
12+
token = os.environ.get('GH_TOKEN')
13+
repo_name = os.environ.get('GITHUB_REPOSITORY')
14+
15+
if not token:
16+
raise ValueError("GH_TOKEN environment variable is required")
17+
if not repo_name:
18+
raise ValueError("GITHUB_REPOSITORY environment variable is required")
19+
20+
self.repo = Github(token).get_repo(repo_name)
21+
self.schedule_variable, self.current_schedule = self._load_schedule()
22+
23+
def _load_schedule(self):
24+
"""Load the schedule from GitHub Actions variable."""
25+
try:
26+
# Get the NIGHTLY_BUILD_SCHEDULE variable
27+
schedule_var = self.repo.get_variable("NIGHTLY_BUILD_SCHEDULE")
28+
schedule = json.loads(schedule_var.value)
29+
30+
# Initialize lists if they don't exist
31+
schedule.setdefault("active_nightly_builds", [])
32+
schedule.setdefault("patch_base_versions", [])
33+
schedule.setdefault("minor_base_versions", [])
34+
return schedule_var, schedule
35+
except Exception as e:
36+
print(f"Error loading schedule from GitHub: {e}")
37+
sys.exit(1)
38+
39+
def _save_schedule(self):
40+
"""Save the current schedule to GitHub Actions variable."""
41+
try:
42+
schedule_json = json.dumps(self.current_schedule, indent=4, sort_keys=True)
43+
print(f"Updated schedule: {schedule_json}")
44+
# Update the existing variable
45+
self.schedule_variable.edit(schedule_json)
46+
print("Successfully updated NIGHTLY_BUILD_SCHEDULE variable: https://github.com/aws/sagemaker-distribution/settings/variables/actions")
47+
except Exception as e:
48+
print(f"Error saving schedule to GitHub: {e}")
49+
sys.exit(1)
50+
51+
def _sort_lists(self):
52+
"""Sort all version lists in the schedule."""
53+
self.current_schedule["active_nightly_builds"].sort(key=Version)
54+
self.current_schedule["patch_base_versions"].sort(key=Version)
55+
self.current_schedule["minor_base_versions"].sort(key=Version)
56+
57+
def remove_version(self, version):
58+
"""Remove a version from active builds and update base versions accordingly."""
59+
version_obj = semver.VersionInfo.parse(version)
60+
if str(version_obj) != version:
61+
raise ValueError(f"Version must be in x.y.z format, got: {version}")
62+
63+
print(f"Current schedule: {json.dumps(self.current_schedule, indent=4, sort_keys=True)}")
64+
print(f"Removing version: {version}")
65+
66+
if version not in self.current_schedule["active_nightly_builds"]:
67+
print(f"Version {version} not found in active nightly builds schedule.")
68+
return
69+
70+
# Remove from active builds
71+
self.current_schedule["active_nightly_builds"].remove(version)
72+
73+
if version_obj.patch == 0: # Handling minor version
74+
# Remove previous minor version from minor_base_versions
75+
self.current_schedule["minor_base_versions"] = [
76+
v for v in self.current_schedule["minor_base_versions"]
77+
if not v.startswith(f"{version_obj.major}.{version_obj.minor-1}")
78+
]
79+
else: # Handling patch version
80+
prev_version = str(version_obj.replace(patch=version_obj.patch - 1))
81+
if prev_version in self.current_schedule["patch_base_versions"]:
82+
self.current_schedule["patch_base_versions"].remove(prev_version)
83+
84+
self._sort_lists()
85+
self._save_schedule()
86+
87+
def add_next_versions(self, version):
88+
"""Add next version(s) based on the removed version."""
89+
version_obj = semver.VersionInfo.parse(version)
90+
if str(version_obj) != version:
91+
raise ValueError(f"Version must be in x.y.z format, got: {version}")
92+
93+
print(f"Current schedule: {self.current_schedule}")
94+
print(f"Adding next versions for released: {version}")
95+
96+
next_versions = [str(version_obj.bump_patch())]
97+
if version_obj.patch == 0: # Handling minor version
98+
next_versions.append(str(version_obj.bump_minor()))
99+
self.current_schedule["active_nightly_builds"].extend(next_versions)
100+
self.current_schedule["patch_base_versions"].append(version)
101+
self.current_schedule["minor_base_versions"].append(version)
102+
else: # Handling patch version
103+
self.current_schedule["active_nightly_builds"].extend(next_versions)
104+
self.current_schedule["patch_base_versions"].append(version)
105+
prev_version = str(version_obj.replace(patch=version_obj.patch - 1))
106+
if prev_version in self.current_schedule["minor_base_versions"]:
107+
self.current_schedule["minor_base_versions"].remove(prev_version)
108+
self.current_schedule["minor_base_versions"].append(version)
109+
110+
self._sort_lists()
111+
self._save_schedule()
112+
113+
def main():
114+
parser = argparse.ArgumentParser(description='Nightly build helper tool')
115+
subparsers = parser.add_subparsers(dest='command', help='Commands')
116+
117+
# Remove version command
118+
remove_parser = subparsers.add_parser('remove-version',
119+
help='Remove a version from active builds')
120+
remove_parser.add_argument('version',
121+
help='Version to remove (e.g., 1.2.3)')
122+
123+
# Add next versions command
124+
add_parser = subparsers.add_parser('add-next-versions',
125+
help='Add next version(s) based on released version')
126+
add_parser.add_argument('version',
127+
help='Version that was released (e.g., 1.2.3)')
128+
129+
args = parser.parse_args()
130+
131+
if not args.command:
132+
parser.print_help()
133+
sys.exit(1)
134+
135+
try:
136+
helper = NightlyBuildHelper()
137+
138+
if args.command == 'remove-version':
139+
helper.remove_version(args.version)
140+
elif args.command == 'add-next-versions':
141+
helper.add_next_versions(args.version)
142+
else:
143+
print(f"Unknown command: {args.command}")
144+
parser.print_help()
145+
sys.exit(1)
146+
except Exception as e:
147+
print(f"Error: {e}")
148+
sys.exit(1)
149+
150+
if __name__ == "__main__":
151+
main()

0 commit comments

Comments
 (0)