Merge branch 'main' into release-2.7.0

Author: aws-tianquaw

.github/workflows/build-image.yml

@@ -42,7 +42,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: mamba-org/setup-micromamba@v1
         with:
-          environment-file: ./environment.yml
+          environment-file: ./environment.lock
           environment-name: sagemaker-distribution
           init-shell: bash
       - name: Free up disk space
@@ -54,30 +54,51 @@ jobs:
         run: |
           TARGET_VERSION=$(python -c 'import semver; print(semver.bump_${{ inputs.release-type }}("${{ inputs.base-version }}"))')
           echo "target_version=$TARGET_VERSION" >> $GITHUB_OUTPUT
-      - name: Create new branch
+      - name: Create or merge release branch
         # Note - CodeBuild depends on this branch name. Don't change without corresponding backend change.
-        run: git checkout -b release-${{ steps.calc_target.outputs.target_version }}
+        id: check_branch
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          if git ls-remote --exit-code --heads origin release-${{ steps.calc_target.outputs.target_version }}; then
+            echo "Branch exists, merging latest changes in"
+            git checkout release-${{ steps.calc_target.outputs.target_version }}
+            git merge main
+            git push
+          else
+            echo "Branch doesn't exist, creating now..."
+            git checkout -b release-${{ steps.calc_target.outputs.target_version }}
+          fi
       - name: Generate artifacts
         run: python ./src/main.py create-${{ inputs.release-type }}-version-artifacts --base-patch-version ${{ inputs.base-version }} --force
       - name: Commit .in artifacts to branch
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # Generate the artifacts every time, only push if there are changes
         run: |
           git config --local user.email "github-actions[bot]@users.noreply.github.com"
           git config --local user.name "github-actions[bot]"
           git add ./build_artifacts
-          git commit -m 'chore: Generate build artifacts for ${{ steps.calc_target.outputs.target_version }} release'
+          git diff-index --quiet HEAD || git commit -m 'chore: Generate build artifacts for ${{ steps.calc_target.outputs.target_version }} release'
           git push --set-upstream origin release-${{ steps.calc_target.outputs.target_version }}
-      - name: Open pull request
+      - name: Create or fetch release PR
         id: get_pr_id
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        # Note - CodeBuild depends on this PR title. Don't change without corresponding backend change.
+        # Create PR if it doesn't exist; fetch existing PR number if it does
         run: |
-          URL=$(gh pr create -H release-${{ steps.calc_target.outputs.target_version }} \
-                    --title 'release: v${{ steps.calc_target.outputs.target_version }}' -F ./.github/workflows/PR_TEMPLATE.md)
-          PR=$(echo $URL | sed 's:.*/::')
-          echo "pr_id=$PR" >> $GITHUB_OUTPUT
+          if gh pr list --head release-${{ steps.calc_target.outputs.target_version }} --state open --json number | grep -q '"number":'; then
+            echo "PR exists already, just fetching ID"
+            PR_ID=$(gh pr view release-${{ steps.calc_target.outputs.target_version }} --json number | jq -r .number)
+            echo "pr_id=$PR_ID" >> $GITHUB_OUTPUT
+          else
+            echo "PR doesn't exist, creating now..."
+            git push --set-upstream origin release-${{ steps.calc_target.outputs.target_version }}
+            URL=$(gh pr create -H release-${{ steps.calc_target.outputs.target_version }} -B main \
+                  --title 'release: v${{ steps.calc_target.outputs.target_version }}' -F ./.github/workflows/PR_TEMPLATE.md)
+            PR_ID=$(echo $URL | sed 's:.*/::')
+            echo "pr_id=$PR_ID" >> $GITHUB_OUTPUT
+          fi
   call-codebuild-project:
     runs-on: ubuntu-latest
     needs: open-pr

.github/workflows/check-image-size.yml

@@ -41,7 +41,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: mamba-org/setup-micromamba@v1
         with:
-          environment-file: ./environment.yml
+          environment-file: ./environment.lock
           environment-name: sagemaker-distribution
           init-shell: bash
       - name: Free up disk space

.github/workflows/check_code_quality.yml

@@ -31,7 +31,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: mamba-org/setup-micromamba@v1
         with:
-          environment-file: ./environment.yml
+          environment-file: ./environment.lock
           environment-name: sagemaker-distribution
           init-shell: bash
       - name: Free up disk space

.github/workflows/monthly-minor-release.yml renamed to .github/workflows/nightly-minor-build.yml

@@ -1,8 +1,11 @@
-name: Generate monthly minor release(s)
+name: Kickoff nightly Minor version builds
+run-name: Kickoff Minor nightly builds $(date +'%m-%d-%Y')
 on:
-  # Run first day of the month at 8AM PST / 9AM PDT
-  #schedule:
-  #  - cron: '0 16 1 * *'
+  # Run manually
+  workflow_dispatch:
+  # Run on 1st-10th day each month at 4PM PST / 5PM PDT
+  schedule:
+    - cron: '0 0 1-10 * *'
 jobs:
   generate-version-matrix:
     name: Generate version matrix
@@ -12,14 +15,25 @@ jobs:
       matrix: ${{ steps.gen-mat.outputs.matrix }}
     steps:
       - uses: actions/checkout@v4
+      - name: Get supported Major versions
+        id: supported-major-versions
+        # Check support_policy.md to determine active Major versions
+        run: |
+            major_versions=$(sed -n '/^### CPU Images/,/^### GPU Images/{
+                /^### GPU Images/q
+                /^| [0-9]/{
+                    s/^| \([0-9]\+\).*/\1/p
+                }
+            }' support_policy.md | sort -n | uniq | tr '\n' ',' | sed 's/,$//' | sed 's/,/","/g')
+            echo "majors=[\"$major_versions\"]" >> $GITHUB_OUTPUT
       - name: Generate base patch version matrix
         id: gen-mat
-        # Output looks like: matrix={"version":["0.11.0","1.4.1"]}
-        # For each major, get highest minor version. Then get highest patch
-        # for that minor. Use this patch as base version.
+        # Output looks like: matrix={"version":["1.13.1","2.5.0","3.0.0"]}
+        # For each active Major version, get highest Minor version. Then get highest Patch
+        # for that Minor. Use this Patch as base version.
         run: |
           versions=("{\"version\":[")
-          for major in build_artifacts/v*; do
+          for major in build_artifacts/v${{ steps.supported-major-versions.outputs.majors }}; do
             highest_minor=$(ls $major | sort -t. -k2n | tail -n1)
             highest_patch=$(ls $major/$highest_minor | sort -t. -k2n | tail -n1)
             versions+="\"${highest_patch#v}\""
@@ -28,7 +42,7 @@ jobs:
           versions=${versions::-1}
           versions+="]}"
           echo "matrix=$versions" >> $GITHUB_OUTPUT
-  start-monthly-minor:
+  start-minor-build:
     name: Start monthly minor release
     needs: generate-version-matrix
     permissions:
@@ -43,15 +57,3 @@ jobs:
     with:
       release-type: "minor"
       base-version: ${{ matrix.version }}
-  check-image-size:
-    name: Check Image Size
-    needs: start-monthly-minor
-    permissions:
-      pull-requests: write
-      contents: write
-    strategy:
-      matrix: ${{ fromJson(needs.generate-version-matrix.outputs.matrix) }}
-      fail-fast: false
-    uses: aws/sagemaker-distribution/.github/workflows/check-image-size.yml@main
-    with:
-      base-version: ${{ matrix.version }}

.github/workflows/update-environment-lock.yml

@@ -0,0 +1,36 @@
+name: Update environment.lock
+# Update environment.lock weekly, so we can quickly install environment without having
+# to solve environment.yml every time.
+on:
+  # Every Monday
+  schedule:
+    - cron: '0 0 * * MON'
+  # Manually call
+  workflow_dispatch:
+defaults:
+  run:
+    shell: bash -l {0}
+jobs:
+  update-environment:
+    name: Install environment.yml, generate environment.lock
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: ./environment.yml
+          environment-name: sagemaker-distribution
+          init-shell: bash
+      - name: Create environment-lock.yml
+        run: micromamba env export --explicit > environment.lock
+      - name: Run sagemaker-distribution unit tests to check for regressions
+        run: pytest -m unit
+      - name: Commit changes
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add environment.lock
+          git commit -m 'Update environment.lock'
+          git push

CONTRIBUTING.md

@@ -61,7 +61,7 @@ Follow these steps for sending out a pull request for adding new packages:
    ```
    This project uses Conda to manage its dependencies. Run the following to setup your local environment:
 
-   conda env update --file environment.yml -n sagemaker-distribution
+   conda env update --file environment.lock -n sagemaker-distribution
 
    conda activate sagemaker-distribution
 

DEVELOPMENT.md

@@ -3,7 +3,7 @@
 This project uses Conda to manage its dependencies. Run the following to setup your local environment:
 
 ```shell
-conda env update --file environment.yml -n sagemaker-distribution
+conda env update --file environment.lock -n sagemaker-distribution
 ```
 
 ## Tests

build_artifacts/v2/v2.4/v2.4.3/CHANGELOG-cpu.md

@@ -0,0 +1,11 @@
+# Change log: 2.4.3(cpu)
+
+## Upgrades: 
+
+Package | Previous Version | Current Version
+---|---|---
+amazon-q-developer-jupyterlab-ext|3.4.7|3.4.8
+langchain|0.3.23|0.3.24
+uvicorn|0.34.0|0.34.2
+sagemaker-code-editor|1.4.2|1.4.3
+sagemaker-studio-analytics-extension|0.1.6|0.1.7

build_artifacts/v2/v2.4/v2.4.3/CHANGELOG-gpu.md

@@ -0,0 +1,11 @@
+# Change log: 2.4.3(gpu)
+
+## Upgrades: 
+
+Package | Previous Version | Current Version
+---|---|---
+amazon-q-developer-jupyterlab-ext|3.4.7|3.4.8
+langchain|0.3.23|0.3.24
+uvicorn|0.34.0|0.34.2
+sagemaker-code-editor|1.4.2|1.4.3
+sagemaker-studio-analytics-extension|0.1.6|0.1.7