From 5b84907c4f8244aab321a1cf2a432d8700e9a679 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Thu, 14 Nov 2024 11:46:36 -0800 Subject: [PATCH 1/5] Automate SwiftPM updates. (#6836) Summary: Let's make the versioning daily and create a new dedicated branch with swiftpm packages for it, so that the clients can pin themselves to a specific version and never run into checksum mismatch issues when we overwrite the binaries on S3. Reviewed By: GregoryComer Differential Revision: D65908021 --- .github/workflows/apple.yml | 77 ++++++++++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 10 deletions(-) diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index 16b58669c8e..156faf173f3 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -20,12 +20,26 @@ on: - extension/benchmark/apple/** - extension/module/** workflow_dispatch: + schedule: + - cron: '0 10 * * *' # Runs daily at 2 AM PST concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} cancel-in-progress: true jobs: + set-version: + runs-on: ubuntu-22.04 + outputs: + version: ${{ steps.set_version.outputs.version }} + steps: + - name: Set VERSION variable + id: set_version + shell: bash + run: | + VERSION="0.4.0.$(TZ='PST8PDT' date +%Y%m%d)" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + build-demo-ios: name: build-demo-ios uses: pytorch/test-infra/.github/workflows/macos_job.yml@main @@ -39,6 +53,8 @@ jobs: secrets-env: BUILD_CERTIFICATE_BASE64 EXECUTORCH_DEMO_BUILD_PROVISION_PROFILE_BASE64 KEYCHAIN_PASSWORD upload-artifact: ios-apps script: | + set -eux + BUILD_TOOL=cmake .ci/scripts/setup-conda.sh @@ -57,7 +73,7 @@ jobs: # Build and test iOS Demo App PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \ - build/test_ios_ci.sh ${ARTIFACTS_DIR_NAME} + build/test_ios_ci.sh "${ARTIFACTS_DIR_NAME}" # Upload the test demo app to S3 upload-demo-ios: @@ -75,6 +91,7 @@ jobs: shell: bash working-directory: ${{ runner.temp }}/artifacts/ run: | + set -eux ls -lah ./ - name: Upload the artifacts to S3 @@ -112,6 +129,7 @@ jobs: build-frameworks-ios: name: build-frameworks-ios + needs: set-version uses: pytorch/test-infra/.github/workflows/macos_job.yml@main with: runner: macos-latest-xlarge @@ -121,8 +139,10 @@ jobs: upload-artifact: executorch-frameworks-ios timeout: 90 script: | + set -eux + BUILD_TOOL=cmake - VERSION="latest" + VERSION="${{ needs.set-version.outputs.version }}" FRAMEWORKS=( "executorch" "backend_coreml" @@ -171,13 +191,17 @@ jobs: upload-frameworks-ios: runs-on: ubuntu-22.04 - needs: build-frameworks-ios + needs: [build-frameworks-ios, set-version] timeout-minutes: 30 permissions: id-token: write - contents: read + contents: write steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + persist-credentials: true - uses: actions/setup-python@v4 with: python-version: '3.11' @@ -194,15 +218,15 @@ jobs: name: executorch-frameworks-ios path: ${{ runner.temp }}/frameworks-ios/ - name: Only push to S3 when running the workflow manually from main branch - if: ${{ github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' }} + if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main' }} shell: bash run: | - set -eux echo "UPLOAD_ON_MAIN=1" >> "${GITHUB_ENV}" - name: Upload the artifact to ossci-ios S3 bucket shell: bash run: | set -eux + VERSION="${{ needs.set-version.outputs.version }}" pip install awscli==1.32.18 @@ -212,10 +236,43 @@ jobs: fi for FILENAME in "${RUNNER_TEMP}"/frameworks-ios/*.zip; do - [ -e "${FILENAME}" ] || continue - shasum -a 256 "${FILENAME}" - ${AWS_CMD} "${FILENAME}" s3://ossci-ios/executorch/ --acl public-read + FRAMEWORK_NAME=$(basename "${FILENAME}" | sed "s/-${VERSION}.zip//") + CHECKSUM=$(shasum -a 256 "${FILENAME}" | cut -d ' ' -f1) + echo "${FRAMEWORK_NAME} ${CHECKSUM}" >> "${RUNNER_TEMP}/checksums.txt" done + - name: Update SwiftPM + shell: bash + run: | + set -eux + VERSION="${{ needs.set-version.outputs.version }}" + BRANCH="swiftpm-${VERSION}" + + git checkout swiftpm + + if git show-ref --verify --quiet refs/heads/${BRANCH}; then + git checkout "${BRANCH}" + else + git checkout -b "${BRANCH}" + fi + + [[ -f Package.swift ]] || mv Package.swift.template Package.swift + + sed -i "s/__VERSION__/${VERSION}/g" Package.swift + + while read -r FRAMEWORK CHECKSUM; do + sed -i "s/__SHA256_${FRAMEWORK}__/${CHECKSUM}/g" Package.swift + done < "${RUNNER_TEMP}/checksums.txt" + + if [[ "${UPLOAD_ON_MAIN:-0}" == "1" ]]; then + git config --global user.name "PyTorch Bot" + git config --global user.email "pytorchbot@users.noreply.github.com" + git add Package.swift + git commit -m "${VERSION}" + git push origin "${BRANCH}" + else + echo "Draft Package.swift:" + cat Package.swift + fi build-benchmark-app: name: build-benchmark-app @@ -281,5 +338,5 @@ jobs: echo "::group::Build ExecuTorch benchmark app" mkdir -p extension/benchmark/apple/Benchmark/Models ${CONDA_RUN} --no-capture-output \ - build/build_apple_llm_demo.sh ${ARTIFACTS_DIR_NAME} + build/build_apple_llm_demo.sh "${ARTIFACTS_DIR_NAME}" echo "::endgroup::" From 5619e498d16c59a48fa553a1c5a01c90100d5e15 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 14 Nov 2024 12:02:30 -0800 Subject: [PATCH 2/5] Debug git push --- .github/workflows/apple.yml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index 156faf173f3..bbe896576ca 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -193,6 +193,10 @@ jobs: runs-on: ubuntu-22.04 needs: [build-frameworks-ios, set-version] timeout-minutes: 30 + # UNCOMMENT THIS LINE BEFORE LANDING + # envionrment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/nightly') && 'cherry-pick-bot' || '' }} + # FOR TESTING + environment: cherry-pick-bot permissions: id-token: write contents: write @@ -200,17 +204,16 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - persist-credentials: true + token: ${{ secrets.GH_PYTORCHBOT_CHERRY_PICK_TOKEN }} - uses: actions/setup-python@v4 with: python-version: '3.11' cache: pip - - name: configure aws credentials - uses: aws-actions/configure-aws-credentials@v1.7.0 - with: - role-to-assume: arn:aws:iam::308535385114:role/gha_executorch_upload-frameworks-ios - aws-region: us-east-1 + #- name: configure aws credentials + # uses: aws-actions/configure-aws-credentials@v1.7.0 + # with: + # role-to-assume: arn:aws:iam::308535385114:role/gha_executorch_upload-frameworks-ios + # aws-region: us-east-1 - name: Download the artifact uses: actions/download-artifact@v3 with: @@ -263,16 +266,16 @@ jobs: sed -i "s/__SHA256_${FRAMEWORK}__/${CHECKSUM}/g" Package.swift done < "${RUNNER_TEMP}/checksums.txt" - if [[ "${UPLOAD_ON_MAIN:-0}" == "1" ]]; then + #if [[ "${UPLOAD_ON_MAIN:-0}" == "1" ]]; then git config --global user.name "PyTorch Bot" git config --global user.email "pytorchbot@users.noreply.github.com" git add Package.swift git commit -m "${VERSION}" git push origin "${BRANCH}" - else - echo "Draft Package.swift:" - cat Package.swift - fi + #else + # echo "Draft Package.swift:" + # cat Package.swift + #fi build-benchmark-app: name: build-benchmark-app From c3db2a4a2a05644e07ec6f958865bda1d3fd9379 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 14 Nov 2024 12:36:27 -0800 Subject: [PATCH 3/5] Debug --- .github/workflows/apple.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index bbe896576ca..186e344a59d 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -191,7 +191,7 @@ jobs: upload-frameworks-ios: runs-on: ubuntu-22.04 - needs: [build-frameworks-ios, set-version] + # needs: [build-frameworks-ios, set-version] timeout-minutes: 30 # UNCOMMENT THIS LINE BEFORE LANDING # envionrment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/nightly') && 'cherry-pick-bot' || '' }} @@ -226,6 +226,8 @@ jobs: run: | echo "UPLOAD_ON_MAIN=1" >> "${GITHUB_ENV}" - name: Upload the artifact to ossci-ios S3 bucket + # DEBUG + if: false shell: bash run: | set -eux From 12a33f61940f1217be61c015ab5c187d03b1750c Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 14 Nov 2024 12:43:54 -0800 Subject: [PATCH 4/5] Ready to land --- .github/workflows/apple.yml | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index 186e344a59d..6f846a83467 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -191,12 +191,9 @@ jobs: upload-frameworks-ios: runs-on: ubuntu-22.04 - # needs: [build-frameworks-ios, set-version] + needs: [build-frameworks-ios, set-version] timeout-minutes: 30 - # UNCOMMENT THIS LINE BEFORE LANDING - # envionrment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/nightly') && 'cherry-pick-bot' || '' }} - # FOR TESTING - environment: cherry-pick-bot + environment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/nightly') && 'cherry-pick-bot' || '' }} permissions: id-token: write contents: write @@ -204,16 +201,16 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - token: ${{ secrets.GH_PYTORCHBOT_CHERRY_PICK_TOKEN }} + token: ${{ secrets.GH_PYTORCHBOT_CHERRY_PICK_TOKEN || secrets.GITHUB_TOKEN }} - uses: actions/setup-python@v4 with: python-version: '3.11' cache: pip - #- name: configure aws credentials - # uses: aws-actions/configure-aws-credentials@v1.7.0 - # with: - # role-to-assume: arn:aws:iam::308535385114:role/gha_executorch_upload-frameworks-ios - # aws-region: us-east-1 + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_executorch_upload-frameworks-ios + aws-region: us-east-1 - name: Download the artifact uses: actions/download-artifact@v3 with: @@ -226,8 +223,6 @@ jobs: run: | echo "UPLOAD_ON_MAIN=1" >> "${GITHUB_ENV}" - name: Upload the artifact to ossci-ios S3 bucket - # DEBUG - if: false shell: bash run: | set -eux @@ -268,16 +263,16 @@ jobs: sed -i "s/__SHA256_${FRAMEWORK}__/${CHECKSUM}/g" Package.swift done < "${RUNNER_TEMP}/checksums.txt" - #if [[ "${UPLOAD_ON_MAIN:-0}" == "1" ]]; then + if [[ "${UPLOAD_ON_MAIN:-0}" == "1" ]]; then git config --global user.name "PyTorch Bot" git config --global user.email "pytorchbot@users.noreply.github.com" git add Package.swift git commit -m "${VERSION}" git push origin "${BRANCH}" - #else - # echo "Draft Package.swift:" - # cat Package.swift - #fi + else + echo "Draft Package.swift:" + cat Package.swift + fi build-benchmark-app: name: build-benchmark-app From 9bc879aab5889513c31c42e59f58ebd23dc6a183 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 14 Nov 2024 12:45:28 -0800 Subject: [PATCH 5/5] Minor tweak --- .github/workflows/apple.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index 6f846a83467..44016c7abf8 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -193,7 +193,7 @@ jobs: runs-on: ubuntu-22.04 needs: [build-frameworks-ios, set-version] timeout-minutes: 30 - environment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/nightly') && 'cherry-pick-bot' || '' }} + environment: ${{ github.ref == 'refs/heads/main' && 'cherry-pick-bot' || '' }} permissions: id-token: write contents: write