diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000000..8878eea657 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,22 @@ +[//]: # "Copyright Amazon.com Inc. or its affiliates. All Rights Reserved." +[//]: # "SPDX-License-Identifier: CC-BY-SA-4.0" + +# GitHub Actions + +The MPL uses GitHub Actions to trigger our testing. +If a GitHub Action job/workflow is going to be added, and that job should always run, it should be added to `ci.yml`. + +This stops us from having multiple copies of the same event. + +Instead, we build a hierarchy of testing: + +- `routine_ci` determines regular testing events, such as daily, PR, or push to main +- `ci.yml` is triggered by `routine_ci` or other events, and runs all the per runtime testing +- `library__tests` are responsible for a runtimes particular testing suite. + +# `library__tests` + +In general, these should have at least two inputs: + +- The Dafny Verision +- Weather Smithy-Dafny should be re-run diff --git a/.github/workflows/check_dafny_runtime_versions.yml b/.github/workflows/check_dafny_runtime_versions.yml index 1ab75e7f41..681ca9933d 100644 --- a/.github/workflows/check_dafny_runtime_versions.yml +++ b/.github/workflows/check_dafny_runtime_versions.yml @@ -5,10 +5,8 @@ name: Check DafnyRuntimePython Version Consistency on: - pull_request: - push: - branches: - - main + workflow_call: + workflow_dispatch: jobs: check-version-consistency: diff --git a/.github/workflows/check-files.yml b/.github/workflows/check_files.yml similarity index 98% rename from .github/workflows/check-files.yml rename to .github/workflows/check_files.yml index fdf3b53a31..4ad9b5d3ac 100644 --- a/.github/workflows/check-files.yml +++ b/.github/workflows/check_files.yml @@ -3,7 +3,8 @@ name: Check Release Files on: - pull_request: + workflow_call: + workflow_dispatch: jobs: require-approvals: @@ -14,7 +15,7 @@ jobs: env: # unfortunately we can't check if the approver is part of the CODEOWNERS. This is a subset of aws/aws-crypto-tools-team # to add more allowlisted approvers just modify this env variable - maintainers: seebees, texastony, ShubhamChaturvedi7, lucasmcdonald3, josecorella, imabhichow, rishav-karanjit, antonf-amzn, kessplas, RitvikKapila, ajewellamz + maintainers: seebees, texastony, ShubhamChaturvedi7, lucasmcdonald3, josecorella, imabhichow, rishav-karanjit, antonf-amzn, kessplas, ajewellamz steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/check_only_keyword.yml b/.github/workflows/check_only_keyword.yml index 4a8a5c9f3f..8849b67ef1 100644 --- a/.github/workflows/check_only_keyword.yml +++ b/.github/workflows/check_only_keyword.yml @@ -4,7 +4,8 @@ name: Check {:only} decorator presence on: - pull_request: + workflow_call: + workflow_dispatch: jobs: grep-only-verification-keyword: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..cd732f3349 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,119 @@ +# These are all the tests needed in general +name: Continious Integration Tests + +on: + # This workflow is invoked by other workflows with the requested Dafny build. + # It represents all the required testing for the MPL. + workflow_dispatch: + inputs: + dafnyTranspilerVersion: + description: "The Dafny Transpiler and Runtime to use for all but Rust" + required: true + type: string + dafnyVerifyVersion: + description: "The Dafny Verifier Version to use" + required: true + type: string + dafnyRustVersion: + description: "The Dafny Transpiler and Runtime to use for Rust" + required: true + type: string + regenerate-code: + description: "Regenerate code using smithy-dafny" + required: false + default: false + type: boolean + workflow_call: + inputs: + dafnyTranspilerVersion: + description: "The Dafny Transpiler and Runtime to use for all but Rust" + required: true + type: string + dafnyVerifyVersion: + description: "The Dafny Verifier Version to use" + required: true + type: string + dafnyRustVersion: + description: "The Dafny Transpiler and Runtime to use for Rust" + required: true + type: string + regenerate-code: + description: "Regenerate code using smithy-dafny" + required: false + default: false + type: boolean + +jobs: + sem-ver: + uses: ./.github/workflows/sem_ver.yml + check-files: + uses: ./.github/workflows/check_files.yml + check-only: + uses: ./.github/workflows/check_only_keyword.yml + ci-not-grep: + uses: ./.github/workflows/ci_static_analysis.yml + ci-format: + uses: ./.github/workflows/library_format.yml + with: + dafny: ${{ inputs.dafnyVerifyVersion }} + regenerate-code: ${{ inputs.regenerate-code }} + ci-codegen: + uses: ./.github/workflows/library_codegen.yml + with: + dafny: ${{ inputs.dafnyVerifyVersion }} + ci-verification: + uses: ./.github/workflows/library_dafny_verification.yml + with: + dafny: ${{ inputs.dafnyVerifyVersion }} + regenerate-code: ${{ inputs.regenerate-code }} + ci-java: + uses: ./.github/workflows/library_java_tests.yml + with: + dafny: ${{ inputs.dafnyVerifyVersion }} + regenerate-code: ${{ inputs.regenerate-code }} + ci-net: + uses: ./.github/workflows/library_net_tests.yml + with: + dafny: ${{ inputs.dafnyVerifyVersion }} + regenerate-code: ${{ inputs.regenerate-code }} + ci-rust: + uses: ./.github/workflows/library_rust_tests.yml + with: + dafny: ${{ inputs.dafnyRustVersion }} + # Rust always regenerates code + ci-python-check-version-consistency: + uses: ./.github/workflows/check_dafny_runtime_versions.yml + ci-python: + uses: ./.github/workflows/library_python_tests.yml + with: + dafny: ${{ inputs.dafnyVerifyVersion }} + regenerate-code: ${{ inputs.regenerate-code }} + ci-go: + uses: ./.github/workflows/library_go_tests.yml + with: + dafny: ${{ inputs.dafnyVerifyVersion }} + regenerate-code: ${{ inputs.regenerate-code }} + interop-test: + uses: ./.github/workflows/library_interop_tests.yml + with: + dafny: ${{ inputs.dafnyVerifyVersion }} + regenerate-code: ${{ inputs.regenerate-code }} + secrets: inherit + all-required: + if: always() + needs: + - ci-format + - ci-codegen + - ci-verification + - ci-java + - ci-net + - ci-python + - ci-go + - ci-rust + - interop-test + runs-on: ubuntu-22.04 + steps: + - name: Verify all required jobs passed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static_analysis.yml similarity index 81% rename from .github/workflows/ci_static-analysis.yaml rename to .github/workflows/ci_static_analysis.yml index 374c2e4383..7f8eb86cc3 100644 --- a/.github/workflows/ci_static-analysis.yaml +++ b/.github/workflows/ci_static_analysis.yml @@ -2,10 +2,8 @@ name: static analysis on: - pull_request: - push: - branches: - - main + workflow_call: + workflow_dispatch: jobs: not-grep: diff --git a/.github/workflows/dafny_versions.yaml b/.github/workflows/dafny_versions.yml similarity index 100% rename from .github/workflows/dafny_versions.yaml rename to .github/workflows/dafny_versions.yml diff --git a/.github/workflows/daily_ci.yml b/.github/workflows/daily_ci.yml deleted file mode 100644 index 5336818a24..0000000000 --- a/.github/workflows/daily_ci.yml +++ /dev/null @@ -1,66 +0,0 @@ -# This workflow runs every weekday at 15:00 UTC (8AM PDT) -name: Daily CI - -on: - schedule: - - cron: "00 15 * * 1-5" - -jobs: - getVersions: - # Don't run the cron builds on forks - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/dafny_versions.yaml - daily-ci-format: - needs: getVersion - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/library_format.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - daily-ci-codegen: - needs: getVersion - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/library_codegen.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - daily-ci-verification: - needs: getVerifyVersion - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/library_dafny_verification.yml - with: - dafny: ${{needs.getVersions.outputs.verifyVersion}} - daily-ci-java: - needs: getVersion - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/library_java_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - daily-ci-net: - needs: getVersion - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/library_net_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - daily-ci-rust: - needs: getVersion - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/library_rust_tests.yml - with: - dafny: ${{needs.getVersions.outputs.rustVersion}} - daily-ci-python: - needs: getVersion - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/library_python_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - daily-ci-go: - needs: getVersion - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/library_go_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - daily-interop-test: - needs: getVersion - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/library_interop_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} diff --git a/.github/workflows/library_codegen.yml b/.github/workflows/library_codegen.yml index b72322f9a2..6ad91d051d 100644 --- a/.github/workflows/library_codegen.yml +++ b/.github/workflows/library_codegen.yml @@ -78,6 +78,6 @@ jobs: - uses: ./.github/actions/polymorph_codegen with: - dafny: ${{ env.DAFNY_VERSION }} + dafny: ${{ inputs.dafny }} library: ${{ matrix.library }} diff-generated-code: true diff --git a/.github/workflows/library_dafny_verification.yml b/.github/workflows/library_dafny_verification.yml index b6edca15e8..ae66f0d637 100644 --- a/.github/workflows/library_dafny_verification.yml +++ b/.github/workflows/library_dafny_verification.yml @@ -59,7 +59,7 @@ jobs: if: ${{ inputs.regenerate-code }} uses: ./.github/actions/polymorph_codegen with: - dafny: ${{ env.DAFNY_VERSION }} + dafny: ${{ inputs.dafny }} library: ${{ matrix.library }} diff-generated-code: false diff --git a/.github/workflows/library_format.yml b/.github/workflows/library_format.yml index 786d2baac8..5663e69467 100644 --- a/.github/workflows/library_format.yml +++ b/.github/workflows/library_format.yml @@ -16,8 +16,6 @@ on: jobs: format_projects: - # Don't run the nightly build on forks - if: github.event_name != 'schedule' || github.repository_owner == 'aws' strategy: matrix: library: @@ -81,11 +79,9 @@ jobs: # This works because `node` is installed by default on GHA runners CORES=$(node -e 'console.log(os.cpus().length)') make format_net-check + format_java_misc: - # Don't run the nightly build on forks - if: github.event_name != 'schedule' || github.repository_owner == 'aws' runs-on: ubuntu-22.04 - steps: - name: Support longpaths run: | diff --git a/.github/workflows/library_go_tests.yml b/.github/workflows/library_go_tests.yml index 4ed1ccbd04..9e5aa68a66 100644 --- a/.github/workflows/library_go_tests.yml +++ b/.github/workflows/library_go_tests.yml @@ -80,6 +80,14 @@ jobs: - name: Install Smithy-Dafny codegen dependencies uses: ./.github/actions/install_smithy_dafny_codegen_dependencies + - name: Regenerate code using smithy-dafny if necessary + if: ${{ inputs.regenerate-code }} + uses: ./.github/actions/polymorph_codegen + with: + dafny: ${{ inputs.dafny }} + library: ${{ matrix.library }} + diff-generated-code: false + - name: Build ${{ matrix.library }} implementation working-directory: ./${{ matrix.library }} run: | diff --git a/.github/workflows/library_java_tests.yml b/.github/workflows/library_java_tests.yml index ccf1331ded..dec3ffade0 100644 --- a/.github/workflows/library_java_tests.yml +++ b/.github/workflows/library_java_tests.yml @@ -72,7 +72,7 @@ jobs: if: ${{ inputs.regenerate-code }} uses: ./.github/actions/polymorph_codegen with: - dafny: ${{ env.DAFNY_VERSION }} + dafny: ${{ inputs.dafny }} library: ${{ matrix.library }} diff-generated-code: false diff --git a/.github/workflows/library_net_tests.yml b/.github/workflows/library_net_tests.yml index 3d4332970c..22cd8b270f 100644 --- a/.github/workflows/library_net_tests.yml +++ b/.github/workflows/library_net_tests.yml @@ -75,7 +75,7 @@ jobs: if: ${{ inputs.regenerate-code }} uses: ./.github/actions/polymorph_codegen with: - dafny: ${{ env.DAFNY_VERSION }} + dafny: ${{ inputs.dafny }} library: ${{ matrix.library }} diff-generated-code: false diff --git a/.github/workflows/library_python_tests.yml b/.github/workflows/library_python_tests.yml index f21864402e..acc051d09d 100644 --- a/.github/workflows/library_python_tests.yml +++ b/.github/workflows/library_python_tests.yml @@ -81,6 +81,14 @@ jobs: - name: Install Smithy-Dafny codegen dependencies uses: ./.github/actions/install_smithy_dafny_codegen_dependencies + - name: Regenerate code using smithy-dafny if necessary + if: ${{ inputs.regenerate-code }} + uses: ./.github/actions/polymorph_codegen + with: + dafny: ${{ inputs.dafny }} + library: ${{ matrix.library }} + diff-generated-code: false + - name: Build ${{ matrix.library }} implementation working-directory: ./${{ matrix.library }} run: | diff --git a/.github/workflows/library_rust_tests.yml b/.github/workflows/library_rust_tests.yml index 27aa509cb4..b880e80623 100644 --- a/.github/workflows/library_rust_tests.yml +++ b/.github/workflows/library_rust_tests.yml @@ -8,11 +8,6 @@ on: description: "The Dafny version to run" required: true type: string - regenerate-code: - description: "Regenerate code using smithy-dafny" - required: false - default: false - type: boolean jobs: testRust: diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml deleted file mode 100644 index c22f4a8003..0000000000 --- a/.github/workflows/manual.yml +++ /dev/null @@ -1,59 +0,0 @@ -# This workflow invokes other workflows with the requested Dafny build. -# It is primarily meant for manual compatibility testing, -# such as trying out what the next pending nightly build will do ahead of time. -name: Manual CI - -on: - workflow_dispatch: - inputs: - dafny: - description: "The Dafny version to use" - required: true - type: string - regenerate-code: - description: "Regenerate code using smithy-dafny" - required: false - default: true - type: boolean - -jobs: - manual-ci-format: - uses: ./.github/workflows/library_format.yml - with: - dafny: ${{ inputs.dafny }} - regenerate-code: ${{ inputs.regenerate-code }} - manual-ci-verification: - uses: ./.github/workflows/library_dafny_verification.yml - with: - dafny: ${{ inputs.dafny }} - regenerate-code: ${{ inputs.regenerate-code }} - manual-ci-java: - uses: ./.github/workflows/library_java_tests.yml - with: - dafny: ${{ inputs.dafny }} - regenerate-code: ${{ inputs.regenerate-code }} - manual-ci-net: - uses: ./.github/workflows/library_net_tests.yml - with: - dafny: ${{ inputs.dafny }} - regenerate-code: ${{ inputs.regenerate-code }} - manual-ci-rust: - uses: ./.github/workflows/library_rust_tests.yml - with: - dafny: ${{ inputs.dafny }} - regenerate-code: ${{ inputs.regenerate-code }} - manual-ci-python: - uses: ./.github/workflows/library_python_tests.yml - with: - dafny: ${{ inputs.dafny }} - regenerate-code: ${{ inputs.regenerate-code }} - manual-ci-go: - uses: ./.github/workflows/library_go_tests.yml - with: - dafny: ${{ inputs.dafny }} - regenerate-code: ${{ inputs.regenerate-code }} - manual-interop-test: - uses: ./.github/workflows/library_interop_tests.yml - with: - dafny: ${{ inputs.dafny }} - regenerate-code: ${{ inputs.regenerate-code }} diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml deleted file mode 100644 index e7bfae4274..0000000000 --- a/.github/workflows/pull.yml +++ /dev/null @@ -1,74 +0,0 @@ -# This workflow runs for every pull request -name: PR CI - -on: - pull_request: - -jobs: - getVersions: - uses: ./.github/workflows/dafny_versions.yaml - pr-ci-format: - needs: getVersions - uses: ./.github/workflows/library_format.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - pr-ci-codegen: - needs: getVersions - uses: ./.github/workflows/library_codegen.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - pr-ci-verification: - needs: getVersions - uses: ./.github/workflows/library_dafny_verification.yml - with: - dafny: ${{needs.getVersions.outputs.verifyVersion}} - pr-ci-java: - needs: getVersions - uses: ./.github/workflows/library_java_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - pr-ci-net: - needs: getVersions - uses: ./.github/workflows/library_net_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - pr-ci-rust: - needs: getVersions - uses: ./.github/workflows/library_rust_tests.yml - with: - dafny: ${{needs.getVersions.outputs.rustVersion}} - pr-ci-python: - needs: getVersions - uses: ./.github/workflows/library_python_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - pr-ci-go: - needs: getVersions - uses: ./.github/workflows/library_go_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - pr-interop-test: - needs: getVersions - uses: ./.github/workflows/library_interop_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - secrets: inherit - pr-ci-all-required: - if: always() - needs: - - getVersions - - pr-ci-format - - pr-ci-codegen - - pr-ci-verification - - pr-ci-java - - pr-ci-net - - pr-ci-python - - pr-ci-go - - pr-ci-rust - - pr-interop-test - runs-on: ubuntu-22.04 - steps: - - name: Verify all required jobs passed - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml deleted file mode 100644 index 336337f777..0000000000 --- a/.github/workflows/push.yml +++ /dev/null @@ -1,56 +0,0 @@ -# This workflow runs for every push to main -name: Push CI - -on: - push: - branches: - - main - -jobs: - getVersions: - uses: ./.github/workflows/dafny_versions.yaml - push-ci-format: - needs: getVersions - uses: ./.github/workflows/library_format.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - push-ci-codegen: - needs: getVersions - uses: ./.github/workflows/library_codegen.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - push-ci-verification: - needs: getVersions - uses: ./.github/workflows/library_dafny_verification.yml - with: - dafny: ${{needs.getVersions.outputs.verifyVersion}} - push-ci-java: - needs: getVersions - uses: ./.github/workflows/library_java_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - push-ci-net: - needs: getVersions - uses: ./.github/workflows/library_net_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - push-ci-rust: - needs: getVersions - uses: ./.github/workflows/library_rust_tests.yml - with: - dafny: ${{needs.getVersions.outputs.rustVersion}} - push-ci-python: - needs: getVersions - uses: ./.github/workflows/library_python_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - push-ci-go: - needs: getVersions - uses: ./.github/workflows/library_go_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} - pr-interop-test: - needs: getVersions - uses: ./.github/workflows/library_interop_tests.yml - with: - dafny: ${{needs.getVersions.outputs.version}} diff --git a/.github/workflows/routine_ci.yml b/.github/workflows/routine_ci.yml new file mode 100644 index 0000000000..f896c92c7f --- /dev/null +++ b/.github/workflows/routine_ci.yml @@ -0,0 +1,27 @@ +# These are all the tests needed in general +name: Routine CI + +on: + # This workflow runs every weekday at 15:00 UTC (8AM PDT) + schedule: + - cron: "00 15 * * 1-5" + # This worflow runs on every PR + pull_request: + # This workflow runs for every push to main + push: + branches: + - main + +jobs: + getVersions: + uses: ./.github/workflows/dafny_versions.yml + ci: + needs: getVersions + uses: ./.github/workflows/ci.yml + # Don't run the daily CI on forks + if: github.event_name != 'schedule' || github.repository_owner == 'aws' + with: + dafnyTranspilerVersion: ${{needs.getVersions.outputs.version}} + dafnyVerifyVersion: ${{needs.getVersions.outputs.verifyVersion}} + dafnyRustVersion: ${{needs.getVersions.outputs.rustVersion}} + regenerate-code: false diff --git a/.github/workflows/sem_ver.yml b/.github/workflows/sem_ver.yml index 5a83c2ac1c..ace1137b09 100644 --- a/.github/workflows/sem_ver.yml +++ b/.github/workflows/sem_ver.yml @@ -2,7 +2,8 @@ name: Semantic Release Test Installation on: - pull_request: + workflow_call: + workflow_dispatch: jobs: semantic-release: