From 50f2c026806b0c88ae0126ca79ae34ed06c60532 Mon Sep 17 00:00:00 2001 From: Matt Hammerly Date: Thu, 13 Feb 2025 09:39:56 -0800 Subject: [PATCH 1/6] add general CI workflows --- .github/workflows/cache-cleanup.yml | 33 +++++++++++++++++++ .../workflows/enforce-license-compliance.yml | 14 ++++++++ 2 files changed, 47 insertions(+) create mode 100644 .github/workflows/cache-cleanup.yml create mode 100644 .github/workflows/enforce-license-compliance.yml diff --git a/.github/workflows/cache-cleanup.yml b/.github/workflows/cache-cleanup.yml new file mode 100644 index 0000000000..a8da8d01a7 --- /dev/null +++ b/.github/workflows/cache-cleanup.yml @@ -0,0 +1,33 @@ +name: cleanup caches by a branch +on: + pull_request: + types: + - closed + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Cleanup + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge" + + echo "Fetching list of cache key" + cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH -L 100 | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm + done + echo "Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/enforce-license-compliance.yml b/.github/workflows/enforce-license-compliance.yml new file mode 100644 index 0000000000..86be74100e --- /dev/null +++ b/.github/workflows/enforce-license-compliance.yml @@ -0,0 +1,14 @@ +name: Enforce License Compliance + +on: + pull_request: + branches: [main, master] + +jobs: + enforce-license-compliance: + runs-on: ubuntu-latest + steps: + - name: 'Enforce License Compliance' + uses: getsentry/action-enforce-license-compliance@57ba820387a1a9315a46115ee276b2968da51f3d # main + with: + fossa_api_key: ${{ secrets.FOSSA_API_KEY }} From f5fdf46c1c7b8c44084a5bc757079405c28d4295 Mon Sep 17 00:00:00 2001 From: Matt Hammerly Date: Thu, 20 Feb 2025 11:31:02 -0800 Subject: [PATCH 2/6] add worker CI --- .github/workflows/worker-ci.yml | 117 ++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 .github/workflows/worker-ci.yml diff --git a/.github/workflows/worker-ci.yml b/.github/workflows/worker-ci.yml new file mode 100644 index 0000000000..2d16e1b77b --- /dev/null +++ b/.github/workflows/worker-ci.yml @@ -0,0 +1,117 @@ +name: Worker CI + +on: + push: + tags: + - prod-* + branches: + - main + - staging + paths: + - 'apps/worker/**' + - 'apps/worker' + - 'libs/**' + pull_request: + paths: + - 'apps/worker/**' + - 'apps/worker' + - 'libs/**' + merge_group: + paths: + - 'apps/worker/**' + - 'apps/worker' + - 'libs/**' + +permissions: + contents: "read" + id-token: "write" + issues: "write" + pull-requests: "write" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + worker-lint: + name: Run Lint (Worker) + uses: codecov/gha-workflows/.github/workflows/lint.yml@v1.2.31 + with: + working_directory: apps/worker + + worker-mypy: + name: Patch typing (Worker) + uses: codecov/gha-workflows/.github/workflows/mypy.yml@v1.2.31 + with: + working_directory: apps/worker + + worker-build: + name: Build App (Worker) + uses: codecov/gha-workflows/.github/workflows/build-app.yml@v1.2.31 + secrets: inherit + with: + repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || 'codecov/self-hosted-worker' }} + working_directory: apps/worker + + # Runs create-commit, create-report + worker-codecovstartup: + name: Codecov Startup (Worker) + needs: worker-build + uses: codecov/gha-workflows/.github/workflows/codecov-startup.yml@v1.2.31 + secrets: inherit + with: + working_directory: apps/worker + + worker-test: + name: Test (Worker) + needs: [worker-build, worker-codecovstartup] + uses: codecov/gha-workflows/.github/workflows/run-tests.yml@v1.2.31 + secrets: inherit + with: + repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || 'codecov/self-hosted-worker' }} + working_directory: apps/worker + flag_prefix: worker + + worker-build-self-hosted: + name: Build Self Hosted (Worker) + needs: [worker-build, worker-test] + uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.31 + secrets: inherit + with: + repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || 'codecov/self-hosted-worker' }} + working_directory: apps/worker + + worker-staging: + name: Push Staging Image (Worker) + needs: [worker-build, worker-test] + if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/main' || github.event.ref == 'refs/heads/staging') && github.repository_owner == 'codecov' }} + uses: codecov/gha-workflows/.github/workflows/push-env.yml@v1.2.31 + secrets: inherit + with: + environment: staging + repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || 'codecov/self-hosted-worker' }} + working_directory: apps/worker + sentry_project: worker + + worker-production: + name: Push Production Image (Worker) + needs: [worker-build, worker-test] + if: ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/main' && github.repository_owner == 'codecov' }} + uses: codecov/gha-workflows/.github/workflows/push-env.yml@v1.2.31 + secrets: inherit + with: + environment: production + repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || 'codecov/self-hosted-worker' }} + working_directory: apps/worker + sentry_project: worker + + worker-self-hosted: + name: Push Self Hosted Image (Worker) + needs: [worker-build-self-hosted, worker-test] + secrets: inherit + if: ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/main' && github.repository_owner == 'codecov' }} + uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.31 + with: + push_rolling: true + repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || 'codecov/self-hosted-worker' }} + working_directory: apps/worker From 674abf5c1dbe3201157756d6535ae345d939ccb1 Mon Sep 17 00:00:00 2001 From: Matt Hammerly Date: Thu, 20 Feb 2025 11:31:12 -0800 Subject: [PATCH 3/6] add API CI --- .github/workflows/api-ci.yml | 118 +++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 .github/workflows/api-ci.yml diff --git a/.github/workflows/api-ci.yml b/.github/workflows/api-ci.yml new file mode 100644 index 0000000000..a9a239f1dc --- /dev/null +++ b/.github/workflows/api-ci.yml @@ -0,0 +1,118 @@ +name: API CI + +on: + push: + tags: + - prod-* + branches: + - main + - staging + paths: + - 'apps/codecov-api/**' + - 'apps/codecov-api' + - 'libs/**' + pull_request: + paths: + - 'apps/codecov-api/**' + - 'apps/codecov-api' + - 'libs/**' + merge_group: + paths: + - 'apps/codecov-api/**' + - 'apps/codecov-api' + - 'libs/**' + +permissions: + contents: "read" + id-token: "write" + issues: "write" + pull-requests: "write" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + api-lint: + name: Run Lint (API) + uses: codecov/gha-workflows/.github/workflows/lint.yml@v1.2.31 + with: + working_directory: apps/codecov-api + + api-mypy: + name: Patch typing (API) + uses: codecov/gha-workflows/.github/workflows/mypy.yml@v1.2.31 + with: + working_directory: apps/codecov-api + + api-build: + name: Build App (API) + uses: codecov/gha-workflows/.github/workflows/build-app.yml@v1.2.31 + secrets: inherit + with: + repo: ${{ vars.CODECOV_API_IMAGE_V2 || 'codecov/self-hosted-api' }} + working_directory: apps/codecov-api + + # Runs create-commit, create-report + api-codecovstartup: + name: Codecov Startup (API) + needs: api-build + uses: codecov/gha-workflows/.github/workflows/codecov-startup.yml@v1.2.31 + secrets: inherit + with: + working_directory: apps/codecov-api + + api-test: + name: Test (API) + needs: [api-build, api-codecovstartup] + uses: codecov/gha-workflows/.github/workflows/run-tests-split.yml@v1.2.31 + secrets: inherit + with: + run_integration: false + repo: ${{ vars.CODECOV_API_IMAGE_V2 || 'codecov/self-hosted-api' }} + working_directory: apps/codecov-api + flag_prefix: api + + api-build-self-hosted: + name: Build Self Hosted (API) + needs: [api-build, api-test] + uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.31 + secrets: inherit + with: + repo: ${{ vars.CODECOV_API_IMAGE_V2 || 'codecov/self-hosted-api' }} + working_directory: apps/codecov-api + + api-staging: + name: Push Staging Image (API) + needs: [api-build, api-test] + if: ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/staging' && github.repository_owner == 'codecov' }} + uses: codecov/gha-workflows/.github/workflows/push-env.yml@v1.2.31 + secrets: inherit + with: + environment: staging + repo: ${{ vars.CODECOV_API_IMAGE_V2 || 'codecov/self-hosted-api' }} + working_directory: apps/codecov-api + sentry_project: api + + api-production: + name: Push Production Image (API) + needs: [api-build, api-test] + if: ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/main' && github.repository_owner == 'codecov' }} + uses: codecov/gha-workflows/.github/workflows/push-env.yml@v1.2.31 + secrets: inherit + with: + environment: production + repo: ${{ vars.CODECOV_API_IMAGE_V2 || 'codecov/self-hosted-api' }} + working_directory: apps/codecov-api + sentry_project: api + + api-self-hosted: + name: Push Self Hosted Image (API) + needs: [api-build-self-hosted, api-test] + secrets: inherit + if: ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/main' && github.repository_owner == 'codecov' }} + uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.31 + with: + push_rolling: true + repo: ${{ vars.CODECOV_API_IMAGE_V2 || 'codecov/self-hosted-api' }} + working_directory: apps/codecov-api From 6f63b125d6ad5efd958170a8f8bcd9a9d6d51cd5 Mon Sep 17 00:00:00 2001 From: Matt Hammerly Date: Thu, 20 Feb 2025 11:31:21 -0800 Subject: [PATCH 4/6] add shared CI workflow --- .github/workflows/shared-ci.yml | 138 ++++++++++++++++++++++++++++++++ codecov.yml | 18 ++--- 2 files changed, 147 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/shared-ci.yml diff --git a/.github/workflows/shared-ci.yml b/.github/workflows/shared-ci.yml new file mode 100644 index 0000000000..f1a764e285 --- /dev/null +++ b/.github/workflows/shared-ci.yml @@ -0,0 +1,138 @@ +name: Shared CI + +on: + push: + branches: + - main + paths: + - 'libs/shared/**' + - 'libs/shared' + pull_request: + paths: + - 'libs/shared/**' + - 'libs/shared' + merge_group: + paths: + - 'libs/shared/**' + - 'libs/shared' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + # TODO: get mypy running + shared-lint: + name: Run Lint (Shared) + uses: codecov/gha-workflows/.github/workflows/lint.yml@v1.2.31 + with: + working_directory: libs/shared + + shared-codecovstartup: + name: Codecov Startup (Shared) + uses: codecov/gha-workflows/.github/workflows/codecov-startup.yml@v1.2.31 + secrets: inherit + with: + working_directory: libs/shared + + shared-test: + name: Test (Shared) + runs-on: ubuntu-latest + defaults: + run: + working-directory: libs/shared + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + + - name: Build test environment + run: | + make test_env.build + + - name: Bring containers up + run: | + make test_env.up + + - name: Run tests + run: | + make test_env.test + + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: coveragefiles + path: libs/shared/tests/*.coverage.xml + + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: junitfiles + path: libs/shared/tests/*junit*.xml + + shared-upload-to-codecov: + name: Upload to Codecov (shared) + needs: [shared-test] + runs-on: ubuntu-latest + strategy: + matrix: + include: + - codecov_url_secret: CODECOV_URL + codecov_token_secret: CODECOV_ORG_TOKEN + name: prod + - codecov_url_secret: CODECOV_STAGING_URL + codecov_token_secret: CODECOV_ORG_TOKEN_STAGING + name: staging + - codecov_url_secret: CODECOV_QA_URL + codecov_token_secret: CODECOV_QA_ORG + name: qa + - codecov_url_secret: CODECOV_PUBLIC_QA_URL + codecov_token_secret: CODECOV_PUBLIC_QA_TOKEN + name: public qa + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: 'recursive' + + - name: Download coverage + id: download_coverage + uses: actions/download-artifact@v4 + with: + name: coveragefiles + + - name: Download test results + id: download_test_results + uses: actions/download-artifact@v4 + with: + name: junitfiles + + - name: Uploading unit test coverage (${{ matrix.name }}) + uses: codecov/codecov-action@v5 + with: + files: ${{ steps.download_coverage.outputs.download-path }}/unit.coverage.xml + flags: shared-docker-uploader + disable_search: true + # Strange workaround: API has a `codecov` directory in the repo root + # which conflicts with the action's `codecov` binary + use_pypi: true + token: ${{ secrets[matrix.codecov_token_secret] }} + url: ${{ secrets[matrix.codecov_url_secret] }} + working-directory: libs/shared + + - name: Uploading unit test results (${{ matrix.name }}) + uses: codecov/test-results-action@v1 + with: + files: ${{ steps.download_test_results.outputs.download-path }}/unit.junit.xml + flags: shared-docker-uploader + disable_search: true + token: ${{ secrets[matrix.codecov_token_secret] }} + url: ${{ secrets[matrix.codecov_url_secret] }} + # The coverage action will have installed codecovcli with pip. The + # actual binary will be found in $PATH. + binary: codecovcli + working-directory: libs/shared + diff --git a/codecov.yml b/codecov.yml index f8d95eca50..0ab4dc7708 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,16 +1,16 @@ ignore: - "**tests**/test_*.py" -comment: - show_critical_paths: true - -beta_groups: - - "labels" flag_management: - individual_flags: - - name: "smart-labels" - carryforward: true - carryforward_mode: "labels" + default_rules: + carryforward: true + statuses: + - type: project + target: auto + threshold: 1 + - type: patch + target: 90 codecov: + wait_for_ci: false require_ci_to_pass: false From e87a1a40785b603b3909948e071cca7a859770e0 Mon Sep 17 00:00:00 2001 From: Matt Hammerly Date: Thu, 20 Feb 2025 11:32:27 -0800 Subject: [PATCH 5/6] add self-hosted release CI workflows --- .github/workflows/self-hosted-release-pr.yml | 14 +++++++ .github/workflows/self-hosted-release.yml | 40 ++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 .github/workflows/self-hosted-release-pr.yml create mode 100644 .github/workflows/self-hosted-release.yml diff --git a/.github/workflows/self-hosted-release-pr.yml b/.github/workflows/self-hosted-release-pr.yml new file mode 100644 index 0000000000..a01cb559fe --- /dev/null +++ b/.github/workflows/self-hosted-release-pr.yml @@ -0,0 +1,14 @@ +name: Create Self Hosted Release PR + +on: + workflow_dispatch: + inputs: + versionName: + description: "Name of version (ie 23.9.5)" + required: true + +jobs: + create-release-pr: + name: Create PR for Release ${{ github.event.inputs.versionName }} + uses: codecov/gha-workflows/.github/workflows/create-release-pr.yml@v1.2.21 + secrets: inherit diff --git a/.github/workflows/self-hosted-release.yml b/.github/workflows/self-hosted-release.yml new file mode 100644 index 0000000000..a4fbf1e446 --- /dev/null +++ b/.github/workflows/self-hosted-release.yml @@ -0,0 +1,40 @@ +iname: Create Self Hosted Release + +on: + pull_request: + branches: + - main + types: [closed] + +permissions: + contents: "read" + id-token: "write" + +jobs: + create-release: + name: Tag Release ${{ github.head_ref }} and Push Docker image to Docker Hub + if: ${{ github.event.pull_request.merged == true && startsWith(github.head_ref, 'release/') && github.repository_owner == 'codecov' }} + uses: codecov/gha-workflows/.github/workflows/create-release.yml@v1.2.21 + with: + tag_to_prepend: self-hosted- + secrets: inherit + + push-worker-image: + needs: [create-release] + if: ${{ github.event.pull_request.merged == true && startsWith(github.head_ref, 'release/') && github.repository_owner == 'codecov' }} + uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.31 + secrets: inherit + with: + push_release: true + repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || 'codecov/self-hosted-worker' }} + working_directory: apps/worker + + push-api-image: + needs: [create-release] + if: ${{ github.event.pull_request.merged == true && startsWith(github.head_ref, 'release/') && github.repository_owner == 'codecov' }} + uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.31 + secrets: inherit + with: + push_release: true + repo: ${{ vars.CODECOV_API_IMAGE_V2 || 'codecov/self-hosted-api' }} + working_directory: apps/codecov-api From 3df2dcd83483c594ba55b5e4602953b80be4b220 Mon Sep 17 00:00:00 2001 From: Matt Hammerly Date: Thu, 13 Feb 2025 09:50:00 -0800 Subject: [PATCH 6/6] move/update submodule checkouts to apps and libs dirs --- .gitmodules | 12 ++++++------ apps/codecov-api | 1 + apps/worker | 1 + codecov-api | 1 - libs/shared | 1 + shared | 1 - worker | 1 - 7 files changed, 9 insertions(+), 9 deletions(-) create mode 160000 apps/codecov-api create mode 160000 apps/worker delete mode 160000 codecov-api create mode 160000 libs/shared delete mode 160000 shared delete mode 160000 worker diff --git a/.gitmodules b/.gitmodules index e821c96b52..f64a466b33 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ -[submodule "worker"] - path = worker +[submodule "apps/worker"] + path = apps/worker url = git@github.com:codecov/worker.git -[submodule "codecov-api"] - path = codecov-api +[submodule "apps/codecov-api"] + path = apps/codecov-api url = git@github.com:codecov/codecov-api.git -[submodule "shared"] - path = shared +[submodule "libs/shared"] + path = libs/shared url = git@github.com:codecov/shared.git diff --git a/apps/codecov-api b/apps/codecov-api new file mode 160000 index 0000000000..cfb316b631 --- /dev/null +++ b/apps/codecov-api @@ -0,0 +1 @@ +Subproject commit cfb316b6310af71851cf66f61f7d3cafe1e538db diff --git a/apps/worker b/apps/worker new file mode 160000 index 0000000000..d7b53d82cc --- /dev/null +++ b/apps/worker @@ -0,0 +1 @@ +Subproject commit d7b53d82ccdafbc1349d8f2bfb6a17b838966552 diff --git a/codecov-api b/codecov-api deleted file mode 160000 index dc30d7fedd..0000000000 --- a/codecov-api +++ /dev/null @@ -1 +0,0 @@ -Subproject commit dc30d7fedd74793ae07720cf76a5b908fa30546a diff --git a/libs/shared b/libs/shared new file mode 160000 index 0000000000..12f2cec5d3 --- /dev/null +++ b/libs/shared @@ -0,0 +1 @@ +Subproject commit 12f2cec5d3e87d56f717ba3546650c7e05eafe07 diff --git a/shared b/shared deleted file mode 160000 index 76a3263079..0000000000 --- a/shared +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 76a3263079150e47059a3b08c7ef3a29e7e78cf4 diff --git a/worker b/worker deleted file mode 160000 index 201323d188..0000000000 --- a/worker +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 201323d18846108df8cc157ae1b664d98d86d7b6