From dd34d4784ad4770815d0643baaa1cc53db9af7c9 Mon Sep 17 00:00:00 2001 From: kgala2 Date: Mon, 31 Mar 2025 14:47:04 -0700 Subject: [PATCH 1/9] build: add schedule reporter and cb failure reporter files --- .../cloud_build_failure_reporter.yml | 181 ++++++++++++++++++ .github/workflows/schedule_reporter.yml | 29 +++ 2 files changed, 210 insertions(+) create mode 100644 .github/workflows/cloud_build_failure_reporter.yml create mode 100644 .github/workflows/schedule_reporter.yml diff --git a/.github/workflows/cloud_build_failure_reporter.yml b/.github/workflows/cloud_build_failure_reporter.yml new file mode 100644 index 000000000..2be30f0c3 --- /dev/null +++ b/.github/workflows/cloud_build_failure_reporter.yml @@ -0,0 +1,181 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Cloud Build Failure Reporter + +on: + workflow_call: + inputs: + trigger_names: + required: true + type: string + workflow_dispatch: + inputs: + trigger_names: + description: 'Cloud Build trigger names separated by comma.' + required: true + default: '' + +jobs: + report: + + permissions: + issues: 'write' + checks: 'read' + contents: 'read' + + runs-on: 'ubuntu-latest' + + steps: + - uses: 'actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea' # v7 + with: + script: |- + // parse test names + const testNameSubstring = '${{ inputs.trigger_names }}'; + const testNameFound = new Map(); //keeps track of whether each test is found + testNameSubstring.split(',').forEach(testName => { + testNameFound.set(testName, false); + }); + + // label for all issues opened by reporter + const periodicLabel = 'periodic-failure'; + + // check if any reporter opened any issues previously + const prevIssues = await github.paginate(github.rest.issues.listForRepo, { + ...context.repo, + state: 'open', + creator: 'github-actions[bot]', + labels: [periodicLabel] + }); + + // createOrCommentIssue creates a new issue or comments on an existing issue. + const createOrCommentIssue = async function (title, txt) { + if (prevIssues.length < 1) { + console.log('no previous issues found, creating one'); + await github.rest.issues.create({ + ...context.repo, + title: title, + body: txt, + labels: [periodicLabel] + }); + return; + } + // only comment on issue related to the current test + for (const prevIssue of prevIssues) { + if (prevIssue.title.includes(title)){ + console.log( + `found previous issue ${prevIssue.html_url}, adding comment` + ); + + await github.rest.issues.createComment({ + ...context.repo, + issue_number: prevIssue.number, + body: txt + }); + return; + } + } + }; + + // updateIssues comments on any existing issues. No-op if no issue exists. + const updateIssues = async function (checkName, txt) { + if (prevIssues.length < 1) { + console.log('no previous issues found.'); + return; + } + // only comment on issue related to the current test + for (const prevIssue of prevIssues) { + if (prevIssue.title.includes(checkName)){ + console.log(`found previous issue ${prevIssue.html_url}, adding comment`); + await github.rest.issues.createComment({ + ...context.repo, + issue_number: prevIssue.number, + body: txt + }); + } + } + }; + + // Find status of check runs. + // We will find check runs for each commit and then filter for the periodic. + // Checks API only allows for ref and if we use main there could be edge cases where + // the check run happened on a SHA that is different from head. + const commits = await github.paginate(github.rest.repos.listCommits, { + ...context.repo + }); + + const relevantChecks = new Map(); + for (const commit of commits) { + console.log( + `checking runs at ${commit.html_url}: ${commit.commit.message}` + ); + const checks = await github.rest.checks.listForRef({ + ...context.repo, + ref: commit.sha + }); + + // Iterate through each check and find matching names + for (const check of checks.data.check_runs) { + console.log(`Handling test name ${check.name}`); + for (const testName of testNameFound.keys()) { + if (testNameFound.get(testName) === true){ + //skip if a check is already found for this name + continue; + } + if (check.name.includes(testName)) { + relevantChecks.set(check, commit); + testNameFound.set(testName, true); + } + } + } + // Break out of the loop early if all tests are found + const allTestsFound = Array.from(testNameFound.values()).every(value => value === true); + if (allTestsFound){ + break; + } + } + + // Handle each relevant check + relevantChecks.forEach((commit, check) => { + if ( + check.status === 'completed' && + check.conclusion === 'success' + ) { + updateIssues( + check.name, + `[Tests are passing](${check.html_url}) for commit [${commit.sha}](${commit.html_url}).` + ); + } else if (check.status === 'in_progress') { + console.log( + `Check is pending ${check.html_url} for ${commit.html_url}. Retry again later.` + ); + } else { + createOrCommentIssue( + `Cloud Build Failure Reporter: ${check.name} failed`, + `Cloud Build Failure Reporter found test failure for [**${check.name}** ](${check.html_url}) at [${commit.sha}](${commit.html_url}). Please fix the error and then close the issue after the **${check.name}** test passes.` + ); + } + }); + + // no periodic checks found across all commits, report it + const noTestFound = Array.from(testNameFound.values()).every(value => value === false); + if (noTestFound){ + createOrCommentIssue( + 'Missing periodic tests: ${{ inputs.trigger_names }}', + `No periodic test is found for triggers: ${{ inputs.trigger_names }}. Last checked from ${ + commits[0].html_url + } to ${commits[commits.length - 1].html_url}.` + ); + } + \ No newline at end of file diff --git a/.github/workflows/schedule_reporter.yml b/.github/workflows/schedule_reporter.yml new file mode 100644 index 000000000..32a7cf056 --- /dev/null +++ b/.github/workflows/schedule_reporter.yml @@ -0,0 +1,29 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Schedule Reporter + +on: + schedule: + - cron: '0 6 * * *' # Runs at 6 AM every morning + +jobs: + run_reporter: + permissions: + issues: 'write' + checks: 'read' + contents: 'read' + uses: ./.github/workflows/cloud_build_failure_reporter.yml + with: + trigger_names: "pg-integration-test-nightly,pg-continuous-test-on-merge" \ No newline at end of file From 528ddb8332d65e13793ebcbe105f340d98f28622 Mon Sep 17 00:00:00 2001 From: kgala2 Date: Mon, 31 Mar 2025 20:16:08 -0700 Subject: [PATCH 2/9] chore: update triggers on schedule reporter --- .github/workflows/schedule_reporter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/schedule_reporter.yml b/.github/workflows/schedule_reporter.yml index 32a7cf056..b6aa1f451 100644 --- a/.github/workflows/schedule_reporter.yml +++ b/.github/workflows/schedule_reporter.yml @@ -26,4 +26,4 @@ jobs: contents: 'read' uses: ./.github/workflows/cloud_build_failure_reporter.yml with: - trigger_names: "pg-integration-test-nightly,pg-continuous-test-on-merge" \ No newline at end of file + trigger_names: "integration-test-pr-py39,integration-test-pr-py313" \ No newline at end of file From 868127a9511cd7911d05f86346aae161d3ee6482 Mon Sep 17 00:00:00 2001 From: kgala2 Date: Tue, 1 Apr 2025 11:46:01 -0700 Subject: [PATCH 3/9] chore: update triggers for push and nightly tests --- .github/workflows/schedule_reporter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/schedule_reporter.yml b/.github/workflows/schedule_reporter.yml index b6aa1f451..dbda0575b 100644 --- a/.github/workflows/schedule_reporter.yml +++ b/.github/workflows/schedule_reporter.yml @@ -26,4 +26,4 @@ jobs: contents: 'read' uses: ./.github/workflows/cloud_build_failure_reporter.yml with: - trigger_names: "integration-test-pr-py39,integration-test-pr-py313" \ No newline at end of file + trigger_names: "py-continuous-test-on-merge,py-integration-test-nightly" \ No newline at end of file From 751cf41fda79eabf3d1fd4308c46ba0c0229aaa7 Mon Sep 17 00:00:00 2001 From: kgala2 Date: Tue, 1 Apr 2025 12:46:51 -0700 Subject: [PATCH 4/9] chore: add new line at the end of the file --- .github/workflows/cloud_build_failure_reporter.yml | 1 - .github/workflows/schedule_reporter.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cloud_build_failure_reporter.yml b/.github/workflows/cloud_build_failure_reporter.yml index 2be30f0c3..a07e3a676 100644 --- a/.github/workflows/cloud_build_failure_reporter.yml +++ b/.github/workflows/cloud_build_failure_reporter.yml @@ -178,4 +178,3 @@ jobs: } to ${commits[commits.length - 1].html_url}.` ); } - \ No newline at end of file diff --git a/.github/workflows/schedule_reporter.yml b/.github/workflows/schedule_reporter.yml index dbda0575b..bad7e46c2 100644 --- a/.github/workflows/schedule_reporter.yml +++ b/.github/workflows/schedule_reporter.yml @@ -26,4 +26,4 @@ jobs: contents: 'read' uses: ./.github/workflows/cloud_build_failure_reporter.yml with: - trigger_names: "py-continuous-test-on-merge,py-integration-test-nightly" \ No newline at end of file + trigger_names: "py-continuous-test-on-merge,py-integration-test-nightly" From 4d228be6084bc66efaf9b8891f3615b2deef7e57 Mon Sep 17 00:00:00 2001 From: kgala2 Date: Wed, 2 Apr 2025 14:18:10 -0700 Subject: [PATCH 5/9] chore: test breaking tests for cloud build reporter --- .github/workflows/schedule_reporter.yml | 2 +- tests/system/test_pymysql_connection.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/schedule_reporter.yml b/.github/workflows/schedule_reporter.yml index bad7e46c2..953d0defb 100644 --- a/.github/workflows/schedule_reporter.yml +++ b/.github/workflows/schedule_reporter.yml @@ -16,7 +16,7 @@ name: Schedule Reporter on: schedule: - - cron: '0 6 * * *' # Runs at 6 AM every morning + - cron: '30 21 * * *' # Runs at 2:30 PM PT (9:30 PM UTC) every day jobs: run_reporter: diff --git a/tests/system/test_pymysql_connection.py b/tests/system/test_pymysql_connection.py index 2748a8d77..da42f88cd 100644 --- a/tests/system/test_pymysql_connection.py +++ b/tests/system/test_pymysql_connection.py @@ -114,6 +114,9 @@ def test_lazy_pymysql_connection() -> None: password = os.environ["MYSQL_PASS"] db = os.environ["MYSQL_DB"] ip_type = os.environ.get("IP_TYPE", "public") + # test breaking test reporting + if ip_type == "private": + user = os.environ["POSTGRES_USER"] engine, connector = create_sqlalchemy_engine( inst_conn_name, user, password, db, ip_type, "lazy" From fa11a2006305251375c74f03ce7917875278b734 Mon Sep 17 00:00:00 2001 From: kgala2 Date: Wed, 2 Apr 2025 14:41:51 -0700 Subject: [PATCH 6/9] chore: revert tests to stable version --- .github/workflows/schedule_reporter.yml | 2 +- tests/system/test_pymysql_connection.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/schedule_reporter.yml b/.github/workflows/schedule_reporter.yml index 953d0defb..34f795fbf 100644 --- a/.github/workflows/schedule_reporter.yml +++ b/.github/workflows/schedule_reporter.yml @@ -16,7 +16,7 @@ name: Schedule Reporter on: schedule: - - cron: '30 21 * * *' # Runs at 2:30 PM PT (9:30 PM UTC) every day + - cron: '0 22 * * *' # Runs at 3:00 PM PT (10:00 PM UTC) every day jobs: run_reporter: diff --git a/tests/system/test_pymysql_connection.py b/tests/system/test_pymysql_connection.py index da42f88cd..2748a8d77 100644 --- a/tests/system/test_pymysql_connection.py +++ b/tests/system/test_pymysql_connection.py @@ -114,9 +114,6 @@ def test_lazy_pymysql_connection() -> None: password = os.environ["MYSQL_PASS"] db = os.environ["MYSQL_DB"] ip_type = os.environ.get("IP_TYPE", "public") - # test breaking test reporting - if ip_type == "private": - user = os.environ["POSTGRES_USER"] engine, connector = create_sqlalchemy_engine( inst_conn_name, user, password, db, ip_type, "lazy" From fed070fdc9c663b8a78c0a4e8d6c583324705982 Mon Sep 17 00:00:00 2001 From: kgala2 Date: Wed, 2 Apr 2025 14:59:26 -0700 Subject: [PATCH 7/9] chore: update reporter time to 4 PM PT --- .github/workflows/schedule_reporter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/schedule_reporter.yml b/.github/workflows/schedule_reporter.yml index 34f795fbf..dccf357d3 100644 --- a/.github/workflows/schedule_reporter.yml +++ b/.github/workflows/schedule_reporter.yml @@ -16,7 +16,7 @@ name: Schedule Reporter on: schedule: - - cron: '0 22 * * *' # Runs at 3:00 PM PT (10:00 PM UTC) every day + - cron: '0 23 * * *' # Runs at 4:00 PM PT (11:00 PM UTC) every day jobs: run_reporter: From c2a557f5950702144c9243b78184bdeae9a12ce3 Mon Sep 17 00:00:00 2001 From: kgala2 Date: Wed, 2 Apr 2025 15:01:04 -0700 Subject: [PATCH 8/9] chore: update reporter time to 12AM --- .github/workflows/schedule_reporter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/schedule_reporter.yml b/.github/workflows/schedule_reporter.yml index dccf357d3..66c65c7cd 100644 --- a/.github/workflows/schedule_reporter.yml +++ b/.github/workflows/schedule_reporter.yml @@ -16,7 +16,7 @@ name: Schedule Reporter on: schedule: - - cron: '0 23 * * *' # Runs at 4:00 PM PT (11:00 PM UTC) every day + - cron: '0 3 * * *' # Runs at 12:00 AM PT everyday jobs: run_reporter: From 662e4b3ef595caf11bf86df4e873bff9d64a49db Mon Sep 17 00:00:00 2001 From: kgala2 Date: Wed, 2 Apr 2025 15:04:08 -0700 Subject: [PATCH 9/9] chore: update reporter schedule timings to 6 AM --- .github/workflows/schedule_reporter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/schedule_reporter.yml b/.github/workflows/schedule_reporter.yml index 66c65c7cd..bad7e46c2 100644 --- a/.github/workflows/schedule_reporter.yml +++ b/.github/workflows/schedule_reporter.yml @@ -16,7 +16,7 @@ name: Schedule Reporter on: schedule: - - cron: '0 3 * * *' # Runs at 12:00 AM PT everyday + - cron: '0 6 * * *' # Runs at 6 AM every morning jobs: run_reporter: