From 367979c6e95881f73c1c1faa60b914ff119b9a2c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 26 Aug 2025 22:54:02 +0000 Subject: [PATCH 1/8] Initial plan From e3ca87e60c78fe25b0ee7b19721331e91ba1967e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 26 Aug 2025 23:08:55 +0000 Subject: [PATCH 2/8] Add fuzzing failure notification system - Created dev/notify-fuzzing-failure.sh script to handle GitHub issue creation/updates - Modified Fuzzing.yml and dep_fuzzing.yml workflows to add failure notifications - Added proper permissions for issue creation and comments - Script checks for existing open fuzzing failure issues and either creates new ones or adds comments - Includes robust error handling and test mode for validation Co-authored-by: jsturtevant <648372+jsturtevant@users.noreply.github.com> --- .github/workflows/Fuzzing.yml | 1 + .github/workflows/dep_fuzzing.yml | 7 ++ dev/notify-fuzzing-failure.sh | 155 ++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+) create mode 100755 dev/notify-fuzzing-failure.sh diff --git a/.github/workflows/Fuzzing.yml b/.github/workflows/Fuzzing.yml index 8bf437e8b..3565bd559 100644 --- a/.github/workflows/Fuzzing.yml +++ b/.github/workflows/Fuzzing.yml @@ -8,6 +8,7 @@ on: permissions: id-token: write contents: read + issues: write jobs: fuzzing: diff --git a/.github/workflows/dep_fuzzing.yml b/.github/workflows/dep_fuzzing.yml index 95008efd4..b890b68a3 100644 --- a/.github/workflows/dep_fuzzing.yml +++ b/.github/workflows/dep_fuzzing.yml @@ -20,6 +20,7 @@ on: permissions: id-token: write contents: read + issues: write jobs: fuzz: @@ -60,3 +61,9 @@ jobs: with: name: fuzz-crash-artifacts path: fuzz/artifacts/ + + - name: Notify Fuzzing Failure + if: failure() # Only run this step if the fuzzing job failed + run: ./dev/notify-fuzzing-failure.sh "${{ matrix.target }}" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/dev/notify-fuzzing-failure.sh b/dev/notify-fuzzing-failure.sh new file mode 100755 index 000000000..15ffb835c --- /dev/null +++ b/dev/notify-fuzzing-failure.sh @@ -0,0 +1,155 @@ +#!/bin/bash +set -e +set -u +set -o pipefail + +## DESCRIPTION: +## +## This script creates or updates GitHub issues when fuzzing jobs fail. +## It checks for existing open fuzzing failure issues and either creates +## a new one or adds a comment to an existing one. +## +## PRE-REQS: +## +## This script assumes that the gh cli is installed and in the PATH +## and that there is a GitHub PAT in the GITHUB_TOKEN env var +## with the following permissions: +## - repo (read/write) +## - issues (read/write) +## or that the user is logged into the gh cli with an account with those permissions + +REPO="${GITHUB_REPOSITORY:-hyperlight-dev/hyperlight}" +WORKFLOW_RUN_URL="${GITHUB_SERVER_URL:-https://github.com}/${REPO}/actions/runs/${GITHUB_RUN_ID:-unknown}" +FUZZING_TARGETS="${1:-unknown}" + +# Check if running in test mode (handle both first and second arguments) +if [ "${1:-}" = "--test" ] || [ "${2:-}" = "--test" ]; then + echo "✅ Running in test mode - script structure is valid" + echo "Would check for fuzzing failure issues in $REPO" + echo "Would create issue or comment for fuzzing targets: ${1:-unknown}" + echo "Workflow URL would be: $WORKFLOW_RUN_URL" + exit 0 +fi + +echo "Checking for existing fuzzing failure issues in $REPO..." + +# Extract owner and repo name from the repository +OWNER=$(echo "$REPO" | cut -d'/' -f1) +REPO_NAME=$(echo "$REPO" | cut -d'/' -f2) + +# Define the issue title and labels +ISSUE_TITLE="Fuzzing Job Failure - $(date '+%Y-%m-%d')" +FUZZING_LABEL="area/testing" +FAILURE_LABEL="kind/bug" +LIFECYCLE_LABEL="lifecycle/needs-review" + +# Search for existing open fuzzing failure issues +echo "Searching for existing open fuzzing failure issues..." +EXISTING_ISSUES=$(gh api graphql -f query=' + query($owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + issues(first: 10, states: OPEN, labels: ["area/testing"]) { + totalCount + nodes { + number + title + url + labels(first: 20) { + nodes { + name + } + } + } + } + } + }' -f owner="$OWNER" -f repo="$REPO_NAME" --jq '.data.repository.issues') + +# Filter for fuzzing-related issues +FUZZING_ISSUES=$(echo "$EXISTING_ISSUES" | jq '.nodes[] | select(.title | test("Fuzzing.*[Ff]ailure"))' 2>/dev/null || echo "") +FUZZING_ISSUE_COUNT=0 +if [ -n "$FUZZING_ISSUES" ]; then + FUZZING_ISSUE_COUNT=$(echo "$FUZZING_ISSUES" | jq -s 'length' 2>/dev/null || echo "0") +fi + +echo "Found $FUZZING_ISSUE_COUNT existing fuzzing failure issue(s)" + +if [ "$FUZZING_ISSUE_COUNT" -gt 0 ]; then + # Get the most recent fuzzing failure issue + ISSUE_NUMBER=$(echo "$FUZZING_ISSUES" | jq -r '.number' | head -1) + ISSUE_URL=$(echo "$FUZZING_ISSUES" | jq -r '.url' | head -1) + + if [ "$ISSUE_NUMBER" = "null" ] || [ -z "$ISSUE_NUMBER" ]; then + echo "⚠️ Could not parse issue number from fuzzing issues, creating new issue instead" + FUZZING_ISSUE_COUNT=0 + else + echo "Adding comment to existing issue #$ISSUE_NUMBER" + + # Create comment body + COMMENT_BODY="## Fuzzing Job Failed Again + +**Date:** $(date '+%Y-%m-%d %H:%M:%S UTC') +**Workflow Run:** [$WORKFLOW_RUN_URL]($WORKFLOW_RUN_URL) +**Fuzzing Targets:** $FUZZING_TARGETS + +The scheduled fuzzing job has failed again. Please check the workflow logs and artifacts for details. + +### Next Steps +- [ ] Review the workflow logs for error details +- [ ] Download and analyze any crash artifacts +- [ ] Determine if this is a new issue or related to existing problems +- [ ] Fix the underlying issue causing the fuzzing failures" + + # Add comment to the existing issue + if gh issue comment "$ISSUE_NUMBER" --body "$COMMENT_BODY"; then + echo "✅ Added comment to existing issue #$ISSUE_NUMBER: $ISSUE_URL" + else + echo "❌ Failed to add comment to existing issue. Creating new issue instead." + FUZZING_ISSUE_COUNT=0 + fi + fi +fi + +if [ "$FUZZING_ISSUE_COUNT" -eq 0 ]; then + echo "No existing fuzzing failure issues found. Creating new issue..." + + # Create issue body + ISSUE_BODY="## Fuzzing Job Failure Report + +**Date:** $(date '+%Y-%m-%d %H:%M:%S UTC') +**Workflow Run:** [$WORKFLOW_RUN_URL]($WORKFLOW_RUN_URL) +**Fuzzing Targets:** $FUZZING_TARGETS + +The scheduled fuzzing job has failed. This issue was automatically created to track the failure. + +### Details +The fuzzing workflow failed during execution. Please check the workflow logs and any uploaded artifacts for more details. + +### Next Steps +- [ ] Review the workflow logs for error details +- [ ] Download and analyze any crash artifacts if available +- [ ] Determine the root cause of the failure +- [ ] Fix the underlying issue +- [ ] Verify the fix by running fuzzing locally or waiting for the next scheduled run + +### Related Documentation +- [Fuzzing README](https://github.com/$REPO/blob/main/fuzz/README.md) +- [Security Guidance](https://github.com/$REPO/blob/main/docs/security-guidance-for-developers.md) + +--- +*This issue was automatically created by the fuzzing failure notification system.*" + + # Create the new issue + if ISSUE_URL=$(gh issue create \ + --title "$ISSUE_TITLE" \ + --body "$ISSUE_BODY" \ + --label "$FUZZING_LABEL" \ + --label "$FAILURE_LABEL" \ + --label "$LIFECYCLE_LABEL"); then + echo "✅ Created new fuzzing failure issue: $ISSUE_URL" + else + echo "❌ Failed to create new fuzzing failure issue" + exit 1 + fi +fi + +echo "Fuzzing failure notification completed successfully" \ No newline at end of file From 4f8ddb445fca6a3f988ec4bdbc5acb9bab835bc8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 Aug 2025 21:40:28 +0000 Subject: [PATCH 3/8] Move fuzzing failure notification from reusable workflow to scheduled workflow - Removed notification step and issues permission from dep_fuzzing.yml (reusable workflow) - Added separate notify-failure job to Fuzzing.yml (scheduled workflow) - This ensures issues are only created for scheduled cron job failures, not PR runs Co-authored-by: jsturtevant <648372+jsturtevant@users.noreply.github.com> --- .github/workflows/Fuzzing.yml | 15 ++++++++++++++- .github/workflows/dep_fuzzing.yml | 7 ------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Fuzzing.yml b/.github/workflows/Fuzzing.yml index 3565bd559..6b4fd6d97 100644 --- a/.github/workflows/Fuzzing.yml +++ b/.github/workflows/Fuzzing.yml @@ -16,4 +16,17 @@ jobs: with: targets: '["fuzz_host_print", "fuzz_guest_call", "fuzz_host_call"]' # Pass as a JSON array max_total_time: 18000 # 5 hours in seconds - secrets: inherit \ No newline at end of file + secrets: inherit + + notify-failure: + runs-on: ubuntu-latest + needs: fuzzing + if: failure() && needs.fuzzing.result == 'failure' + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Notify Fuzzing Failure + run: ./dev/notify-fuzzing-failure.sh "fuzz_host_print,fuzz_guest_call,fuzz_host_call" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/dep_fuzzing.yml b/.github/workflows/dep_fuzzing.yml index b890b68a3..95008efd4 100644 --- a/.github/workflows/dep_fuzzing.yml +++ b/.github/workflows/dep_fuzzing.yml @@ -20,7 +20,6 @@ on: permissions: id-token: write contents: read - issues: write jobs: fuzz: @@ -61,9 +60,3 @@ jobs: with: name: fuzz-crash-artifacts path: fuzz/artifacts/ - - - name: Notify Fuzzing Failure - if: failure() # Only run this step if the fuzzing job failed - run: ./dev/notify-fuzzing-failure.sh "${{ matrix.target }}" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From ebb4acb1c8df68b0431f80853e7499cfc945a937 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 Aug 2025 22:03:51 +0000 Subject: [PATCH 4/8] Add --repo parameter to gh commands for testing support - Added --repo "$REPO" to gh issue comment command (line 103) - Added --repo "$REPO" to gh issue create command (line 148) - This enables testing against specific repositories by setting the REPO variable Co-authored-by: jsturtevant <648372+jsturtevant@users.noreply.github.com> --- dev/notify-fuzzing-failure.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/notify-fuzzing-failure.sh b/dev/notify-fuzzing-failure.sh index 15ffb835c..6a7bff5dc 100755 --- a/dev/notify-fuzzing-failure.sh +++ b/dev/notify-fuzzing-failure.sh @@ -100,7 +100,7 @@ The scheduled fuzzing job has failed again. Please check the workflow logs and a - [ ] Fix the underlying issue causing the fuzzing failures" # Add comment to the existing issue - if gh issue comment "$ISSUE_NUMBER" --body "$COMMENT_BODY"; then + if gh issue comment "$ISSUE_NUMBER" --body "$COMMENT_BODY" --repo "$REPO"; then echo "✅ Added comment to existing issue #$ISSUE_NUMBER: $ISSUE_URL" else echo "❌ Failed to add comment to existing issue. Creating new issue instead." @@ -144,7 +144,8 @@ The fuzzing workflow failed during execution. Please check the workflow logs and --body "$ISSUE_BODY" \ --label "$FUZZING_LABEL" \ --label "$FAILURE_LABEL" \ - --label "$LIFECYCLE_LABEL"); then + --label "$LIFECYCLE_LABEL" \ + --repo "$REPO"); then echo "✅ Created new fuzzing failure issue: $ISSUE_URL" else echo "❌ Failed to create new fuzzing failure issue" From 9f73599bd6422be09a9105a1360dd675afcde6b6 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 28 Aug 2025 20:34:07 +0000 Subject: [PATCH 5/8] Rebase and revert the code change from copilot Signed-off-by: James Sturtevant --- .github/workflows/Fuzzing.yml | 5 +++-- dev/notify-fuzzing-failure.sh | 13 ++++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/Fuzzing.yml b/.github/workflows/Fuzzing.yml index 6b4fd6d97..a3c19778a 100644 --- a/.github/workflows/Fuzzing.yml +++ b/.github/workflows/Fuzzing.yml @@ -8,7 +8,6 @@ on: permissions: id-token: write contents: read - issues: write jobs: fuzzing: @@ -21,7 +20,9 @@ jobs: notify-failure: runs-on: ubuntu-latest needs: fuzzing - if: failure() && needs.fuzzing.result == 'failure' + if: always() && needs.fuzzing.result == 'failure' + permissions: + issues: write steps: - name: Checkout code uses: actions/checkout@v5 diff --git a/dev/notify-fuzzing-failure.sh b/dev/notify-fuzzing-failure.sh index 6a7bff5dc..2ff67d5a3 100755 --- a/dev/notify-fuzzing-failure.sh +++ b/dev/notify-fuzzing-failure.sh @@ -14,9 +14,11 @@ set -o pipefail ## This script assumes that the gh cli is installed and in the PATH ## and that there is a GitHub PAT in the GITHUB_TOKEN env var ## with the following permissions: -## - repo (read/write) ## - issues (read/write) ## or that the user is logged into the gh cli with an account with those permissions +## +## Run this script locally like: +## GITHUB_REPOSITORY="fork/hyperlight" GITHUB_RUN_ID=1 ./dev/notify-fuzzing-failure.sh "fuzz_host_print,fuzz_guest_call,fuzz_host_call" REPO="${GITHUB_REPOSITORY:-hyperlight-dev/hyperlight}" WORKFLOW_RUN_URL="${GITHUB_SERVER_URL:-https://github.com}/${REPO}/actions/runs/${GITHUB_RUN_ID:-unknown}" @@ -91,13 +93,7 @@ if [ "$FUZZING_ISSUE_COUNT" -gt 0 ]; then **Workflow Run:** [$WORKFLOW_RUN_URL]($WORKFLOW_RUN_URL) **Fuzzing Targets:** $FUZZING_TARGETS -The scheduled fuzzing job has failed again. Please check the workflow logs and artifacts for details. - -### Next Steps -- [ ] Review the workflow logs for error details -- [ ] Download and analyze any crash artifacts -- [ ] Determine if this is a new issue or related to existing problems -- [ ] Fix the underlying issue causing the fuzzing failures" +The scheduled fuzzing job has failed again. Please check the workflow logs and artifacts for details." # Add comment to the existing issue if gh issue comment "$ISSUE_NUMBER" --body "$COMMENT_BODY" --repo "$REPO"; then @@ -129,7 +125,6 @@ The fuzzing workflow failed during execution. Please check the workflow logs and - [ ] Download and analyze any crash artifacts if available - [ ] Determine the root cause of the failure - [ ] Fix the underlying issue -- [ ] Verify the fix by running fuzzing locally or waiting for the next scheduled run ### Related Documentation - [Fuzzing README](https://github.com/$REPO/blob/main/fuzz/README.md) From 588cff23297646bf685b71d9afc78b5301a96ec7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 28 Aug 2025 22:24:02 +0000 Subject: [PATCH 6/8] Add kind/fuzzing label for better issue filtering and management Co-authored-by: jsturtevant <648372+jsturtevant@users.noreply.github.com> --- dev/notify-fuzzing-failure.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dev/notify-fuzzing-failure.sh b/dev/notify-fuzzing-failure.sh index 2ff67d5a3..8c358e629 100755 --- a/dev/notify-fuzzing-failure.sh +++ b/dev/notify-fuzzing-failure.sh @@ -43,6 +43,7 @@ REPO_NAME=$(echo "$REPO" | cut -d'/' -f2) ISSUE_TITLE="Fuzzing Job Failure - $(date '+%Y-%m-%d')" FUZZING_LABEL="area/testing" FAILURE_LABEL="kind/bug" +FUZZING_KIND_LABEL="kind/fuzzing" LIFECYCLE_LABEL="lifecycle/needs-review" # Search for existing open fuzzing failure issues @@ -50,7 +51,7 @@ echo "Searching for existing open fuzzing failure issues..." EXISTING_ISSUES=$(gh api graphql -f query=' query($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { - issues(first: 10, states: OPEN, labels: ["area/testing"]) { + issues(first: 10, states: OPEN, labels: ["kind/fuzzing"]) { totalCount nodes { number @@ -66,8 +67,8 @@ EXISTING_ISSUES=$(gh api graphql -f query=' } }' -f owner="$OWNER" -f repo="$REPO_NAME" --jq '.data.repository.issues') -# Filter for fuzzing-related issues -FUZZING_ISSUES=$(echo "$EXISTING_ISSUES" | jq '.nodes[] | select(.title | test("Fuzzing.*[Ff]ailure"))' 2>/dev/null || echo "") +# Filter for fuzzing-related issues (now all results should be fuzzing issues due to label filter) +FUZZING_ISSUES=$(echo "$EXISTING_ISSUES" | jq '.nodes[]' 2>/dev/null || echo "") FUZZING_ISSUE_COUNT=0 if [ -n "$FUZZING_ISSUES" ]; then FUZZING_ISSUE_COUNT=$(echo "$FUZZING_ISSUES" | jq -s 'length' 2>/dev/null || echo "0") @@ -139,6 +140,7 @@ The fuzzing workflow failed during execution. Please check the workflow logs and --body "$ISSUE_BODY" \ --label "$FUZZING_LABEL" \ --label "$FAILURE_LABEL" \ + --label "$FUZZING_KIND_LABEL" \ --label "$LIFECYCLE_LABEL" \ --repo "$REPO"); then echo "✅ Created new fuzzing failure issue: $ISSUE_URL" From 6e3b129d72df525bdee6df1d1451fb72d8a1e8f1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 29 Aug 2025 18:30:42 +0000 Subject: [PATCH 7/8] Update fuzzing label from kind/fuzzing to area/fuzzing Co-authored-by: jsturtevant <648372+jsturtevant@users.noreply.github.com> --- dev/notify-fuzzing-failure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/notify-fuzzing-failure.sh b/dev/notify-fuzzing-failure.sh index 8c358e629..46e3da6e2 100755 --- a/dev/notify-fuzzing-failure.sh +++ b/dev/notify-fuzzing-failure.sh @@ -43,7 +43,7 @@ REPO_NAME=$(echo "$REPO" | cut -d'/' -f2) ISSUE_TITLE="Fuzzing Job Failure - $(date '+%Y-%m-%d')" FUZZING_LABEL="area/testing" FAILURE_LABEL="kind/bug" -FUZZING_KIND_LABEL="kind/fuzzing" +FUZZING_KIND_LABEL="area/fuzzing" LIFECYCLE_LABEL="lifecycle/needs-review" # Search for existing open fuzzing failure issues @@ -51,7 +51,7 @@ echo "Searching for existing open fuzzing failure issues..." EXISTING_ISSUES=$(gh api graphql -f query=' query($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { - issues(first: 10, states: OPEN, labels: ["kind/fuzzing"]) { + issues(first: 10, states: OPEN, labels: ["area/fuzzing"]) { totalCount nodes { number From 1583008d20b83154645bff09cf414a5b80bbbdaf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 29 Aug 2025 21:12:54 +0000 Subject: [PATCH 8/8] Rename variable names for clarity: testing_label and fuzzing_label Co-authored-by: jsturtevant <648372+jsturtevant@users.noreply.github.com> --- dev/notify-fuzzing-failure.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/notify-fuzzing-failure.sh b/dev/notify-fuzzing-failure.sh index 46e3da6e2..f4a6ac04f 100755 --- a/dev/notify-fuzzing-failure.sh +++ b/dev/notify-fuzzing-failure.sh @@ -41,9 +41,9 @@ REPO_NAME=$(echo "$REPO" | cut -d'/' -f2) # Define the issue title and labels ISSUE_TITLE="Fuzzing Job Failure - $(date '+%Y-%m-%d')" -FUZZING_LABEL="area/testing" +TESTING_LABEL="area/testing" FAILURE_LABEL="kind/bug" -FUZZING_KIND_LABEL="area/fuzzing" +FUZZING_LABEL="area/fuzzing" LIFECYCLE_LABEL="lifecycle/needs-review" # Search for existing open fuzzing failure issues @@ -138,9 +138,9 @@ The fuzzing workflow failed during execution. Please check the workflow logs and if ISSUE_URL=$(gh issue create \ --title "$ISSUE_TITLE" \ --body "$ISSUE_BODY" \ - --label "$FUZZING_LABEL" \ + --label "$TESTING_LABEL" \ --label "$FAILURE_LABEL" \ - --label "$FUZZING_KIND_LABEL" \ + --label "$FUZZING_LABEL" \ --label "$LIFECYCLE_LABEL" \ --repo "$REPO"); then echo "✅ Created new fuzzing failure issue: $ISSUE_URL"