IXUCA-Smoke-Watchdog #924
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: IXUCA-Smoke-Watchdog | |
| on: | |
| workflow_run: | |
| workflows: ["IXUCA-Smoke-Schedule"] | |
| types: [completed] | |
| schedule: | |
| - cron: "*/15 * * * *" # 排队超时检测 | |
| workflow_dispatch: | |
| permissions: | |
| actions: write | |
| contents: read | |
| concurrency: | |
| group: ixuca-smoke-watchdog | |
| cancel-in-progress: true | |
| jobs: | |
| check-and-alert: | |
| name: Check IXUCA smoke status and alert | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Fetch previous watchdog state artifact | |
| id: fetch_state | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const artifactsResp = await github.rest.actions.listArtifactsForRepo({ | |
| owner, | |
| repo, | |
| per_page: 100 | |
| }); | |
| const artifacts = artifactsResp.data.artifacts || []; | |
| const candidates = artifacts | |
| .filter(a => a.name === "ixuca-smoke-watchdog-state" && !a.expired) | |
| .sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); | |
| if (candidates.length > 0) { | |
| core.setOutput("artifact_id", String(candidates[0].id)); | |
| core.info(`Found previous state artifact: ${candidates[0].id}`); | |
| } else { | |
| core.info("No previous state artifact found."); | |
| } | |
| - name: Download previous watchdog state zip | |
| if: steps.fetch_state.outputs.artifact_id != '' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const artifactId = Number("${{ steps.fetch_state.outputs.artifact_id }}"); | |
| const fs = require("fs"); | |
| const response = await github.rest.actions.downloadArtifact({ | |
| owner, | |
| repo, | |
| artifact_id: artifactId, | |
| archive_format: "zip" | |
| }); | |
| fs.writeFileSync("state.zip", Buffer.from(response.data)); | |
| core.info("Downloaded previous state artifact to state.zip"); | |
| - name: Extract previous watchdog state | |
| if: steps.fetch_state.outputs.artifact_id != '' | |
| run: | | |
| python3 - <<'PY' | |
| import zipfile | |
| with zipfile.ZipFile("state.zip") as zf: | |
| if "state.json" not in zf.namelist(): | |
| raise SystemExit("state.json not found in artifact") | |
| with zf.open("state.json") as src, open("prev_state.json", "wb") as dst: | |
| dst.write(src.read()) | |
| print("Extracted previous state to prev_state.json") | |
| PY | |
| - name: Evaluate and handle smoke status | |
| id: evaluate | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const workflow_id = "ixuca-smoke-schedule.yml"; | |
| const fs = require("fs"); | |
| const now = new Date(); | |
| const oneDayMs = 24 * 60 * 60 * 1000; | |
| const QUEUE_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour | |
| // 默认状态 | |
| let prevState = { | |
| status: "OK", | |
| last_conclusion: null, | |
| consecutive_success: 0, | |
| last_alerted_at: null, | |
| last_alert_type: "", | |
| pending_double_check: false | |
| }; | |
| if (fs.existsSync("prev_state.json")) { | |
| try { | |
| prevState = { ...prevState, ...JSON.parse(fs.readFileSync("prev_state.json", "utf8")) }; | |
| } catch (err) { | |
| core.warning(`Failed to parse prev_state.json: ${err.message}`); | |
| } | |
| } | |
| const triggerEvent = context.eventName; | |
| let shouldAlert = false; | |
| let alertType = ""; | |
| let alertText = ""; | |
| let needDoubleCheck = false; | |
| core.info(`Trigger event: ${triggerEvent}`); | |
| // 辅助函数：启动 double-check | |
| async function triggerDoubleCheck() { | |
| core.info("Triggering double-check workflow..."); | |
| await github.rest.actions.createWorkflowDispatch({ | |
| owner, | |
| repo, | |
| workflow_id, | |
| ref: "develop", | |
| inputs: { | |
| check_type: "double_check" | |
| } | |
| }); | |
| } | |
| // 辅助函数：发送告警 | |
| function getAlertMessage(type, url) { | |
| if (type === "CONTAINER_ERROR") { | |
| return `[CONTAINER_ERROR] CI容器异常：连续两次 Smoke 测试失败。\nrun_url: ${url}`; | |
| } else if (type === "SCHEDULING_ERROR") { | |
| return `[SCHEDULING_ERROR] CI调度异常：连续两次 Smoke 测试取消或排队超时。\nrun_url: ${url}`; | |
| } else if (type === "RECOVERED") { | |
| return `[RECOVERED] 问题已修复：连续两次 Smoke 测试成功。\nrun_url: ${url}`; | |
| } | |
| return ""; | |
| } | |
| // 辅助函数：检查是否需要告警（去重） | |
| function shouldSendAlert(type) { | |
| const prevAlertAt = prevState.last_alerted_at ? new Date(prevState.last_alerted_at) : null; | |
| const sameIssue = prevState.last_alert_type === type; | |
| if (type === "RECOVERED") { | |
| // 恢复通知：之前有异常状态才发 | |
| return prevState.status !== "OK"; | |
| } | |
| // 新问题或不同问题：直接告警 | |
| if (!sameIssue) { | |
| return true; | |
| } | |
| // 相同问题：24小时内不重复告警 | |
| if (prevAlertAt && now - prevAlertAt < oneDayMs) { | |
| core.info(`Same issue ${type} already alerted at ${prevAlertAt.toISOString()}, skipping.`); | |
| return false; | |
| } | |
| return true; // 超过24小时，可以重复告警 | |
| } | |
| if (triggerEvent === "workflow_run") { | |
| // ============ workflow_run 触发：处理任务完成 ============ | |
| const currentRun = context.payload.workflow_run; | |
| const conclusion = currentRun.conclusion; | |
| const htmlUrl = currentRun.html_url; | |
| const runEvent = currentRun.event; | |
| // 判断 check_type： | |
| // - schedule 触发的任务 event 是 "schedule" | |
| // - workflow_dispatch 触发的任务 event 是 "workflow_dispatch"，需要进一步看 inputs | |
| let checkType = "scheduled"; | |
| if (runEvent === "workflow_dispatch") { | |
| // 从 workflow_run 获取 inputs 需要额外查询 | |
| // 这里简化处理：如果是从 workflow_dispatch 触发的，检查原因 | |
| // 通过重新运行 ID 来获取 inputs | |
| try { | |
| const runDetail = await github.rest.actions.getWorkflowRun({ | |
| owner, | |
| repo, | |
| run_id: currentRun.id | |
| }); | |
| const inputs = runDetail.data.inputs || {}; | |
| checkType = inputs.check_type || "scheduled"; | |
| } catch (err) { | |
| core.warning(`Failed to get workflow run details: ${err.message}`); | |
| } | |
| } | |
| core.info(`Workflow run completed: conclusion=${conclusion}, check_type=${checkType}, url=${htmlUrl}`); | |
| if (conclusion === "success") { | |
| // 成功处理 | |
| prevState.consecutive_success++; | |
| prevState.pending_double_check = false; | |
| prevState.last_conclusion = "success"; | |
| // 连续两次成功才认为恢复 | |
| if (prevState.consecutive_success >= 2 && prevState.status !== "OK") { | |
| if (shouldSendAlert("RECOVERED")) { | |
| shouldAlert = true; | |
| alertType = "RECOVERED"; | |
| alertText = getAlertMessage("RECOVERED", htmlUrl); | |
| } | |
| prevState.status = "OK"; | |
| prevState.last_alert_type = ""; | |
| prevState.last_alerted_at = null; | |
| } | |
| core.info(`Success: consecutive_success=${prevState.consecutive_success}`); | |
| } else if (conclusion === "failure") { | |
| prevState.last_conclusion = "failure"; | |
| prevState.consecutive_success = 0; | |
| if (checkType === "scheduled" && !prevState.pending_double_check) { | |
| // 首次失败，启动 double-check | |
| needDoubleCheck = true; | |
| prevState.pending_double_check = true; | |
| core.info("First failure detected, triggering double-check..."); | |
| } else if (checkType === "double_check" && prevState.pending_double_check) { | |
| // double-check 也失败，确认容器异常 | |
| if (shouldSendAlert("CONTAINER_ERROR")) { | |
| shouldAlert = true; | |
| alertType = "CONTAINER_ERROR"; | |
| alertText = getAlertMessage("CONTAINER_ERROR", htmlUrl); | |
| } | |
| prevState.status = "CONTAINER_ERROR"; | |
| prevState.pending_double_check = false; | |
| core.info("Double-check also failed, confirming container error."); | |
| } | |
| } else if (conclusion === "cancelled") { | |
| prevState.last_conclusion = "cancelled"; | |
| prevState.consecutive_success = 0; | |
| if (checkType === "scheduled" && !prevState.pending_double_check) { | |
| // 首次取消，启动 double-check | |
| needDoubleCheck = true; | |
| prevState.pending_double_check = true; | |
| core.info("First cancellation detected, triggering double-check..."); | |
| } else if (checkType === "double_check" && prevState.pending_double_check) { | |
| // double-check 也取消，确认调度异常 | |
| if (shouldSendAlert("SCHEDULING_ERROR")) { | |
| shouldAlert = true; | |
| alertType = "SCHEDULING_ERROR"; | |
| alertText = getAlertMessage("SCHEDULING_ERROR", htmlUrl); | |
| } | |
| prevState.status = "SCHEDULING_ERROR"; | |
| prevState.pending_double_check = false; | |
| core.info("Double-check also cancelled, confirming scheduling error."); | |
| } | |
| } | |
| } else if (triggerEvent === "schedule") { | |
| // ============ schedule 触发：排队超时检测 ============ | |
| core.info("Running queue timeout check..."); | |
| try { | |
| const runsResp = await github.rest.actions.listWorkflowRuns({ | |
| owner, | |
| repo, | |
| workflow_id, | |
| status: "queued", | |
| per_page: 5 | |
| }); | |
| const runs = runsResp.data.workflow_runs || []; | |
| core.info(`Found ${runs.length} queued runs.`); | |
| for (const run of runs) { | |
| const runTime = new Date(run.created_at); | |
| const elapsed = now - runTime; | |
| core.info(`Run ${run.id}: created_at=${run.created_at}, elapsed=${Math.round(elapsed / 60000)} minutes`); | |
| if (elapsed > QUEUE_TIMEOUT_MS) { | |
| core.info(`Run ${run.id} has been queued for over 1 hour, cancelling and triggering double-check.`); | |
| // 取消该运行 | |
| try { | |
| await github.rest.actions.cancelWorkflowRun({ | |
| owner, | |
| repo, | |
| run_id: run.id | |
| }); | |
| core.info(`Cancelled run ${run.id}`); | |
| } catch (err) { | |
| core.warning(`Failed to cancel run ${run.id}: ${err.message}`); | |
| } | |
| // 启动 double-check | |
| needDoubleCheck = true; | |
| prevState.pending_double_check = true; | |
| break; | |
| } | |
| } | |
| } catch (err) { | |
| core.warning(`Failed to list workflow runs: ${err.message}`); | |
| } | |
| } else { | |
| // workflow_dispatch 手动触发：执行完整的检查逻辑（兼容旧行为） | |
| core.info("Manual trigger, performing full check..."); | |
| const runsResp = await github.rest.actions.listWorkflowRuns({ | |
| owner, | |
| repo, | |
| workflow_id, | |
| status: "completed", | |
| per_page: 10 | |
| }); | |
| const runs = runsResp.data.workflow_runs || []; | |
| const recent = runs.slice(0, 2); | |
| if (recent.length < 2) { | |
| core.info("Not enough completed runs to evaluate. Need at least 2."); | |
| } else { | |
| const latest = recent[0]; | |
| const previous = recent[1]; | |
| const schedulingIssue = | |
| latest.conclusion === "cancelled" && | |
| previous.conclusion === "cancelled"; | |
| const smokeFailed = latest.conclusion === "failure"; | |
| let status = "OK"; | |
| if (schedulingIssue) { | |
| status = "SCHEDULING_ERROR"; | |
| } else if (smokeFailed) { | |
| status = "CONTAINER_ERROR"; | |
| } | |
| if (status !== "OK" && shouldSendAlert(status)) { | |
| shouldAlert = true; | |
| alertType = status; | |
| alertText = getAlertMessage(status, latest.html_url); | |
| if (schedulingIssue) { | |
| alertText += `\nprevious: ${previous.html_url}`; | |
| } | |
| } | |
| prevState.status = status; | |
| if (latest.conclusion === "success") { | |
| prevState.consecutive_success++; | |
| } else { | |
| prevState.consecutive_success = 0; | |
| } | |
| prevState.last_conclusion = latest.conclusion; | |
| } | |
| } | |
| // 触发 double-check（如果需要） | |
| if (needDoubleCheck) { | |
| await triggerDoubleCheck(); | |
| } | |
| // 更新告警时间 | |
| if (shouldAlert && alertType && alertType !== "RECOVERED") { | |
| prevState.last_alerted_at = now.toISOString(); | |
| prevState.last_alert_type = alertType; | |
| } else if (shouldAlert && alertType === "RECOVERED") { | |
| prevState.last_alerted_at = null; | |
| prevState.last_alert_type = ""; | |
| } | |
| // 保存状态 | |
| const nextState = { | |
| status: prevState.status, | |
| last_conclusion: prevState.last_conclusion, | |
| consecutive_success: prevState.consecutive_success, | |
| last_alerted_at: prevState.last_alerted_at, | |
| last_alert_type: prevState.last_alert_type, | |
| pending_double_check: prevState.pending_double_check, | |
| updated_at: now.toISOString() | |
| }; | |
| fs.writeFileSync("state.json", JSON.stringify(nextState, null, 2)); | |
| core.info(`State saved: ${JSON.stringify(nextState, null, 2)}`); | |
| core.setOutput("should_alert", shouldAlert ? "true" : "false"); | |
| core.setOutput("alert_type", alertType); | |
| core.setOutput("alert_text", alertText); | |
| - name: Send alert to Baidu IM robot | |
| if: steps.evaluate.outputs.should_alert == 'true' | |
| env: | |
| WEBHOOK_URL: ${{ secrets.BAIDU_IM_WEBHOOK_URL }} | |
| ALERT_TEXT: ${{ steps.evaluate.outputs.alert_text }} | |
| ALERT_TYPE: ${{ steps.evaluate.outputs.alert_type }} | |
| TO_ID: "11992367" | |
| run: | | |
| python3 - <<'PY' | |
| import json | |
| import os | |
| import time | |
| import urllib.request | |
| webhook_url = os.environ.get("WEBHOOK_URL", "").strip() | |
| if not webhook_url: | |
| raise SystemExit("Need secret BAIDU_IM_WEBHOOK_URL for alerting.") | |
| alert_text = os.environ["ALERT_TEXT"] | |
| alert_type = os.environ["ALERT_TYPE"] | |
| to_id = int(os.environ.get("TO_ID", "11992367")) | |
| payload = { | |
| "message": { | |
| "header": { | |
| "toid": to_id, | |
| "totype": "GROUP", | |
| "msgtype": "TEXT", | |
| "clientmsgid": int(time.time() * 1000), | |
| "role": "robot" | |
| }, | |
| "body": [ | |
| { | |
| "content": f"[{alert_type}] {alert_text}", | |
| "type": "TEXT" | |
| } | |
| ] | |
| } | |
| } | |
| req = urllib.request.Request( | |
| webhook_url, | |
| data=json.dumps(payload).encode("utf-8"), | |
| headers={"Content-Type": "application/json"}, | |
| method="POST", | |
| ) | |
| with urllib.request.urlopen(req, timeout=20) as resp: | |
| print(f"alert sent, status={resp.status}") | |
| PY | |
| - name: Upload watchdog state | |
| if: steps.evaluate.conclusion == 'success' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ixuca-smoke-watchdog-state | |
| path: state.json | |
| retention-days: 7 |