1- # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
1+ # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+ # SPDX-License-Identifier: Apache-2.0
23#
34# Licensed under the Apache License, Version 2.0 (the "License");
45# you may not use this file except in compliance with the License.
@@ -16,238 +17,27 @@ name: Approve Test Queue
1617
1718on :
1819 schedule :
19- - cron : ' */5 * * * *' # Runs every 5 minutes
20+ - cron : " */5 * * * *"
2021 # Scheduled events can be delayed, so refill released queue slots immediately.
2122 workflow_run :
2223 workflows : ["CICD NeMo"]
2324 types : [completed]
24- workflow_dispatch : # Allows manual triggering
25+ workflow_dispatch :
2526
2627concurrency :
2728 group : approve-test-queue
2829 cancel-in-progress : false
2930
3031jobs :
31- approve-queue :
32- runs-on : ubuntu-latest
33- environment : main
34- steps :
35- - name : Checkout repository
36- uses : actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
37-
38- - name : Set up Python
39- uses : actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
40- with :
41- python-version : " 3.12"
42-
43- - name : Install dependencies
44- run : |
45- python -m pip install --upgrade pip
46- pip install requests
47-
48- name : Approve waiting deployments
49- env :
50- GITHUB_TOKEN : ${{ secrets.PAT }}
51- MAX_CONCURRENCY : ${{ vars.MAX_CONCURRENCY || 1 }}
52- run : |
53- python - <<EOF
54- import os
55- import requests
56-
57-
58- # GitHub API configuration
59- GITHUB_TOKEN = os.environ["GITHUB_TOKEN"]
60- REPO = os.environ["GITHUB_REPOSITORY"]
61- MAX_CONCURRENCY = int(os.environ["MAX_CONCURRENCY"])
62- API_BASE = f"https://api.github.com/repos/{REPO}"
63-
64- # Headers for GitHub API
65- headers = {
66- "Authorization": f"token {GITHUB_TOKEN}",
67- "Accept": "application/vnd.github.v3+json",
68- "X-GitHub-Api-Version": "2022-11-28",
69- }
70-
71- def make_request(endpoint, method="GET", data=None, allow_no_pending_deployments=False):
72- """Make a request to the GitHub API with error handling."""
73- url = f"{API_BASE}/{endpoint}"
74- try:
75- if method == "GET":
76- response = requests.get(url, headers=headers)
77- else:
78- response = requests.post(url, headers=headers, json=data)
79- response.raise_for_status()
80- response_json = response.json()
81- if hasattr(response, "links") and "actions/runs?status" in endpoint:
82- response_json["next"] = response.links.get("next", {}).get("url")
83-
84- return response_json
85- except requests.exceptions.HTTPError as e:
86- if (
87- allow_no_pending_deployments
88- and e.response is not None
89- and e.response.status_code == 422
90- ):
91- try:
92- response_json = e.response.json()
93- except ValueError:
94- response_json = {}
95- if "No pending deployment requests to approve or reject" in str(response_json.get("errors", "")):
96- print(f"No pending deployment requests remain for {endpoint}; skipping")
97- return {"skipped": True, "reason": "no_pending_deployments"}
98-
99- print(f"Error making request to {endpoint}: {str(e)}")
100- if e.response is not None:
101- print(f"Response: {e.response.text}")
102- return None
103- except requests.exceptions.RequestException as e:
104- print(f"Error making request to {endpoint}: {str(e)}")
105- if e.response is not None:
106- print(f"Response: {e.response.text}")
107- return None
108-
109-
110- def get_workflow_runs(status):
111- """Get all workflow runs for a given status."""
112- all_results = []
113- endpoint = f"actions/runs?status={status}"
114- while endpoint:
115- response = make_request(endpoint)
116- if not response:
117- break
118-
119- all_results.extend(response.get("workflow_runs", []))
120- endpoint = None
121- next_url = response.get("next")
122- if next_url:
123- endpoint = f"actions/runs?{next_url.split('?')[1]}"
124-
125- return all_results
126-
127-
128- def filter_cicd_runs(workflow_runs):
129- """Keep only CICD workflow runs."""
130- return [run for run in workflow_runs if run.get("name") == "CICD NeMo"]
131-
132-
133- def print_workflow_run_details(label, workflow_runs):
134- """Print the runs that are counted against concurrency."""
135- if not workflow_runs:
136- print(f"{label}: none")
137- return
138-
139- print(f"{label}:")
140- for run in workflow_runs:
141- print(
142- " "
143- f"id={run.get('id')} "
144- f"status={run.get('status')} "
145- f"branch={run.get('head_branch')} "
146- f"title={run.get('display_title')}"
147- )
148-
149-
150- # Get current running and queued workflows
151- print("Fetching workflow runs...")
152- queued_workflow_runs = filter_cicd_runs(get_workflow_runs("queued"))
153- in_progress_workflow_runs = filter_cicd_runs(get_workflow_runs("in_progress"))
154- print_workflow_run_details("Queued CICD workflows counted against concurrency", queued_workflow_runs)
155- print_workflow_run_details("Running CICD workflows counted against concurrency", in_progress_workflow_runs)
156-
157- # Count running and queued workflows
158- queued_workflows = len(queued_workflow_runs)
159- in_progress_workflows = len(in_progress_workflow_runs)
160-
161- total_workflows = queued_workflows + in_progress_workflows
162- print(f"Current queued workflows: {queued_workflows}")
163- print(f"Current running workflows: {in_progress_workflows}")
164- print(f"Total workflows: {total_workflows}")
165- print(f"Max concurrency: {MAX_CONCURRENCY}")
166-
167- if total_workflows >= MAX_CONCURRENCY:
168- print("Maximum concurrency reached, no new approvals will be made")
169- exit(0)
170-
171- # Get waiting CI workflows for test environment
172- print("Fetching deployments...")
173- pending_workflows = filter_cicd_runs(get_workflow_runs("waiting"))
174-
175- # Sort deployments by creation date (oldest first)
176- print("Sorting workflows...")
177- pending_workflows = sorted(pending_workflows, key=lambda x: x.get("created_at", ""))
178-
179- # Process each deployment
180- print("Processing ...")
181- for workflow in pending_workflows:
182- if total_workflows >= MAX_CONCURRENCY:
183- print("Maximum concurrency reached, stopping approvals")
184- break
185-
186- workflow_id = workflow.get("id")
187- workflow_name = workflow.get("display_title") or workflow.get("name") or "<unknown>"
188- if not workflow_id:
189- print(f"Skipping workflow without a run id: {workflow_name}")
190- continue
191- print(f"Approving workflow {workflow_name} with Run Id: {workflow_id}")
192-
193- deployment_url = f"actions/runs/{workflow_id}/pending_deployments"
194- pending_deployments = make_request(deployment_url)
195- if not pending_deployments:
196- print(f"No pending deployments found for workflow {workflow_name}; skipping")
197- continue
198-
199- environment_ids = []
200- environment_names = []
201- for deployment in pending_deployments:
202- environment = deployment.get("environment") or {}
203- environment_id = environment.get("id")
204- environment_name = environment.get("name") or "<unknown>"
205- if not environment_id:
206- print(f"Skipping deployment without an environment id for workflow {workflow_name}")
207- continue
208-
209- environment_ids.append(environment_id)
210- environment_names.append(environment_name)
211-
212- if not environment_ids:
213- print(f"No pending deployments with environment ids found for workflow {workflow_name}")
214- exit(1)
215-
216- # Approve the deployment
217- status_data = {
218- "environment_ids": environment_ids,
219- "state": "approved",
220- "comment": "Automatically approved by queue manager"
221- }
222- result = make_request(
223- deployment_url,
224- method="POST",
225- data=status_data,
226- allow_no_pending_deployments=True,
227- )
228-
229- if result is None:
230- print(f"Failed to approve environments {environment_names} for workflow {workflow_name}")
231- exit(1)
232- if isinstance(result, dict) and result.get("skipped"):
233- continue
234-
235- total_workflows += 1
236-
237- EOF
238- notify :
239- if : failure()
240- runs-on : ubuntu-latest
241- needs : [approve-queue]
242- steps :
243- - name : Notify
244- env :
245- SLACK_WEBHOOK : ${{ secrets.SLACK_WEBHOOK }}
246- SLACK_WEBHOOK_ADMIN : <!subteam^${{ secrets.SLACK_WEBHOOK_ADMIN }}>
247- GITHUB_RUN_ID : ${{ github.run_id }}
248- GITHUB_REPOSITORY : ${{ github.repository }}
249- run : |
250- curl -X POST \
251- -H 'Content-type: application/json' \
252- --data "{\"text\":\":robot_joy: <https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}|Test-queue-approval-bot workflow> failed. Please review manually.\n\ncc ${SLACK_WEBHOOK_ADMIN}\"}" \
253- $SLACK_WEBHOOK
32+ approve-test-queue :
33+ if : github.repository == 'NVIDIA-NeMo/Speech'
34+ uses : NVIDIA-NeMo/FW-CI-templates/.github/workflows/_test_approval_queue.yml@f07495d7a01aad5578a407db8e0c4f4e395375f6 # v1.9.1
35+ with :
36+ workflow_name : CICD NeMo
37+ max_concurrency_internal : ${{ fromJSON(vars.MAX_CONCURRENCY || '1') }}
38+ max_concurrency_external : ${{ fromJSON(vars.MAX_CONCURRENCY || '1') }}
39+ secrets :
40+ PAT : ${{ secrets.PAT }}
41+ NVIDIA_MANAGEMENT_ORG_PAT : ${{ secrets.NVIDIA_MANAGEMENT_ORG_PAT }}
42+ SLACK_CI_CHANNEL_WEBHOOK : ${{ secrets.SLACK_WEBHOOK }}
43+ SLACK_TEAM_GROUP_ID : ${{ secrets.SLACK_WEBHOOK_ADMIN }}
0 commit comments