Skip to content

Commit 36f87e8

Browse files
davidfowldavidfowl
authored
Refactor job retrieval logic to implement manual pagination for fetching failed jobs (dotnet#13327)
1 parent 846e42e commit 36f87e8

File tree

1 file changed

+99
-74
lines changed

1 file changed

+99
-74
lines changed

‎tools/scripts/DownloadFailingJobLogs.cs‎

Lines changed: 99 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
// Downloads logs and artifacts for failed jobs from a GitHub Actions workflow run.
2+
// Usage: dotnet run DownloadFailingJobLogs.cs <run-id>
3+
14
using System.Globalization;
25
using System.IO.Compression;
36
using System.Text.Json;
@@ -15,33 +18,46 @@
1518

1619
Console.WriteLine($"Finding failed jobs for run {runId}...");
1720

18-
// Get all jobs for the run
19-
var getJobsProcess = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo
21+
// Fetch all jobs for this run (paginated, 100 per page)
22+
var jobs = new List<JsonElement>();
23+
int jobsPage = 1;
24+
bool hasMoreJobPages = true;
25+
26+
while (hasMoreJobPages)
2027
{
21-
FileName = "gh",
22-
Arguments = $"api repos/{repo}/actions/runs/{runId}/jobs --paginate",
23-
RedirectStandardOutput = true,
24-
UseShellExecute = false
25-
});
28+
var (success, jobsJson) = await RunGhAsync($"api repos/{repo}/actions/runs/{runId}/jobs?page={jobsPage}&per_page=100");
29+
if (!success)
30+
{
31+
Console.WriteLine($"Error getting jobs page {jobsPage}: {jobsJson}");
32+
return;
33+
}
2634

27-
var jobsJson = await getJobsProcess!.StandardOutput.ReadToEndAsync().ConfigureAwait(false);
28-
await getJobsProcess.WaitForExitAsync().ConfigureAwait(false);
35+
using var jobsDoc = JsonDocument.Parse(jobsJson);
36+
if (jobsDoc.RootElement.TryGetProperty("jobs", out var jobsArray))
37+
{
38+
var jobsOnPage = jobsArray.GetArrayLength();
39+
if (jobsOnPage == 0)
40+
{
41+
hasMoreJobPages = false;
42+
}
2943

30-
if (getJobsProcess.ExitCode != 0)
31-
{
32-
Console.WriteLine($"Error getting jobs: exit code {getJobsProcess.ExitCode}");
33-
return;
34-
}
44+
foreach (var job in jobsArray.EnumerateArray())
45+
{
46+
jobs.Add(job.Clone());
47+
}
3548

36-
// Parse jobs - gh --paginate returns multiple JSON objects concatenated
37-
var jobs = new List<JsonElement>();
38-
var reader = new Utf8JsonReader(System.Text.Encoding.UTF8.GetBytes(jobsJson));
39-
while (JsonDocument.TryParseValue(ref reader, out var doc))
40-
{
41-
if (doc != null && doc.RootElement.TryGetProperty("jobs", out var jobsArray))
49+
// If we got fewer than 100 jobs, this is the last page
50+
if (jobsOnPage < 100)
51+
{
52+
hasMoreJobPages = false;
53+
}
54+
}
55+
else
4256
{
43-
jobs.AddRange(jobsArray.EnumerateArray());
57+
hasMoreJobPages = false;
4458
}
59+
60+
jobsPage++;
4561
}
4662

4763
Console.WriteLine($"Found {jobs.Count} total jobs");
@@ -76,26 +92,17 @@
7692

7793
// Download job logs
7894
Console.WriteLine("Downloading job logs...");
79-
var downloadProcess = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo
80-
{
81-
FileName = "gh",
82-
Arguments = $"api repos/{repo}/actions/jobs/{jobId}/logs",
83-
RedirectStandardOutput = true,
84-
UseShellExecute = false
85-
});
86-
87-
var logs = await downloadProcess!.StandardOutput.ReadToEndAsync().ConfigureAwait(false);
88-
await downloadProcess.WaitForExitAsync().ConfigureAwait(false);
95+
var (logsSuccess, logs) = await RunGhAsync($"api repos/{repo}/actions/jobs/{jobId}/logs");
8996

90-
if (downloadProcess.ExitCode == 0)
97+
if (logsSuccess)
9198
{
9299
// Save logs to file
93100
var safeName = Regex.Replace(jobName ?? $"job_{jobId}", @"[^a-zA-Z0-9_-]", "_");
94101
var filename = $"failed_job_{counter}_{safeName}.log";
95102
File.WriteAllText(filename, logs);
96103
Console.WriteLine($"Saved job logs to: {filename} ({logs.Length} characters)");
97104

98-
// Extract and display test failures
105+
// Parse logs for test failures and exceptions
99106
Console.WriteLine("\nSearching for test failures in job logs...");
100107
var failedTestPattern = @"Failed\s+(.+?)\s*\[";
101108
var errorPattern = @"Error Message:\s*(.+?)(?:\r?\n|$)";
@@ -152,13 +159,11 @@
152159
}
153160
else
154161
{
155-
Console.WriteLine($"Error downloading job logs: exit code {downloadProcess.ExitCode}");
162+
Console.WriteLine($"Error downloading job logs: {logs}");
156163
}
157164

158-
// Try to download artifact based on job name
159-
// Job name format: "Tests / Integrations macos (Hosting.Azure) / Hosting.Azure (macos-latest)"
160-
// Artifact name format: "logs-{testShortName}-{os}"
161-
// Extract testShortName and os from job name
165+
// Try to find and download the test artifact based on job name pattern
166+
// e.g. "Tests / Integrations macos (Hosting.Azure) / Hosting.Azure (macos-latest)" -> "logs-Hosting.Azure-macos-latest"
162167
var artifactMatch = Regex.Match(jobName ?? "", @".*\(([^)]+)\)\s*/\s*\S+\s+\(([^)]+)\)");
163168
if (artifactMatch.Success)
164169
{
@@ -168,41 +173,28 @@
168173

169174
Console.WriteLine($"\nAttempting to download artifact: {artifactName}");
170175

171-
// Query all artifacts (manual pagination through all pages)
176+
// Search for matching artifact (paginated, 100 per page)
172177
string? artifactId = null;
173178
var allArtifactNames = new List<string>();
174-
175-
// Manually paginate through all pages (GitHub API returns 30 per page by default, with up to 100 per page max)
176179
int page = 1;
177180
bool hasMorePages = true;
178181

179182
while (hasMorePages)
180183
{
181-
var getArtifactsProcess = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo
184+
var (artifactsSuccess, artifactsJson) = await RunGhAsync($"api repos/{repo}/actions/runs/{runId}/artifacts?page={page}&per_page=100");
185+
if (!artifactsSuccess)
182186
{
183-
FileName = "gh",
184-
Arguments = $"api repos/{repo}/actions/runs/{runId}/artifacts?page={page}&per_page=100",
185-
RedirectStandardOutput = true,
186-
UseShellExecute = false
187-
});
188-
189-
var artifactsJson = await getArtifactsProcess!.StandardOutput.ReadToEndAsync().ConfigureAwait(false);
190-
await getArtifactsProcess.WaitForExitAsync().ConfigureAwait(false);
191-
192-
if (getArtifactsProcess.ExitCode != 0)
193-
{
194-
Console.WriteLine($"Error getting artifacts page {page}: exit code {getArtifactsProcess.ExitCode}");
187+
Console.WriteLine($"Error getting artifacts page {page}: {artifactsJson}");
195188
break;
196189
}
197190

198-
var artifactsDoc = JsonDocument.Parse(artifactsJson);
191+
using var artifactsDoc = JsonDocument.Parse(artifactsJson);
199192
if (artifactsDoc.RootElement.TryGetProperty("artifacts", out var artifactsArray))
200193
{
201194
var artifactsOnPage = artifactsArray.GetArrayLength();
202195
if (artifactsOnPage == 0)
203196
{
204197
hasMorePages = false;
205-
break;
206198
}
207199

208200
foreach (var artifact in artifactsArray.EnumerateArray())
@@ -241,29 +233,16 @@
241233

242234
// Download artifact
243235
var artifactZip = $"artifact_{counter}_{testShortName}_{os}.zip";
244-
var downloadArtifactProcess = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo
245-
{
246-
FileName = "gh",
247-
Arguments = $"api repos/{repo}/actions/artifacts/{artifactId}/zip",
248-
RedirectStandardOutput = true,
249-
UseShellExecute = false
250-
});
236+
var downloadSuccess = await DownloadGhFileAsync($"api repos/{repo}/actions/artifacts/{artifactId}/zip", artifactZip);
251237

252-
using (var fileStream = File.Create(artifactZip))
253-
{
254-
await downloadArtifactProcess!.StandardOutput.BaseStream.CopyToAsync(fileStream).ConfigureAwait(false);
255-
}
256-
await downloadArtifactProcess.WaitForExitAsync().ConfigureAwait(false);
257-
258-
if (downloadArtifactProcess.ExitCode == 0)
238+
if (downloadSuccess)
259239
{
260240
Console.WriteLine($"Downloaded artifact to: {artifactZip}");
261241

262-
// Unzip artifact using System.IO.Compression
242+
// Extract and look for .trx test result files
263243
var extractDir = $"artifact_{counter}_{testShortName}_{os}";
264244
try
265245
{
266-
// Delete existing directory if it exists
267246
if (Directory.Exists(extractDir))
268247
{
269248
Directory.Delete(extractDir, true);
@@ -272,7 +251,6 @@
272251
ZipFile.ExtractToDirectory(artifactZip, extractDir);
273252
Console.WriteLine($"Extracted artifact to: {extractDir}");
274253

275-
// List .trx files
276254
var trxFiles = Directory.GetFiles(extractDir, "*.trx", SearchOption.AllDirectories);
277255
if (trxFiles.Length > 0)
278256
{
@@ -294,7 +272,7 @@
294272
}
295273
else
296274
{
297-
Console.WriteLine($"Error downloading artifact: exit code {downloadArtifactProcess.ExitCode}");
275+
Console.WriteLine("Error downloading artifact");
298276
}
299277
}
300278
else
@@ -315,3 +293,50 @@
315293
Console.WriteLine($"Failed jobs: {failedJobs.Count}");
316294
Console.WriteLine($"Logs downloaded: {counter}");
317295
Console.WriteLine($"\nAll logs saved in current directory with pattern: failed_job_*.log");
296+
297+
// Runs gh CLI and returns the text output
298+
async Task<(bool Success, string Output)> RunGhAsync(string arguments)
299+
{
300+
using var process = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo
301+
{
302+
FileName = "gh",
303+
Arguments = arguments,
304+
RedirectStandardOutput = true,
305+
UseShellExecute = false
306+
});
307+
308+
if (process is null)
309+
{
310+
return (false, "Failed to start process");
311+
}
312+
313+
var output = await process.StandardOutput.ReadToEndAsync().ConfigureAwait(false);
314+
await process.WaitForExitAsync().ConfigureAwait(false);
315+
316+
return (process.ExitCode == 0, output);
317+
}
318+
319+
// Runs gh CLI and streams binary output to a file
320+
async Task<bool> DownloadGhFileAsync(string arguments, string outputPath)
321+
{
322+
using var process = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo
323+
{
324+
FileName = "gh",
325+
Arguments = arguments,
326+
RedirectStandardOutput = true,
327+
UseShellExecute = false
328+
});
329+
330+
if (process is null)
331+
{
332+
return false;
333+
}
334+
335+
using (var fileStream = File.Create(outputPath))
336+
{
337+
await process.StandardOutput.BaseStream.CopyToAsync(fileStream).ConfigureAwait(false);
338+
}
339+
await process.WaitForExitAsync().ConfigureAwait(false);
340+
341+
return process.ExitCode == 0;
342+
}

0 commit comments

Comments
 (0)