From 0763a9b711f156f2e27101bcabbbdf4fbd685e41 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Wed, 13 Nov 2024 15:49:10 +0100 Subject: [PATCH 1/2] PoC download externally stored evaluator logs --- extensions/ql-vscode/package.json | 4 + .../compare-performance-view.ts | 317 +++++++++++++++++- extensions/ql-vscode/src/extension.ts | 1 + 3 files changed, 317 insertions(+), 5 deletions(-) diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 07a57ec03e8..169bf9ad228 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -751,6 +751,10 @@ "dark": "media/dark/github.svg" } }, + { + "command": "codeQL.compare-performance.downloadExternalLogs", + "title": "CodeQL: Download External Logs for Performance Comparison" + }, { "command": "codeQL.setCurrentDatabase", "title": "CodeQL: Set Current Database" diff --git a/extensions/ql-vscode/src/compare-performance/compare-performance-view.ts b/extensions/ql-vscode/src/compare-performance/compare-performance-view.ts index d682c3ac785..bcb6e12f0b7 100644 --- a/extensions/ql-vscode/src/compare-performance/compare-performance-view.ts +++ b/extensions/ql-vscode/src/compare-performance/compare-performance-view.ts @@ -1,8 +1,17 @@ -import { ViewColumn } from "vscode"; - +import { execFileSync } from "child_process"; +import { + createWriteStream, + ensureDir, + existsSync, + readdirSync, + remove, +} from "fs-extra"; +import path, { basename, join } from "path"; +import { Uri, ViewColumn } from "vscode"; import type { CodeQLCliServer } from "../codeql-cli/cli"; import type { App } from "../common/app"; import { redactableError } from "../common/errors"; +import { createTimeoutSignal } from "../common/fetch-stream"; import type { FromComparePerformanceViewMessage, ToComparePerformanceViewMessage, @@ -12,16 +21,27 @@ import { showAndLogExceptionWithTelemetry } from "../common/logging"; import { extLogger } from "../common/logging/vscode"; import type { WebviewPanelConfig } from "../common/vscode/abstract-webview"; import { AbstractWebview } from "../common/vscode/abstract-webview"; +import type { ProgressCallback } from "../common/vscode/progress"; +import { reportStreamProgress, withProgress } from "../common/vscode/progress"; import { telemetryListener } from "../common/vscode/telemetry"; -import type { HistoryItemLabelProvider } from "../query-history/history-item-label-provider"; -import { PerformanceOverviewScanner } from "../log-insights/performance-comparison"; -import { scanLog } from "../log-insights/log-scanner"; +import { downloadTimeout } from "../config"; import type { ResultsView } from "../local-queries"; +import { scanLog } from "../log-insights/log-scanner"; +import { PerformanceOverviewScanner } from "../log-insights/performance-comparison"; +import type { HistoryItemLabelProvider } from "../query-history/history-item-label-provider"; +import { tmpDir } from "../tmp-dir"; + +type ComparePerformanceCommands = { + "codeQL.compare-performance.downloadExternalLogs": () => Promise; +}; export class ComparePerformanceView extends AbstractWebview< ToComparePerformanceViewMessage, FromComparePerformanceViewMessage > { + private workingDirectory; + private LOG_DOWNLOAD_PROGRESS_STEPS = 3; + constructor( app: App, public cliServer: CodeQLCliServer, // TODO: make private @@ -30,6 +50,10 @@ export class ComparePerformanceView extends AbstractWebview< private resultsView: ResultsView, ) { super(app); + this.workingDirectory = path.join( + app.globalStoragePath, + "compare-performance", + ); } async showResults(fromJsonLog: string, toJsonLog: string) { @@ -94,4 +118,287 @@ export class ComparePerformanceView extends AbstractWebview< break; } } + + async downloadExternalLogs(): Promise { + const client = await this.app.credentials.getOctokit(); + async function getArtifactDownloadUrl( + url: string, + ): Promise<{ url: string; bytes: number; id: string }> { + const pattern = + /https:\/\/github.com\/([^/]+)\/([^/]+)\/actions\/runs\/([^/]+)\/artifacts\/([^/]+)/; + const match = url.match(pattern); + if (!match) { + throw new Error(`Invalid artifact URL: ${url}`); + } + const [, owner, repo, , artifact_id] = match; + const response = await client.request( + "HEAD /repos/{owner}/{repo}/actions/artifacts/{artifact_id}/{archive_format}", + { + owner, + repo, + artifact_id, + archive_format: "zip", + }, + ); + if (!response.headers["content-length"]) { + throw new Error( + `No content-length header found for artifact URL: ${url}`, + ); + } + return { + url: response.url, + bytes: response.headers["content-length"], + id: `artifacts/${owner}/${repo}/${artifact_id}`, + }; + } + + const downloadLog = async (originalUrl: string) => { + const { + url, + bytes, + id: artifactDiskId, + } = await getArtifactDownloadUrl(originalUrl); + const logPath = path.join( + this.workingDirectory, + `logs-of/${artifactDiskId}`, + ); + if (existsSync(logPath) && readdirSync(logPath).length > 0) { + void extLogger.log( + `Skipping log download and extraction to existing '${logPath}'...`, + ); + } + await withProgress( + async (progress) => { + const downloadPath = path.join(this.workingDirectory, artifactDiskId); + if ( + existsSync(downloadPath) && + readdirSync(downloadPath).length > 0 + ) { + void extLogger.log( + `Skipping download to existing '${artifactDiskId}'...`, + ); + } else { + await ensureDir(downloadPath); + void extLogger.log( + `Downloading from ${artifactDiskId} (bytes: ${bytes}) ${downloadPath}...`, + ); + await this.fetchAndUnzip(url, downloadPath, progress); + } + if (existsSync(logPath) && readdirSync(logPath).length >= 0) { + void extLogger.log( + `Skipping log extraction to existing '${logPath}'...`, + ); + } else { + await ensureDir(logPath); + // find the lone tar.gz file in the unzipped directory + const unzippedFiles = readdirSync(downloadPath); + const tarGzFiles = unzippedFiles.filter((f) => + f.endsWith(".tar.gz"), + ); + if (tarGzFiles.length !== 1) { + throw new Error( + `Expected exactly one .tar.gz file in the unzipped directory, but found: ${tarGzFiles.join( + ", ", + )}`, + ); + } + await this.untargz( + path.join(downloadPath, tarGzFiles[0]), + logPath, + progress, + ); + } + }, + { + title: `Downloading evaluator logs (${(bytes / 1024 / 1024).toFixed(1)} MB}`, + }, + ); + }; + // hardcoded URLs from https://github.com/codeql-dca-runners/codeql-dca-worker_javascript/actions/runs/11816721194 + const url1 = + "https://github.com/codeql-dca-runners/codeql-dca-worker_javascript/actions/runs/11816721194/artifacts/2181621080"; + const url2 = + "https://github.com/codeql-dca-runners/codeql-dca-worker_javascript/actions/runs/11816721194/artifacts/2181601861"; + + await Promise.all([downloadLog(url1), downloadLog(url2)]); + void extLogger.log(`Downloaded logs to ${this.workingDirectory}`); + + return; + } + + private async fetchAndUnzip( + contentUrl: string, + // (see below) requestHeaders: { [key: string]: string }, + unzipPath: string, + progress?: ProgressCallback, + ) { + // Although it is possible to download and stream directly to an unzipped directory, + // we need to avoid this for two reasons. The central directory is located at the + // end of the zip file. It is the source of truth of the content locations. Individual + // file headers may be incorrect. Additionally, saving to file first will reduce memory + // pressure compared with unzipping while downloading the archive. + + const archivePath = join(tmpDir.name, `archive-${Date.now()}.zip`); + + progress?.({ + maxStep: this.LOG_DOWNLOAD_PROGRESS_STEPS, + message: "Downloading content", + step: 1, + }); + + const { + signal, + onData, + dispose: disposeTimeout, + } = createTimeoutSignal(downloadTimeout()); + + let response: Response; + try { + response = await this.checkForFailingResponse( + await fetch(contentUrl, { + // XXX disabled header forwarding headers: requestHeaders, + signal, + }), + "Error downloading content", + ); + } catch (e) { + disposeTimeout(); + + if (e instanceof DOMException && e.name === "AbortError") { + const thrownError = new Error("The request timed out."); + thrownError.stack = e.stack; + throw thrownError; + } + + throw e; + } + + const body = response.body; + if (!body) { + throw new Error("No response body found"); + } + + const archiveFileStream = createWriteStream(archivePath); + + const contentLength = response.headers.get("content-length"); + const totalNumBytes = contentLength + ? parseInt(contentLength, 10) + : undefined; + + const reportProgress = reportStreamProgress( + "Downloading log", + totalNumBytes, + progress, + ); + + try { + const reader = body.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) { + break; + } + + onData(); + reportProgress(value?.length ?? 0); + + await new Promise((resolve, reject) => { + archiveFileStream.write(value, (err) => { + if (err) { + reject(err); + } + resolve(undefined); + }); + }); + } + + await new Promise((resolve, reject) => { + archiveFileStream.close((err) => { + if (err) { + reject(err); + } + resolve(undefined); + }); + }); + } catch (e) { + // Close and remove the file if an error occurs + archiveFileStream.close(() => { + void remove(archivePath); + }); + + if (e instanceof DOMException && e.name === "AbortError") { + const thrownError = new Error("The download timed out."); + thrownError.stack = e.stack; + throw thrownError; + } + + throw e; + } finally { + disposeTimeout(); + } + + await this.readAndUnzip( + Uri.file(archivePath).toString(true), + unzipPath, + progress, + ); + + // remove archivePath eagerly since these archives can be large. + await remove(archivePath); + } + + private async checkForFailingResponse( + response: Response, + errorMessage: string, + ): Promise { + if (response.ok) { + return response; + } + + // An error downloading the content. Attempt to extract the reason behind it. + const text = await response.text(); + let msg: string; + try { + const obj = JSON.parse(text); + msg = + obj.error || obj.message || obj.reason || JSON.stringify(obj, null, 2); + } catch { + msg = text; + } + throw new Error(`${errorMessage}.\n\nReason: ${msg}`); + } + + private async readAndUnzip( + zipUrl: string, + unzipPath: string, + progress?: ProgressCallback, + ) { + const zipFile = Uri.parse(zipUrl).fsPath; + progress?.({ + maxStep: this.LOG_DOWNLOAD_PROGRESS_STEPS, + step: 2, + message: `Unzipping into ${basename(unzipPath)}`, + }); + execFileSync("unzip", ["-q", "-d", unzipPath, zipFile]); + } + private async untargz( + tarballPath: string, + untarPath: string, + progress?: ProgressCallback, + ) { + progress?.({ + maxStep: this.LOG_DOWNLOAD_PROGRESS_STEPS, + step: 3, + message: `Untarring into ${basename(untarPath)}`, + }); + void extLogger.log(`Untarring ${tarballPath} into ${untarPath}`); + execFileSync("tar", ["-xzf", tarballPath, "-C", untarPath]); + } + + public getCommands(): ComparePerformanceCommands { + return { + "codeQL.compare-performance.downloadExternalLogs": + this.downloadExternalLogs.bind(this), + }; + } } diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index 53498287a46..ee90bcb979f 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -1076,6 +1076,7 @@ async function activateWithInstalledDistribution( ...testUiCommands, ...mockServer.getCommands(), ...debuggerUI.getCommands(), + ...comparePerformanceView.getCommands(), }; for (const [commandName, command] of Object.entries(allCommands)) { From 57c9949d8481f6edbc1b10f25f7ee997b161495f Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Thu, 14 Nov 2024 10:52:45 +0100 Subject: [PATCH 2/2] fixup! PoC download externally stored evaluator logs --- .../src/compare-performance/compare-performance-view.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extensions/ql-vscode/src/compare-performance/compare-performance-view.ts b/extensions/ql-vscode/src/compare-performance/compare-performance-view.ts index bcb6e12f0b7..3d88f8fe5b1 100644 --- a/extensions/ql-vscode/src/compare-performance/compare-performance-view.ts +++ b/extensions/ql-vscode/src/compare-performance/compare-performance-view.ts @@ -184,7 +184,7 @@ export class ComparePerformanceView extends AbstractWebview< ); await this.fetchAndUnzip(url, downloadPath, progress); } - if (existsSync(logPath) && readdirSync(logPath).length >= 0) { + if (existsSync(logPath) && readdirSync(logPath).length > 0) { void extLogger.log( `Skipping log extraction to existing '${logPath}'...`, ); @@ -226,6 +226,10 @@ export class ComparePerformanceView extends AbstractWebview< return; } + /** + * XXX Almost identical copy of the one in `database-fetcher.ts`. + * There ought to be a generic `downloadArtifactOrSimilar` + */ private async fetchAndUnzip( contentUrl: string, // (see below) requestHeaders: { [key: string]: string }, @@ -381,6 +385,7 @@ export class ComparePerformanceView extends AbstractWebview< }); execFileSync("unzip", ["-q", "-d", unzipPath, zipFile]); } + private async untargz( tarballPath: string, untarPath: string,