From 551a6aa5c0028631b24ea2adee45b8d2b921eb08 Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Thu, 18 Sep 2025 13:38:08 +0000 Subject: [PATCH] refactor: store screenshots as URLs This makes it easy for custom report fetchers and uploaders to store screenshots in e.g. Firebase Storage. --- .../src/app/pages/report-viewer/report-viewer.ts | 8 ++++---- runner/builder/builder-types.ts | 2 +- runner/builder/worker.ts | 2 +- runner/file-system-utils.ts | 2 +- runner/ratings/autoraters/rate-files.ts | 6 +++--- runner/ratings/autoraters/visuals-rater.ts | 11 +++++++---- .../built-in-ratings/visual-appearance-rating.ts | 4 ++-- runner/reporting/report-logging.ts | 15 +++++++++------ runner/utils/screenshots.ts | 7 +++++++ 9 files changed, 35 insertions(+), 22 deletions(-) create mode 100644 runner/utils/screenshots.ts diff --git a/report-app/src/app/pages/report-viewer/report-viewer.ts b/report-app/src/app/pages/report-viewer/report-viewer.ts index 82f3989..70ff5e8 100644 --- a/report-app/src/app/pages/report-viewer/report-viewer.ts +++ b/report-app/src/app/pages/report-viewer/report-viewer.ts @@ -236,10 +236,10 @@ export class ReportViewer { protected getScreenshotUrl(result: AssessmentResult): string | null { for (let i = result.attemptDetails.length - 1; i > -1; i--) { - const screenshot = result.attemptDetails[i].buildResult.screenshotBase64; - - if (screenshot) { - return `data:image/png;base64,${screenshot}`; + const screenshotUrl = + result.attemptDetails[i].buildResult.screenshotPngUrl; + if (screenshotUrl) { + return screenshotUrl; } } return null; diff --git a/runner/builder/builder-types.ts b/runner/builder/builder-types.ts index f95d343..463af75 100644 --- a/runner/builder/builder-types.ts +++ b/runner/builder/builder-types.ts @@ -56,7 +56,7 @@ export interface BuildResult { status: BuildResultStatus; message: string; errorType?: BuildErrorType; - screenshotBase64?: string; // Base64 encoded PNG screenshot + screenshotPngUrl?: string; missingDependency?: string; runtimeErrors?: string; /** JSON report from the Safety Web runner, if available. */ diff --git a/runner/builder/worker.ts b/runner/builder/worker.ts index 26bcd58..85e4ea2 100644 --- a/runner/builder/worker.ts +++ b/runner/builder/worker.ts @@ -156,7 +156,7 @@ process.on('message', async (message: BuildWorkerMessage) => { result = { status: BuildResultStatus.SUCCESS, message: 'Application built successfully!', - screenshotBase64: screenshotBase64Data, + screenshotPngUrl: `data:image/png;base64,${screenshotBase64Data}`, runtimeErrors: runtimeErrors.join('\n'), axeViolations, safetyWebReportJson, diff --git a/runner/file-system-utils.ts b/runner/file-system-utils.ts index 7e3bbc1..c15a04b 100644 --- a/runner/file-system-utils.ts +++ b/runner/file-system-utils.ts @@ -65,7 +65,7 @@ export async function removeFolderWithSymlinks(dir: string) { /** Write a file and creates the necessary directory structure. */ export async function safeWriteFile( filePath: string, - content: string, + content: string | Buffer, encoding?: BufferEncoding ): Promise { const directory = dirname(filePath); diff --git a/runner/ratings/autoraters/rate-files.ts b/runner/ratings/autoraters/rate-files.ts index 162758e..89b5836 100644 --- a/runner/ratings/autoraters/rate-files.ts +++ b/runner/ratings/autoraters/rate-files.ts @@ -21,7 +21,7 @@ export async function autoRateFiles( environment: Environment, files: LlmResponseFile[], appPrompt: string, - screenshotBase64: string | null + screenshotPngUrl: string | null ): Promise { console.log(`Autorater is using '${model}' model. \n`); @@ -39,7 +39,7 @@ export async function autoRateFiles( // Visual (screenshot) scoring... let visualRating = undefined; - if (screenshotBase64) { + if (screenshotPngUrl) { console.log('⏳ Awaiting visual scoring results...'); visualRating = await autoRateAppearance( llm, @@ -47,7 +47,7 @@ export async function autoRateFiles( model, environment, appPrompt, - screenshotBase64, + screenshotPngUrl, 'command-line' ); console.log(`${greenCheckmark()} Visual scoring is successful.`); diff --git a/runner/ratings/autoraters/visuals-rater.ts b/runner/ratings/autoraters/visuals-rater.ts index 3a0e8f2..9f6cd5d 100644 --- a/runner/ratings/autoraters/visuals-rater.ts +++ b/runner/ratings/autoraters/visuals-rater.ts @@ -8,6 +8,7 @@ import { import { GenkitRunner } from '../../codegen/genkit/genkit-runner.js'; import defaultVisualRaterPrompt from './visual-rating-prompt.js'; import { Environment } from '../../configuration/environment.js'; +import { screenshotUrlToPngBuffer } from '../../utils/screenshots.js'; /** * Automatically rate the appearance of a screenshot using an LLM. @@ -16,7 +17,7 @@ import { Environment } from '../../configuration/environment.js'; * @param model Model to use for the rating. * @param environment Environment in which the rating is running. * @param appPrompt Prompt to be used for the rating. - * @param screenshotBase64 Screenshot to be rated. + * @param screenshotPngUrl Screenshot PNG URL to be rated. * @param label Label for the rating, used for logging. */ export async function autoRateAppearance( @@ -25,7 +26,7 @@ export async function autoRateAppearance( model: string, environment: Environment, appPrompt: string, - screenshotBase64: string, + screenshotPngUrl: string, label: string ): Promise { const prompt = environment.renderPrompt(defaultVisualRaterPrompt, null, { @@ -38,8 +39,10 @@ export async function autoRateAppearance( content: [ { media: { - base64PngImage: screenshotBase64, - url: `data:image/png;base64,${screenshotBase64}`, + base64PngImage: ( + await screenshotUrlToPngBuffer(screenshotPngUrl) + ).toString('base64'), + url: screenshotPngUrl, }, }, ], diff --git a/runner/ratings/built-in-ratings/visual-appearance-rating.ts b/runner/ratings/built-in-ratings/visual-appearance-rating.ts index eb08061..0bea82d 100644 --- a/runner/ratings/built-in-ratings/visual-appearance-rating.ts +++ b/runner/ratings/built-in-ratings/visual-appearance-rating.ts @@ -20,7 +20,7 @@ export const visualAppearanceRating: LLMBasedRating = { id: 'common-autorater-visuals', model: DEFAULT_AUTORATER_MODEL_NAME, rate: async (ctx) => { - if (ctx.buildResult.screenshotBase64 === undefined) { + if (ctx.buildResult.screenshotPngUrl === undefined) { return { state: RatingState.SKIPPED, message: 'No screenshot available', @@ -36,7 +36,7 @@ export const visualAppearanceRating: LLMBasedRating = { ctx.model, ctx.environment, ctx.fullPromptText, - ctx.buildResult.screenshotBase64, + ctx.buildResult.screenshotPngUrl, ctx.currentPromptDef.name ); } catch (e) { diff --git a/runner/reporting/report-logging.ts b/runner/reporting/report-logging.ts index ff2a92e..cb36940 100644 --- a/runner/reporting/report-logging.ts +++ b/runner/reporting/report-logging.ts @@ -105,13 +105,16 @@ export async function writeReportToDisk( // Write screenshot to fs first, since we'll remove this info // from JSON later in this function. - if (attempt.buildResult.screenshotBase64) { + if (attempt.buildResult.screenshotPngUrl) { const screenshotFilePath = join(attemptPath, 'screenshot.png'); - await safeWriteFile( - screenshotFilePath, - attempt.buildResult.screenshotBase64, - 'base64' - ); + + // Note: In practice this is a base64 data URL, but `fetch` conveniently + // allows us to extract the content for writing a PNG to disk. + const screenshotContent = await ( + await fetch(attempt.buildResult.screenshotPngUrl) + ).arrayBuffer(); + + await safeWriteFile(screenshotFilePath, Buffer.from(screenshotContent)); } // Write the safety web report if it exists. diff --git a/runner/utils/screenshots.ts b/runner/utils/screenshots.ts new file mode 100644 index 0000000..f63252d --- /dev/null +++ b/runner/utils/screenshots.ts @@ -0,0 +1,7 @@ +/** Converts a screenshot PNG URL to a PNG buffer with the image contents. */ +export async function screenshotUrlToPngBuffer(screenshotPngUrl: string) { + // Note: In practice this is a base64 data URL, but `fetch` conveniently + // allows us to extract the content for writing a PNG to disk. + const screenshotContent = await (await fetch(screenshotPngUrl)).arrayBuffer(); + return Buffer.from(screenshotContent); +}