fix: detect actual image format in take_screenshot tool

Fixes #583

The take_screenshot tool was incorrectly declaring the MIME type based
on the requested format parameter rather than detecting the actual
format of the image data returned by Puppeteer.

This caused a mismatch when Puppeteer returned a different format than
requested, leading to Claude API validation errors like:
"Image does not match the provided media type image/jpeg"

Changes:
- Add detectImageFormat() utility to identify actual image format from
  binary data by inspecting magic numbers (PNG, JPEG, WebP)
- Update screenshot tool to detect and use the actual format instead
  of blindly trusting the request parameter
- Add comprehensive unit tests for format detection

Co-Authored-By: Claude <[email protected]>

Author: marcandreuf

src/tools/screenshot.ts

@@ -7,6 +7,7 @@
 import {zod} from '../third_party/index.js';
 import type {ElementHandle, Page} from '../third_party/index.js';
 
+import {detectImageFormat} from '../utils/imageFormat.js';
 import {ToolCategory} from './categories.js';
 import {defineTool} from './ToolDefinition.js';
 
@@ -83,18 +84,22 @@ export const screenshot = defineTool({
       );
     }
 
+    // Detect the actual format of the screenshot data
+    // Puppeteer may not always return the requested format
+    const actualFormat = detectImageFormat(screenshot);
+
     if (request.params.filePath) {
       const file = await context.saveFile(screenshot, request.params.filePath);
       response.appendResponseLine(`Saved screenshot to ${file.filename}.`);
     } else if (screenshot.length >= 2_000_000) {
       const {filename} = await context.saveTemporaryFile(
         screenshot,
-        `image/${request.params.format}`,
+        actualFormat,
       );
       response.appendResponseLine(`Saved screenshot to ${filename}.`);
     } else {
       response.attachImage({
-        mimeType: `image/${request.params.format}`,
+        mimeType: actualFormat,
         data: Buffer.from(screenshot).toString('base64'),
       });
     }

src/utils/imageFormat.ts

@@ -0,0 +1,55 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Detects the actual image format from binary data by inspecting magic numbers.
+ *
+ * @param data - The image data as a Uint8Array or Buffer
+ * @returns The detected MIME type ('image/png', 'image/jpeg', or 'image/webp')
+ * @throws Error if the format cannot be detected
+ */
+export function detectImageFormat(
+  data: Uint8Array | Buffer,
+): 'image/png' | 'image/jpeg' | 'image/webp' {
+  if (data.length < 12) {
+    throw new Error('Image data too small to detect format');
+  }
+
+  // Check PNG: starts with 89 50 4E 47 (‰PNG)
+  if (
+    data[0] === 0x89 &&
+    data[1] === 0x50 &&
+    data[2] === 0x4e &&
+    data[3] === 0x47
+  ) {
+    return 'image/png';
+  }
+
+  // Check JPEG: starts with FF D8 FF
+  if (data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) {
+    return 'image/jpeg';
+  }
+
+  // Check WebP: starts with "RIFF" and contains "WEBP" at offset 8
+  if (
+    data[0] === 0x52 && // R
+    data[1] === 0x49 && // I
+    data[2] === 0x46 && // F
+    data[3] === 0x46 && // F
+    data[8] === 0x57 && // W
+    data[9] === 0x45 && // E
+    data[10] === 0x42 && // B
+    data[11] === 0x50 // P
+  ) {
+    return 'image/webp';
+  }
+
+  throw new Error(
+    `Unable to detect image format. First bytes: ${Array.from(data.slice(0, 12))
+      .map(b => b.toString(16).padStart(2, '0'))
+      .join(' ')}`,
+  );
+}

tests/utils/imageFormat.test.ts

@@ -0,0 +1,77 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import assert from 'node:assert';
+import {describe, it} from 'node:test';
+
+import {detectImageFormat} from '../../src/utils/imageFormat.js';
+
+describe('imageFormat', () => {
+  describe('detectImageFormat', () => {
+    it('detects PNG format', () => {
+      // PNG magic number: 89 50 4E 47 0D 0A 1A 0A
+      const pngData = new Uint8Array([
+        0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
+      ]);
+      const format = detectImageFormat(pngData);
+      assert.equal(format, 'image/png');
+    });
+
+    it('detects JPEG format', () => {
+      // JPEG magic number: FF D8 FF
+      const jpegData = new Uint8Array([
+        0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01,
+      ]);
+      const format = detectImageFormat(jpegData);
+      assert.equal(format, 'image/jpeg');
+    });
+
+    it('detects WebP format', () => {
+      // WebP magic number: RIFF ... WEBP
+      const webpData = new Uint8Array([
+        0x52,
+        0x49,
+        0x46,
+        0x46, // RIFF
+        0x00,
+        0x00,
+        0x00,
+        0x00, // file size (placeholder)
+        0x57,
+        0x45,
+        0x42,
+        0x50, // WEBP
+      ]);
+      const format = detectImageFormat(webpData);
+      assert.equal(format, 'image/webp');
+    });
+
+    it('throws error for data that is too small', () => {
+      const smallData = new Uint8Array([0x89, 0x50]);
+      assert.throws(
+        () => detectImageFormat(smallData),
+        /Image data too small to detect format/,
+      );
+    });
+
+    it('throws error for unknown format', () => {
+      const unknownData = new Uint8Array([
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      ]);
+      assert.throws(
+        () => detectImageFormat(unknownData),
+        /Unable to detect image format/,
+      );
+    });
+
+    it('works with Buffer objects', () => {
+      const pngBuffer = Buffer.from([
+        0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
+      ]);
+      const format = detectImageFormat(pngBuffer);
+      assert.equal(format, 'image/png');
+    });
+  });
+});