adding samples, test, lints

Author: Guiners

genai/bounding-box/boundingbox-with-txt-img.js

@@ -14,131 +14,144 @@
 
 'use strict';
 
-// [START googlegenaisdk_embeddings_docretrieval_with_txt]
+// [START googlegenaisdk_boundingbox_with_txt_img]
 const {GoogleGenAI} = require('@google/genai');
 
+const {createCanvas, loadImage} = require('canvas');
+const fetch = require('node-fetch');
+const fs = require('fs');
+
 const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT;
 const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global';
-//todo notworking
+
+async function fetchImageAsBase64(uri) {
+  const response = await fetch(uri);
+  const buffer = await response.buffer();
+  return buffer.toString('base64');
+}
+
+async function plotBoundingBoxes(imageUri, boundingBoxes) {
+  console.log('Creating bounding boxes');
+  const image = await loadImage(imageUri);
+  const canvas = createCanvas(image.width, image.height);
+  const ctx = canvas.getContext('2d');
+
+  ctx.drawImage(image, 0, 0);
+
+  const colors = ['red', 'blue', 'green', 'orange'];
+
+  boundingBoxes.forEach((bbox, i) => {
+    const [yMin, xMin, yMax, xMax] = bbox.box_2d;
+
+    const absYMin = Math.floor((yMin / 1000) * image.height);
+    const absXMin = Math.floor((xMin / 1000) * image.width);
+    const absYMax = Math.floor((yMax / 1000) * image.height);
+    const absXMax = Math.floor((xMax / 1000) * image.width);
+
+    ctx.strokeStyle = colors[i % colors.length];
+    ctx.lineWidth = 4;
+    ctx.strokeRect(absXMin, absYMin, absXMax - absXMin, absYMax - absYMin);
+
+    ctx.fillStyle = colors[i % colors.length];
+    ctx.font = '20px Arial';
+    ctx.fillText(bbox.label, absXMin + 8, absYMin + 20);
+  });
+
+  fs.writeFileSync('output.png', canvas.toBuffer('image/png'));
+  console.log('Saved output to file: output.png');
+}
+
 async function generateContent(
   projectId = GOOGLE_CLOUD_PROJECT,
   location = GOOGLE_CLOUD_LOCATION
 ) {
-
-  /**
-   * Represents a bounding box with its 2D coordinates and associated label.
-   */
-  class BoundingBox {
-    /**
-     * @param {number[]} box2d - A list of integers representing the 2D coordinates of the bounding box
-     *                           in the format [y_min, x_min, y_max, x_max].
-     * @param {string} label - The label or class associated with the object in the bounding box.
-     */
-    constructor(box2d, label) {
-      if (!Array.isArray(box2d) || box2d.length !== 4 || !box2d.every(Number.isInteger)) {
-        throw new Error('box2d must be an array of 4 integers');
-      }
-      if (typeof label !== 'string') {
-        throw new Error('label must be a string');
-      }
-      this.box2d = box2d;
-      this.label = label;
-    }
-  }
-
-  /**
-   * Helper function to plot bounding boxes on an image
-   * @param {string} imageUri
-   * @param {BoundingBox[]} boundingBoxes
-   */
-  async function plotBoundingBoxes(imageUri, boundingBoxes) {
-    const image = await loadImage(imageUri);
-    const width = image.width;
-    const height = image.height;
-
-    const canvas = createCanvas(width, height);
-    const ctx = canvas.getContext('2d');
-
-    ctx.drawImage(image, 0, 0, width, height);
-
-    const colors = ['red', 'green', 'blue', 'orange', 'purple', 'yellow', 'cyan', 'magenta', 'lime', 'pink'];
-
-    boundingBoxes.forEach((bbox, i) => {
-      const absYMin = Math.floor((bbox.box2d[0] / 1000) * height);
-      const absXMin = Math.floor((bbox.box2d[1] / 1000) * width);
-      const absYMax = Math.floor((bbox.box2d[2] / 1000) * height);
-      const absXMax = Math.floor((bbox.box2d[3] / 1000) * width);
-
-      const color = colors[i % colors.length];
-
-      ctx.strokeStyle = color;
-      ctx.lineWidth = 4;
-      ctx.strokeRect(absXMin, absYMin, absXMax - absXMin, absYMax - absYMin);
-
-      if (bbox.label) {
-        ctx.fillStyle = color;
-        ctx.font = '20px Arial';
-        ctx.fillText(bbox.label, absXMin + 8, absYMin + 20);
-      }
-    });
-
-    // Save or return buffer
-    return canvas.toBuffer('image/png');
-  }
-
   const ai = new GoogleGenAI({
     vertexai: true,
     project: projectId,
     location: location,
   });
 
-  const systemInstructions = 'Return bounding boxes as an array with labels.\n' +
-    '        Never return masks. Limit to 25 objects.\n' +
-    '        If an object is present multiple times, give each object a unique label\n' +
-    '        according to its distinct characteristics (colors, size, position, etc..).'
-
-  const imageUri = "https://storage.googleapis.com/generativeai-downloads/images/socks.jpg";
-
-  const prompt = [
-    { file_uri: imageUri, mime_type: 'image/jpeg' }, // zamiast Part.fromUri
-    "Output the positions of the socks with a face. Label according to position in the image."
+  const systemInstruction = `
+    Return bounding boxes as an array with labels.
+    Never return masks. Limit to 25 objects.
+    If an object is present multiple times, give each object a unique label
+    according to its distinct characteristics (colors, size, position, etc.).
+  `;
+
+  const safetySettings = [
+    {
+      category: 'HARM_CATEGORY_DANGEROUS_CONTENT',
+      threshold: 'BLOCK_ONLY_HIGH',
+    },
   ];
 
-  const config = {
-    systemInstructions: systemInstructions,
-    temperature: 0.5,
-    safetySettings: [
-      {
-        category: 'HARM_CATEGORY_DANGEROUS_CONTENT',
-        threshold: 'BLOCK_ONLY_HIGH',
+  const imageUri =
+    'https://storage.googleapis.com/generativeai-downloads/images/socks.jpg';
+  const base64Image = await fetchImageAsBase64(imageUri);
+
+  const boundingBoxSchema = {
+    type: 'ARRAY',
+    description: 'List of bounding boxes for detected objects',
+    items: {
+      type: 'OBJECT',
+      title: 'BoundingBox',
+      description: 'Represents a bounding box with coordinates and label',
+      properties: {
+        box_2d: {
+          type: 'ARRAY',
+          description:
+            'Bounding box coordinates in format [y_min, x_min, y_max, x_max]',
+          items: {
+            type: 'INTEGER',
+            format: 'int32',
+          },
+          minItems: '4',
+          maxItems: '4',
+        },
+        label: {
+          type: 'STRING',
+          description: 'Label describing the object within the bounding box',
+        },
       },
-    ],
-    responseMimeType: 'application/json'
+      required: ['box_2d', 'label'],
+    },
   };
 
-
   const response = await ai.models.generateContent({
     model: 'gemini-2.5-flash',
-    contents: prompt,
-    config: config
+    contents: [
+      {
+        role: 'user',
+        parts: [
+          {
+            text: 'Output the positions of the socks with a face. Label according to position in the image.',
+          },
+          {
+            inlineData: {
+              data: base64Image,
+              mimeType: 'image/jpeg',
+            },
+          },
+        ],
+      },
+    ],
+    config: {
+      systemInstruction: systemInstruction,
+      safetySettings: safetySettings,
+      responseMimeType: 'application/json',
+      temperature: 0.5,
+      responseSchema: boundingBoxSchema,
+    },
   });
 
-  console.log(response.text);
+  const candidate = response.candidates[0].content.parts[0].text;
+  const boundingBoxes = JSON.parse(candidate);
 
-  let boundingBoxes = [];
-  try {
-    boundingBoxes = JSON.parse(response.text).map(b => new BoundingBox(b.box_2d, b.label));
-  } catch (err) {
-    console.error('Failed to parse response:', err);
-  }
+  console.log('Bounding boxes:', boundingBoxes);
 
   await plotBoundingBoxes(imageUri, boundingBoxes);
-
-
-  return response;
+  return boundingBoxes;
 }
-// [END googlegenaisdk_embeddings_docretrieval_with_txt]
+// [END googlegenaisdk_boundingbox_with_txt_img]
 
-module.exports = {
-  generateContent,
-};
+module.exports = {generateContent};

genai/package.json

@@ -23,6 +23,8 @@
     "mocha": "^10.0.0",
     "sinon": "^18.0.0",
     "uuid": "^10.0.0",
-    "proxyquire": "^2.1.3"
+    "proxyquire": "^2.1.3",
+    "canvas": "^3.1.0",
+    "node-fetch": "^2.7.0"
   }
 }

genai/test/boundingbox-with-txt-img.test.js

@@ -21,9 +21,9 @@ const projectId = process.env.CAIP_PROJECT_ID;
 const sample = require('../bounding-box/boundingbox-with-txt-img');
 
 describe('boundingbox-with-txt-img', async () => {
-  it('should return the total token count for a text prompt', async function (){
-    this.timeout(10000);
+  it('should return the bounding box', async function () {
+    this.timeout(100000);
     const output = await sample.generateContent(projectId);
-    assert(output > 0);
+    assert(output.length > 0);
   });
 });