adding samples, test, lints

Author: Guiners

genai/bounding-box/boundingbox-with-txt-img.js

@@ -19,13 +19,12 @@ const {GoogleGenAI} = require('@google/genai');
 
 const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT;
 const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global';
-
+//todo notworking
 async function generateContent(
   projectId = GOOGLE_CLOUD_PROJECT,
   location = GOOGLE_CLOUD_LOCATION
 ) {
 
-
   /**
    * Represents a bounding box with its 2D coordinates and associated label.
    */
@@ -47,24 +46,94 @@ async function generateContent(
     }
   }
 
+  /**
+   * Helper function to plot bounding boxes on an image
+   * @param {string} imageUri
+   * @param {BoundingBox[]} boundingBoxes
+   */
+  async function plotBoundingBoxes(imageUri, boundingBoxes) {
+    const image = await loadImage(imageUri);
+    const width = image.width;
+    const height = image.height;
+
+    const canvas = createCanvas(width, height);
+    const ctx = canvas.getContext('2d');
+
+    ctx.drawImage(image, 0, 0, width, height);
+
+    const colors = ['red', 'green', 'blue', 'orange', 'purple', 'yellow', 'cyan', 'magenta', 'lime', 'pink'];
+
+    boundingBoxes.forEach((bbox, i) => {
+      const absYMin = Math.floor((bbox.box2d[0] / 1000) * height);
+      const absXMin = Math.floor((bbox.box2d[1] / 1000) * width);
+      const absYMax = Math.floor((bbox.box2d[2] / 1000) * height);
+      const absXMax = Math.floor((bbox.box2d[3] / 1000) * width);
+
+      const color = colors[i % colors.length];
+
+      ctx.strokeStyle = color;
+      ctx.lineWidth = 4;
+      ctx.strokeRect(absXMin, absYMin, absXMax - absXMin, absYMax - absYMin);
+
+      if (bbox.label) {
+        ctx.fillStyle = color;
+        ctx.font = '20px Arial';
+        ctx.fillText(bbox.label, absXMin + 8, absYMin + 20);
+      }
+    });
+
+    // Save or return buffer
+    return canvas.toBuffer('image/png');
+  }
+
   const ai = new GoogleGenAI({
     vertexai: true,
     project: projectId,
     location: location,
   });
 
+  const systemInstructions = 'Return bounding boxes as an array with labels.\n' +
+    '        Never return masks. Limit to 25 objects.\n' +
+    '        If an object is present multiple times, give each object a unique label\n' +
+    '        according to its distinct characteristics (colors, size, position, etc..).'
+
+  const imageUri = "https://storage.googleapis.com/generativeai-downloads/images/socks.jpg";
+
+  const prompt = [
+    { file_uri: imageUri, mime_type: 'image/jpeg' }, // zamiast Part.fromUri
+    "Output the positions of the socks with a face. Label according to position in the image."
+  ];
+
+  const config = {
+    systemInstructions: systemInstructions,
+    temperature: 0.5,
+    safetySettings: [
+      {
+        category: 'HARM_CATEGORY_DANGEROUS_CONTENT',
+        threshold: 'BLOCK_ONLY_HIGH',
+      },
+    ],
+    responseMimeType: 'application/json'
+  };
 
-  const response = await ai.models.embedContent({
-    model: 'gemini-embedding-001',
+
+  const response = await ai.models.generateContent({
+    model: 'gemini-2.5-flash',
     contents: prompt,
-    config: {
-      taskType: 'RETRIEVAL_DOCUMENT', // Optional
-      outputDimensionality: 3072,  // Optional
-      title: "Driver's License"  // Optional
-    }
+    config: config
   });
 
-  console.log(response);
+  console.log(response.text);
+
+  let boundingBoxes = [];
+  try {
+    boundingBoxes = JSON.parse(response.text).map(b => new BoundingBox(b.box_2d, b.label));
+  } catch (err) {
+    console.error('Failed to parse response:', err);
+  }
+
+  await plotBoundingBoxes(imageUri, boundingBoxes);
+
 
   return response;
 }

genai/test/boundingbox-with-txt-img.test.js

@@ -21,7 +21,8 @@ const projectId = process.env.CAIP_PROJECT_ID;
 const sample = require('../bounding-box/boundingbox-with-txt-img');
 
 describe('boundingbox-with-txt-img', async () => {
-  it('should return the total token count for a text prompt', async () => {
+  it('should return the total token count for a text prompt', async function (){
+    this.timeout(10000);
     const output = await sample.generateContent(projectId);
     assert(output > 0);
   });

package.json

@@ -31,8 +31,10 @@
     "typescript": "^5.0.4"
   },
   "dependencies": {
+    "canvas": "^3.1.2",
     "commander": "^12.0.0",
     "eslint": "^8.57.0",
+    "node-fetch": "^3.3.2",
     "proxyquire": "^2.1.3"
   }
 }