Improving OpenAI image handling

Author: GTFalcao

components/llmwhisperer/actions/extract-text/extract-text.mjs

@@ -99,7 +99,9 @@ export default {
   },
   methods: {
     getHeaders() {
-      return "application/octet-stream";
+      return {
+        "Content-Type": "application/octet-stream",
+      };
     },
     async getData(data) {
       return getFileStream(data);

components/openai/actions/chat/chat.mjs

@@ -43,7 +43,7 @@ export default {
     images: {
       label: "Images",
       type: "string[]",
-      description: "Provide one or more images to [OpenAI's vision model](https://platform.openai.com/docs/guides/vision). Accepts URLs or base64 encoded strings. Compatible with the `gpt4-vision-preview` model",
+      description: "Provide one or more images to [OpenAI's vision model](https://platform.openai.com/docs/guides/vision). Each entry should be either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`) Compatible with the `gpt4-vision-preview` model",
       optional: true,
     },
     audio: {

components/openai/actions/common/common.mjs

@@ -85,11 +85,15 @@ export default {
       let content = [];
       if (this.images) {
         for (const image of this.images) {
+          const stream = await getFileStream(image);
+          const chunks = [];
+          for await (const chunk of stream) {
+            chunks.push(chunk);
+          }
+          const base64Image = Buffer.concat(chunks).toString("base64");
           content.push({
-            "type": "image_url",
-            "image_url": {
-              "url": image,
-            },
+            "type": "input_image",
+            "image_url": `data:image/jpeg;base64,${base64Image}`,
           });
         }
       }