Merge remote-tracking branch 'origin/master' into issue-14432

Author: michelle0927

components/openai/actions/chat/chat.mjs

@@ -1,6 +1,7 @@
 import openai from "../../openai.app.mjs";
 import common from "../common/common.mjs";
 import constants from "../../common/constants.mjs";
+import { ConfigurationError } from "@pipedream/platform";
 
 export default {
   ...common,
@@ -38,7 +39,13 @@ export default {
     images: {
       label: "Images",
       type: "string[]",
-      description: "Provide one or more images to [OpenAI's vision model](https://platform.openai.com/docs/guides/vision). Accepts URLs or base64 encoded strings. Compatible with the `gpt4-vision-preview model`",
+      description: "Provide one or more images to [OpenAI's vision model](https://platform.openai.com/docs/guides/vision). Accepts URLs or base64 encoded strings. Compatible with the `gpt4-vision-preview` model",
+      optional: true,
+    },
+    audio: {
+      type: "string",
+      label: "Audio",
+      description: "Provide the file path to an audio file in the `/tmp` directory. For use with the `gpt-4o-audio-preview` model. Currently supports `wav` and `mp3` files.",
       optional: true,
     },
     responseFormat: {
@@ -65,6 +72,10 @@ export default {
     };
   },
   async run({ $ }) {
+    if (this.audio && !this.modelId.includes("gpt-4o-audio-preview")) {
+      throw new ConfigurationError("Use of audio files requires using the `gpt-4o-audio-preview` model.");
+    }
+
     const args = this._getChatArgs();
 
     const response = await this.openai.createChatCompletion({

components/openai/actions/common/common.mjs

@@ -1,6 +1,7 @@
 import { ConfigurationError } from "@pipedream/platform";
 import constants from "../../common/constants.mjs";
 import { parse } from "../../common/helpers.mjs";
+import fs from "fs";
 
 const CHAT_DOCS_MESSAGE_FORMAT_URL = "https://platform.openai.com/docs/guides/chat/introduction";
 
@@ -92,6 +93,20 @@ export default {
         }
       }
 
+      if (this.audio) {
+        const fileContent = fs.readFileSync(this.audio.includes("tmp/")
+          ? this.audio
+          : `/tmp/${this.audio}`).toString("base64");
+        const extension = this.audio.match(/\.(\w+)$/)?.[1];
+        content.push({
+          type: "input_audio",
+          input_audio: {
+            data: fileContent,
+            format: extension,
+          },
+        });
+      }
+
       content.push({
         "type": "text",
         "text": this.userMessage,