accept audio input

Author: michelle0927

components/openai/actions/chat/chat.mjs

@@ -5,7 +5,7 @@ import constants from "../../common/constants.mjs";
 export default {
   ...common,
   name: "Chat",
-  version: "0.2.0",
+  version: "0.2.1",
   key: "openai-chat",
   description: "The Chat API, using the `gpt-3.5-turbo` or `gpt-4` model. [See the documentation](https://platform.openai.com/docs/api-reference/chat)",
   type: "action",
@@ -38,7 +38,13 @@ export default {
     images: {
       label: "Images",
       type: "string[]",
-      description: "Provide one or more images to [OpenAI's vision model](https://platform.openai.com/docs/guides/vision). Accepts URLs or base64 encoded strings. Compatible with the `gpt4-vision-preview model`",
+      description: "Provide one or more images to [OpenAI's vision model](https://platform.openai.com/docs/guides/vision). Accepts URLs or base64 encoded strings. Compatible with the `gpt4-vision-preview` model",
+      optional: true,
+    },
+    audio: {
+      type: "string",
+      label: "Audio",
+      description: "Provide the file path to an audio file in the `/tmp` directory. Compatible with the `gpt-4o-audio-preview` model. Currently supports `wav` and `mp3` files.",
       optional: true,
     },
     responseFormat: {

components/openai/actions/common/common.mjs

@@ -1,6 +1,7 @@
 import { ConfigurationError } from "@pipedream/platform";
 import constants from "../../common/constants.mjs";
 import { parse } from "../../common/helpers.mjs";
+import fs from "fs";
 
 const CHAT_DOCS_MESSAGE_FORMAT_URL = "https://platform.openai.com/docs/guides/chat/introduction";
 
@@ -92,6 +93,20 @@ export default {
         }
       }
 
+      if (this.audio) {
+        const fileContent = fs.readFileSync(this.audio.includes("tmp/")
+          ? this.audio
+          : `/tmp/${this.audio}`).toString("base64");
+        const extension = this.audio.match(/\.(\w+)$/)?.[1];
+        content.push({
+          type: "input_audio",
+          input_audio: {
+            data: fileContent,
+            format: extension,
+          },
+        });
+      }
+
       content.push({
         "type": "text",
         "text": this.userMessage,

components/openai/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/openai",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "Pipedream OpenAI Components",
   "main": "openai.app.mjs",
   "keywords": [