support video on bedrock

Author: narengogi

src/globals.ts

@@ -215,6 +215,8 @@ export const fileExtensionMimeTypeMap = {
   mpegps: 'video/mpegps',
   flv: 'video/flv',
   webm: 'video/webm',
+  mkv: 'video/mkv',
+  three_gpp: 'video/3gpp',
 };
 
 export const imagesMimeTypes = [
@@ -238,6 +240,19 @@ export const documentMimeTypes = [
   fileExtensionMimeTypeMap.txt,
 ];
 
+export const videoMimeTypes = [
+  fileExtensionMimeTypeMap.mkv,
+  fileExtensionMimeTypeMap.mov,
+  fileExtensionMimeTypeMap.mp4,
+  fileExtensionMimeTypeMap.webm,
+  fileExtensionMimeTypeMap.flv,
+  fileExtensionMimeTypeMap.mpeg,
+  fileExtensionMimeTypeMap.mpg,
+  fileExtensionMimeTypeMap.wmv,
+  fileExtensionMimeTypeMap.three_gpp,
+  fileExtensionMimeTypeMap.avi,
+];
+
 export enum BatchEndpoints {
   CHAT_COMPLETIONS = '/v1/chat/completions',
   COMPLETIONS = '/v1/completions',

src/providers/bedrock/chatComplete.ts

@@ -3,6 +3,7 @@ import {
   documentMimeTypes,
   fileExtensionMimeTypeMap,
   imagesMimeTypes,
+  videoMimeTypes,
 } from '../../globals';
 import {
   Message,
@@ -190,7 +191,16 @@ const getMessageContent = (message: Message) => {
               format: fileFormat,
             },
           });
-        } else if (documentMimeTypes.includes(mimeType)) {
+        } else if (videoMimeTypes.includes(mimeType)) {
+          out.push({
+            video: {
+              format: fileFormat,
+              source: {
+                bytes,
+              },
+            },
+          });
+        } else {
           out.push({
             document: {
               format: fileFormat,
@@ -204,25 +214,46 @@ const getMessageContent = (message: Message) => {
       } else if (item.type === 'file') {
         const mimeType = item.file?.mime_type || fileExtensionMimeTypeMap.pdf;
         const fileFormat = mimeType.split('/')[1];
-        if (item.file?.file_url) {
+        if (imagesMimeTypes.includes(mimeType)) {
           out.push({
-            document: {
+            image: {
+              source: {
+                ...(item.file?.file_data && { bytes: item.file.file_data }),
+                ...(item.file?.file_url && {
+                  s3Location: {
+                    uri: item.file.file_url,
+                  },
+                }),
+              },
+              format: fileFormat,
+            },
+          });
+        } else if (videoMimeTypes.includes(mimeType)) {
+          out.push({
+            video: {
               format: fileFormat,
-              name: item.file.file_name || crypto.randomUUID(),
               source: {
-                s3Location: {
-                  uri: item.file.file_url,
-                },
+                ...(item.file?.file_data && { bytes: item.file.file_data }),
+                ...(item.file?.file_url && {
+                  s3Location: {
+                    uri: item.file.file_url,
+                  },
+                }),
               },
             },
           });
-        } else if (item.file?.file_data) {
+        } else {
           out.push({
             document: {
               format: fileFormat,
-              name: item.file.file_name || crypto.randomUUID(),
+              name: item.file?.file_name || crypto.randomUUID(),
               source: {
-                bytes: item.file.file_data,
+                ...(item.file?.file_data && { bytes: item.file.file_data }),
+                ...(item.file?.file_url && {
+                  s3Location: {
+                    uri: item.file.file_url,
+                  },
+                }),
               },
             },
           });

src/providers/bedrock/types.ts

@@ -136,7 +136,11 @@ export type BedrockContentItem = {
   };
   image?: {
     source: {
-      bytes: string;
+      bytes?: string;
+      s3Location?: {
+        uri: string;
+        bucketOwner?: string;
+      };
     };
     format: string;
   };
@@ -147,6 +151,17 @@ export type BedrockContentItem = {
       bytes?: string;
       s3Location?: {
         uri: string;
+        bucketOwner?: string;
+      };
+    };
+  };
+  video?: {
+    format: string;
+    source: {
+      bytes?: string;
+      s3Location?: {
+        uri: string;
+        bucketOwner?: string;
       };
     };
   };