Skip to content

feat: enable video uploads for Gemini 2.5 Pro models #6145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
roomote wants to merge 1 commit into from
Closed

feat: enable video uploads for Gemini 2.5 Pro models #6145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/types/src/model.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ export const modelInfoSchema = z.object({
maxThinkingTokens: z.number().nullish(),
contextWindow: z.number(),
supportsImages: z.boolean().optional(),
supportsVideo: z.boolean().optional(),
supportsComputerUse: z.boolean().optional(),
supportsPromptCache: z.boolean(),
supportsReasoningBudget: z.boolean().optional(),
Expand Down
6 changes: 6 additions & 0 deletions packages/types/src/providers/gemini.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ export const geminiModels = {
maxTokens: 64_000,
contextWindow: 1_048_576,
supportsImages: true,
supportsVideo: true,
supportsPromptCache: true,
inputPrice: 0.3,
outputPrice: 2.5,
Expand All @@ -64,6 +65,7 @@ export const geminiModels = {
maxTokens: 65_535,
contextWindow: 1_048_576,
supportsImages: true,
supportsVideo: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
Expand All @@ -72,6 +74,7 @@ export const geminiModels = {
maxTokens: 65_535,
contextWindow: 1_048_576,
supportsImages: true,
supportsVideo: true,
supportsPromptCache: true,
inputPrice: 2.5, // This is the pricing for prompts above 200k tokens.
outputPrice: 15,
Expand All @@ -96,6 +99,7 @@ export const geminiModels = {
maxTokens: 65_535,
contextWindow: 1_048_576,
supportsImages: true,
supportsVideo: true,
supportsPromptCache: true,
inputPrice: 2.5, // This is the pricing for prompts above 200k tokens.
outputPrice: 15,
Expand All @@ -120,6 +124,7 @@ export const geminiModels = {
maxTokens: 65_535,
contextWindow: 1_048_576,
supportsImages: true,
supportsVideo: true,
supportsPromptCache: true,
inputPrice: 2.5, // This is the pricing for prompts above 200k tokens.
outputPrice: 15,
Expand All @@ -146,6 +151,7 @@ export const geminiModels = {
maxTokens: 64_000,
contextWindow: 1_048_576,
supportsImages: true,
supportsVideo: true,
supportsPromptCache: true,
inputPrice: 2.5, // This is the pricing for prompts above 200k tokens.
outputPrice: 15,
Expand Down
41 changes: 26 additions & 15 deletions webview-ui/src/components/chat/ChatTextArea.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ interface ChatTextAreaProps {
onSend: () => void
onSelectImages: () => void
shouldDisableImages: boolean
supportsVideo?: boolean
onHeightChange?: (height: number) => void
mode: Mode
setMode: (value: Mode) => void
Expand All @@ -64,6 +65,7 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
onSend,
onSelectImages,
shouldDisableImages,
supportsVideo = false,
onHeightChange,
mode,
setMode,
Expand Down Expand Up @@ -598,17 +600,21 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
return
}

const acceptedTypes = ["png", "jpeg", "webp"]
const acceptedImageTypes = ["png", "jpeg", "webp"]
const acceptedVideoTypes = supportsVideo ? ["mp4", "mov", "avi", "webm"] : []

const imageItems = Array.from(items).filter((item) => {
const mediaItems = Array.from(items).filter((item) => {
const [type, subtype] = item.type.split("/")
return type === "image" && acceptedTypes.includes(subtype)
return (
(type === "image" && acceptedImageTypes.includes(subtype)) ||
(type === "video" && acceptedVideoTypes.includes(subtype))
)
})

if (!shouldDisableImages && imageItems.length > 0) {
if (!shouldDisableImages && mediaItems.length > 0) {
e.preventDefault()

const imagePromises = imageItems.map((item) => {
const mediaPromises = mediaItems.map((item) => {
return new Promise<string | null>((resolve) => {
const blob = item.getAsFile()

Expand All @@ -633,8 +639,8 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
})
})

const imageDataArray = await Promise.all(imagePromises)
const dataUrls = imageDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null)
const mediaDataArray = await Promise.all(mediaPromises)
const dataUrls = mediaDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null)

if (dataUrls.length > 0) {
setSelectedImages((prevImages) => [...prevImages, ...dataUrls].slice(0, MAX_IMAGES_PER_MESSAGE))
Expand All @@ -643,7 +649,7 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
}
}
},
[shouldDisableImages, setSelectedImages, cursorPosition, setInputValue, inputValue, t],
[shouldDisableImages, setSelectedImages, cursorPosition, setInputValue, inputValue, t, supportsVideo],
)

const handleMenuMouseDown = useCallback(() => {
Expand Down Expand Up @@ -732,15 +738,19 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
const files = Array.from(e.dataTransfer.files)

if (files.length > 0) {
const acceptedTypes = ["png", "jpeg", "webp"]
const acceptedImageTypes = ["png", "jpeg", "webp"]
const acceptedVideoTypes = supportsVideo ? ["mp4", "mov", "avi", "webm"] : []

const imageFiles = files.filter((file) => {
const mediaFiles = files.filter((file) => {
const [type, subtype] = file.type.split("/")
return type === "image" && acceptedTypes.includes(subtype)
return (
(type === "image" && acceptedImageTypes.includes(subtype)) ||
(type === "video" && acceptedVideoTypes.includes(subtype))
)
})

if (!shouldDisableImages && imageFiles.length > 0) {
const imagePromises = imageFiles.map((file) => {
if (!shouldDisableImages && mediaFiles.length > 0) {
const mediaPromises = mediaFiles.map((file) => {
return new Promise<string | null>((resolve) => {
const reader = new FileReader()

Expand All @@ -758,8 +768,8 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
})
})

const imageDataArray = await Promise.all(imagePromises)
const dataUrls = imageDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null)
const mediaDataArray = await Promise.all(mediaPromises)
const dataUrls = mediaDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null)

if (dataUrls.length > 0) {
setSelectedImages((prevImages) =>
Expand All @@ -785,6 +795,7 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
shouldDisableImages,
setSelectedImages,
t,
supportsVideo,
],
)

Expand Down
1 change: 1 addition & 0 deletions webview-ui/src/components/chat/ChatView.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1848,6 +1848,7 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
onSend={() => handleSendMessage(inputValue, selectedImages)}
onSelectImages={selectImages}
shouldDisableImages={shouldDisableImages}
supportsVideo={model?.supportsVideo || false}
onHeightChange={() => {
if (isAtBottom) {
scrollToBottomAuto()
Expand Down