fix: vision working

Author: xavidop

.gitignore

@@ -53,5 +53,6 @@ examples/chat/chat
 examples/tool_calling/tool_calling
 examples/embeddings/embeddings
 examples/streaming/streaming
+examples/vision/vision
 .genkit/
 node_modules/

README.md

@@ -392,6 +392,7 @@ The repository includes comprehensive examples:
 - **`examples/chat/`** - Multi-turn conversation with context
 - **`examples/embeddings/`** - Text embeddings generation
 - **`examples/tool_calling/`** - Function calling with multiple tools
+- **`examples/vision/`** - Multimodal image analysis
 
 ### Running Examples
 
@@ -415,6 +416,10 @@ go run main.go
 # Run tool calling example
 cd ../tool_calling
 go run main.go
+
+# Run vision example
+cd ../vision
+go run main.go
 ```
 
 ## Features in Detail

azureaifoundry_plugin.go

@@ -188,6 +188,24 @@ func (a *AzureAIFoundry) generateText(ctx context.Context, modelName string, inp
 	return a.generateTextSync(ctx, params, input)
 }
 
+// hasMultimodalContent checks if a message contains multimodal content (text + images)
+func (a *AzureAIFoundry) hasMultimodalContent(msg *ai.Message) bool {
+	hasText := false
+	hasMedia := false
+
+	for _, part := range msg.Content {
+		if part.IsText() {
+			hasText = true
+		}
+		if part.IsMedia() {
+			hasMedia = true
+		}
+	}
+
+	// Return true if it has media, or if it has multiple parts (regardless of media)
+	return hasMedia || (hasText && len(msg.Content) > 1)
+}
+
 // convertMessagesToOpenAI converts Genkit messages to OpenAI message format
 func (a *AzureAIFoundry) convertMessagesToOpenAI(messages []*ai.Message) []openai.ChatCompletionMessageParamUnion {
 	var openAIMessages []openai.ChatCompletionMessageParamUnion
@@ -207,13 +225,48 @@ func (a *AzureAIFoundry) convertMessagesToOpenAI(messages []*ai.Message) []opena
 				},
 			})
 		case ai.RoleUser:
-			openAIMessages = append(openAIMessages, openai.ChatCompletionMessageParamUnion{
-				OfUser: &openai.ChatCompletionUserMessageParam{
-					Content: openai.ChatCompletionUserMessageParamContentUnion{
-						OfString: openai.String(msg.Content[0].Text),
+			// Check if message contains multimodal content (text + images)
+			if a.hasMultimodalContent(msg) {
+				// Handle multimodal content with array of content parts
+				var contentParts []openai.ChatCompletionContentPartUnionParam
+
+				for _, part := range msg.Content {
+					if part.IsText() {
+						contentParts = append(contentParts, openai.ChatCompletionContentPartUnionParam{
+							OfText: &openai.ChatCompletionContentPartTextParam{
+								Text: part.Text,
+							},
+						})
+					} else if part.IsMedia() {
+						// Handle image/media content
+						// Media parts store the URL in the Text field
+						contentParts = append(contentParts, openai.ChatCompletionContentPartUnionParam{
+							OfImageURL: &openai.ChatCompletionContentPartImageParam{
+								ImageURL: openai.ChatCompletionContentPartImageImageURLParam{
+									URL: part.Text,
+								},
+							},
+						})
+					}
+				}
+
+				openAIMessages = append(openAIMessages, openai.ChatCompletionMessageParamUnion{
+					OfUser: &openai.ChatCompletionUserMessageParam{
+						Content: openai.ChatCompletionUserMessageParamContentUnion{
+							OfArrayOfContentParts: contentParts,
+						},
 					},
-				},
-			})
+				})
+			} else {
+				// Simple text-only message
+				openAIMessages = append(openAIMessages, openai.ChatCompletionMessageParamUnion{
+					OfUser: &openai.ChatCompletionUserMessageParam{
+						Content: openai.ChatCompletionUserMessageParamContentUnion{
+							OfString: openai.String(msg.Content[0].Text),
+						},
+					},
+				})
+			}
 		case ai.RoleModel:
 			// Extract all content parts and tool requests
 			var textContent string

examples/vision/main.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"context"
+	"log"
+
+	"github.com/firebase/genkit/go/ai"
+	"github.com/firebase/genkit/go/genkit"
+	azureaifoundry "github.com/xavidop/genkit-azure-foundry-go"
+	"github.com/xavidop/genkit-azure-foundry-go/examples/common"
+)
+
+func main() {
+	ctx := context.Background()
+
+	// Setup Genkit with Azure AI Foundry
+	g, azurePlugin, err := common.SetupGenkit(ctx, nil)
+	if err != nil {
+		log.Fatalf("Failed to setup Genkit: %v", err)
+	}
+
+	// Define a GPT-5 model with vision support
+	gpt5Model := azurePlugin.DefineModel(g, azureaifoundry.ModelDefinition{
+		Name:           "gpt-5", // Your deployment name in Azure
+		Type:           "chat",
+		SupportsVision: true,
+	}, nil)
+
+	log.Println("Starting Vision (Multimodal) example...")
+	log.Println("This example demonstrates how to analyze images")
+	log.Println()
+
+	// Analyze an image from a URL
+	log.Println("===Analyzing an image from URL ===")
+	imageURL := "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
+
+	response1, err := genkit.Generate(ctx, g,
+		ai.WithModel(gpt5Model),
+		ai.WithMessages(ai.NewUserMessage(
+			ai.NewTextPart("What's in this image? Describe it in detail."),
+			ai.NewMediaPart("image/jpeg", imageURL),
+		)),
+	)
+
+	if err != nil {
+		log.Printf("Error analyzing image from URL: %v", err)
+	} else {
+		log.Printf("Response: %s\n", response1.Text())
+	}
+	log.Println()
+
+	log.Println("Vision example completed!")
+}