diff --git a/pkg/epp/scheduling/types/types.go b/pkg/epp/scheduling/types/types.go index 2685a22d0..0865c7624 100644 --- a/pkg/epp/scheduling/types/types.go +++ b/pkg/epp/scheduling/types/types.go @@ -97,7 +97,16 @@ func (r *ChatCompletionsRequest) String() string { messagesLen := 0 for _, msg := range r.Messages { - messagesLen += len(msg.Content) + switch content := msg.Content.(type) { + case string: + messagesLen += len(content) + case []interface{}: + // For content arrays, we'll count each element as contributing some length + messagesLen += len(content) * 10 // arbitrary multiplier for array elements + default: + // For other types, just add a small constant + messagesLen += 1 + } } return fmt.Sprintf("{MessagesLength: %d}", messagesLen) @@ -106,7 +115,7 @@ func (r *ChatCompletionsRequest) String() string { // Message represents a single message in a chat-completions request. type Message struct { Role string - Content string // TODO: support multi-modal content + Content any // Support both string and array content (OpenAI content blocks) } type Pod interface { diff --git a/test/e2e/epp/e2e_test.go b/test/e2e/epp/e2e_test.go index c3ff49e58..0fa08e121 100644 --- a/test/e2e/epp/e2e_test.go +++ b/test/e2e/epp/e2e_test.go @@ -39,6 +39,16 @@ import ( testutils "sigs.k8s.io/gateway-api-inference-extension/test/utils" ) +type ContentBlock struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *ImageBlock `json:"image_url,omitempty"` +} + +type ImageBlock struct { + Url string `json:"url"` +} + var _ = ginkgo.Describe("InferencePool", func() { var infObjective *v1alpha2.InferenceObjective ginkgo.BeforeEach(func() { @@ -201,6 +211,24 @@ func verifyTrafficRouting() { {"role": "user", "content": "Now summarize your thoughts."}, }, }, + { + api: "/chat/completions", + promptOrMessages: []map[string]any{ + { + "role": "user", + "content": []ContentBlock{ + { + Type: "text", + Text: `What's in this image?`, + }, + { + Type: "image_url", + ImageURL: &ImageBlock{Url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"}, + }, + }, + }, + }, + }, } { ginkgo.By(fmt.Sprintf("Verifying connectivity through the inference extension with %s api and prompt/messages: %v", t.api, t.promptOrMessages))