Skip to content

Commit cb816a0

Browse files
geoandgeoand
committed
Introduce support for Video in AI services
Closes: #1880
1 parent 66bd02a commit cb816a0

File tree

5 files changed

+118
-15
lines changed

5 files changed

+118
-15
lines changed

‎core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/AiServicesProcessor.java‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1956,6 +1956,11 @@ private AiServiceMethodCreateInfo.UserMessageInfo gatherUserMessageInfo(MethodIn
19561956
MethodParameterInfo pdfUrlParam = method.parameters().get(pdfParamPosition.get());
19571957
validatePdfUrlParam(pdfUrlParam);
19581958
}
1959+
Optional<Integer> videoParamPosition = determineVideoParamPosition(method);
1960+
if (videoParamPosition.isPresent()) {
1961+
MethodParameterInfo videoUrlParam = method.parameters().get(videoParamPosition.get());
1962+
validateVideoUrlParam(videoUrlParam);
1963+
}
19591964

19601965
AnnotationInstance userMessageInstance = method.declaredAnnotation(LangChain4jDotNames.USER_MESSAGE);
19611966
if (userMessageInstance != null) {
@@ -1974,7 +1979,7 @@ private AiServiceMethodCreateInfo.UserMessageInfo gatherUserMessageInfo(MethodIn
19741979
return AiServiceMethodCreateInfo.UserMessageInfo.fromTemplate(
19751980
AiServiceMethodCreateInfo.TemplateInfo.fromText(userMessageTemplate,
19761981
TemplateParameterInfo.toNameToArgsPositionMap(templateParams)),
1977-
userNameParamPosition, imageParamPosition, audioParamPosition, pdfParamPosition);
1982+
userNameParamPosition, imageParamPosition, audioParamPosition, pdfParamPosition, videoParamPosition);
19781983
} else {
19791984
Optional<AnnotationInstance> userMessageOnMethodParam = method.annotations(LangChain4jDotNames.USER_MESSAGE)
19801985
.stream()
@@ -1987,11 +1992,13 @@ private AiServiceMethodCreateInfo.UserMessageInfo gatherUserMessageInfo(MethodIn
19871992
Short.valueOf(userMessageOnMethodParam.get().target().asMethodParameter().position())
19881993
.intValue(),
19891994
TemplateParameterInfo.toNameToArgsPositionMap(templateParams)),
1990-
userNameParamPosition, imageParamPosition, audioParamPosition, pdfParamPosition);
1995+
userNameParamPosition, imageParamPosition, audioParamPosition, pdfParamPosition,
1996+
videoParamPosition);
19911997
} else {
19921998
return AiServiceMethodCreateInfo.UserMessageInfo.fromMethodParam(
19931999
userMessageOnMethodParam.get().target().asMethodParameter().position(),
1994-
userNameParamPosition, imageParamPosition, audioParamPosition, pdfParamPosition);
2000+
userNameParamPosition, imageParamPosition, audioParamPosition, pdfParamPosition,
2001+
videoParamPosition);
19952002
}
19962003
} else {
19972004
Set<String> templateParamNames = Collections.EMPTY_SET;
@@ -2026,7 +2033,7 @@ private AiServiceMethodCreateInfo.UserMessageInfo gatherUserMessageInfo(MethodIn
20262033
return AiServiceMethodCreateInfo.UserMessageInfo.fromTemplate(
20272034
AiServiceMethodCreateInfo.TemplateInfo.fromText("", Map.of()), Optional.empty(),
20282035
Optional.empty(),
2029-
Optional.empty(), Optional.empty());
2036+
Optional.empty(), Optional.empty(), Optional.empty());
20302037
}
20312038

20322039
throw illegalConfigurationForMethod(
@@ -2038,11 +2045,11 @@ private AiServiceMethodCreateInfo.UserMessageInfo gatherUserMessageInfo(MethodIn
20382045
if (userMessageParamPosition == -1) {
20392046
// There is no user message
20402047
return new AiServiceMethodCreateInfo.UserMessageInfo(Optional.empty(), Optional.empty(), Optional.empty(),
2041-
Optional.empty(), Optional.empty(), Optional.empty());
2048+
Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty());
20422049
} else {
20432050
return AiServiceMethodCreateInfo.UserMessageInfo.fromMethodParam(userMessageParamPosition,
20442051
userNameParamPosition,
2045-
imageParamPosition, audioParamPosition, pdfParamPosition);
2052+
imageParamPosition, audioParamPosition, pdfParamPosition, videoParamPosition);
20462053

20472054
}
20482055
}
@@ -2082,6 +2089,17 @@ private static Optional<Integer> determinePdfParamPosition(MethodInfo method) {
20822089
.map(pi -> (int) pi.position()).findFirst();
20832090
}
20842091

2092+
private static Optional<Integer> determineVideoParamPosition(MethodInfo method) {
2093+
Optional<Integer> result = method.annotations(LangChain4jDotNames.VIDEO_URL).stream().filter(
2094+
IS_METHOD_PARAMETER_ANNOTATION).map(METHOD_PARAMETER_POSITION_FUNCTION).findFirst();
2095+
if (result.isPresent()) {
2096+
return result;
2097+
}
2098+
// we don't need @VideoUrl if the parameter is of type PdfFile
2099+
return method.parameters().stream().filter(pi -> pi.type().name().equals(LangChain4jDotNames.VIDEO))
2100+
.map(pi -> (int) pi.position()).findFirst();
2101+
}
2102+
20852103
private void validateImageUrlParam(MethodParameterInfo param) {
20862104
if (param == null) {
20872105
throw new IllegalArgumentException("Unhandled @ImageUrl annotation");
@@ -2121,6 +2139,19 @@ private void validatePdfUrlParam(MethodParameterInfo param) {
21212139
throw new IllegalArgumentException("Unhandled @PdfUrl type '" + type.name() + "'");
21222140
}
21232141

2142+
private void validateVideoUrlParam(MethodParameterInfo param) {
2143+
if (param == null) {
2144+
throw new IllegalArgumentException("Unhandled @VideoUrl annotation");
2145+
}
2146+
Type type = param.type();
2147+
DotName typeName = type.name();
2148+
if (typeName.equals(DotNames.STRING) || typeName.equals(DotNames.URI) || typeName.equals(DotNames.URL)
2149+
|| typeName.equals(LangChain4jDotNames.VIDEO)) {
2150+
return;
2151+
}
2152+
throw new IllegalArgumentException("Unhandled @VideoUrl type '" + type.name() + "'");
2153+
}
2154+
21242155
private Optional<AiServiceMethodCreateInfo.MetricsTimedInfo> gatherMetricsTimedInfo(MethodInfo method,
21252156
boolean addMicrometerMetrics) {
21262157
if (!addMicrometerMetrics) {

‎core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/LangChain4jDotNames.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import io.quarkiverse.langchain4j.PdfUrl;
4040
import io.quarkiverse.langchain4j.RegisterAiService;
4141
import io.quarkiverse.langchain4j.SeedMemory;
42+
import io.quarkiverse.langchain4j.VideoUrl;
4243
import io.quarkiverse.langchain4j.runtime.aiservice.ChatEvent;
4344
import io.quarkiverse.langchain4j.runtime.aiservice.QuarkusAiServiceContextQualifier;
4445

@@ -62,6 +63,7 @@ public class LangChain4jDotNames {
6263
static final DotName IMAGE_URL = DotName.createSimple(ImageUrl.class);
6364
static final DotName AUDIO_URL = DotName.createSimple(AudioUrl.class);
6465
static final DotName PDF_URL = DotName.createSimple(PdfUrl.class);
66+
static final DotName VIDEO_URL = DotName.createSimple(VideoUrl.class);
6567
static final DotName MODERATE = DotName.createSimple(Moderate.class);
6668
static final DotName MEMORY_ID = DotName.createSimple(MemoryId.class);
6769
static final DotName DESCRIPTION = DotName.createSimple(Description.class);
@@ -119,6 +121,7 @@ public class LangChain4jDotNames {
119121
static final DotName IMAGE = DotName.createSimple(Image.class);
120122
static final DotName AUDIO = DotName.createSimple(dev.langchain4j.data.audio.Audio.class);
121123
static final DotName PDF_FILE = DotName.createSimple(PdfFile.class);
124+
static final DotName VIDEO = DotName.createSimple(dev.langchain4j.data.video.Video.class);
122125
static final DotName RESULT = DotName.createSimple(Result.class);
123126
public static final DotName TOOL_PROVIDER = DotName.createSimple(ToolProvider.class);
124127
// Using the class name to keep the McpToolBox annotation in the mcp module

‎core/runtime/src/main/java/io/quarkiverse/langchain4j/VideoUrl.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package io.quarkiverse.langchain4j;
2+
3+
import static java.lang.annotation.ElementType.PARAMETER;
4+
import static java.lang.annotation.RetentionPolicy.RUNTIME;
5+
6+
import java.lang.annotation.Retention;
7+
import java.lang.annotation.Target;
8+
9+
/**
10+
* This annotation is useful when an AiService is meant to describe a video as the value of the method parameter annotated
11+
* with @VideoUrl
12+
* will be used as an {@link dev.langchain4j.data.message.VideoContent}.
13+
* <p>
14+
* <p>
15+
* The following code contains an example of how this can be used:
16+
*
17+
* <pre>
18+
* {@code
19+
* @RegisterAiService(chatMemoryProviderSupplier = RegisterAiService.NoChatMemoryProviderSupplier.class)
20+
* public interface VideoDescriber {
21+
*
22+
* @UserMessage("Describe the video")
23+
* Report describe(@VideoUrl String url);
24+
* }
25+
* </pre>
26+
*
27+
* There can be at most one instance of {@code VideoUrl} per method and the supported types are the following:
28+
* <ul>
29+
* <li>String</li>
30+
* <li>URL</li>
31+
* <li>URI</li>
32+
* <li>dev.langchain4j.data.video.Video</li>
33+
* </ul>
34+
*
35+
*/
36+
@Retention(RUNTIME)
37+
@Target({ PARAMETER })
38+
public @interface VideoUrl {
39+
40+
}

‎core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/AiServiceMethodCreateInfo.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -257,21 +257,24 @@ public record UserMessageInfo(Optional<TemplateInfo> template,
257257
Optional<Integer> userNameParamPosition,
258258
Optional<Integer> imageParamPosition,
259259
Optional<Integer> audioParamPosition,
260-
Optional<Integer> pdfParamPosition) {
260+
Optional<Integer> pdfParamPosition,
261+
Optional<Integer> videoParamPosition) {
261262

262263
public static UserMessageInfo fromMethodParam(int paramPosition, Optional<Integer> userNameParamPosition,
263264
Optional<Integer> imageParamPosition, Optional<Integer> audioParamPosition,
264-
Optional<Integer> pdfParamPosition) {
265+
Optional<Integer> pdfParamPosition,
266+
Optional<Integer> videoParamPosition) {
265267
return new UserMessageInfo(Optional.empty(), Optional.of(paramPosition),
266-
userNameParamPosition, imageParamPosition, audioParamPosition, pdfParamPosition);
268+
userNameParamPosition, imageParamPosition, audioParamPosition, pdfParamPosition, videoParamPosition);
267269
}
268270

269271
public static UserMessageInfo fromTemplate(TemplateInfo templateInfo, Optional<Integer> userNameParamPosition,
270272
Optional<Integer> imageUrlParamPosition,
271273
Optional<Integer> audioParamPosition,
272-
Optional<Integer> pdfParamPosition) {
274+
Optional<Integer> pdfParamPosition,
275+
Optional<Integer> videoParamPosition) {
273276
return new UserMessageInfo(Optional.of(templateInfo), Optional.empty(), userNameParamPosition,
274-
imageUrlParamPosition, audioParamPosition, pdfParamPosition);
277+
imageUrlParamPosition, audioParamPosition, pdfParamPosition, videoParamPosition);
275278
}
276279
}
277280

‎core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/AiServiceMethodImplementationSupport.java‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,9 @@
5151
import dev.langchain4j.data.message.TextContent;
5252
import dev.langchain4j.data.message.ToolExecutionResultMessage;
5353
import dev.langchain4j.data.message.UserMessage;
54+
import dev.langchain4j.data.message.VideoContent;
5455
import dev.langchain4j.data.pdf.PdfFile;
56+
import dev.langchain4j.data.video.Video;
5557
import dev.langchain4j.guardrail.ChatExecutor;
5658
import dev.langchain4j.guardrail.GuardrailRequestParams;
5759
import dev.langchain4j.invocation.InvocationContext;
@@ -965,6 +967,7 @@ private static UserMessage prepareUserMessage(AiServiceContext context, AiServic
965967
ImageContent imageContent = null;
966968
AudioContent audioContent = null;
967969
PdfFileContent pdfFileContent = null;
970+
VideoContent videoContent = null;
968971
if (userMessageInfo.userNameParamPosition().isPresent()) {
969972
userName = methodArgs[userMessageInfo.userNameParamPosition().get()]
970973
.toString(); // LangChain4j does this, but might want to make anything other than a String a
@@ -1030,6 +1033,26 @@ private static UserMessage prepareUserMessage(AiServiceContext context, AiServic
10301033
+ createInfo.getMethodName());
10311034
}
10321035
}
1036+
if (userMessageInfo.videoParamPosition().isPresent()) {
1037+
Object videoParamValue = methodArgs[userMessageInfo.videoParamPosition().get()];
1038+
if (videoParamValue instanceof String s) {
1039+
videoContent = VideoContent.from(s);
1040+
} else if (videoParamValue instanceof URI u) {
1041+
videoContent = VideoContent.from(u);
1042+
} else if (videoParamValue instanceof URL u) {
1043+
try {
1044+
videoContent = VideoContent.from(u.toURI());
1045+
} catch (URISyntaxException e) {
1046+
throw new RuntimeException(e);
1047+
}
1048+
} else if (videoParamValue instanceof Video v) {
1049+
videoContent = VideoContent.from(v);
1050+
} else {
1051+
throw new IllegalStateException("Unsupported parameter type '" + videoParamValue.getClass()
1052+
+ "' annotated with @AudioUrl. Offending AiService is '" + createInfo.getInterfaceName() + "#"
1053+
+ createInfo.getMethodName());
1054+
}
1055+
}
10331056

10341057
if (userMessageInfo.template().isPresent()) {
10351058
AiServiceMethodCreateInfo.TemplateInfo templateInfo = userMessageInfo.template().get();
@@ -1063,7 +1086,7 @@ private static UserMessage prepareUserMessage(AiServiceContext context, AiServic
10631086
}
10641087

10651088
Prompt prompt = PromptTemplate.from(templateText).apply(templateVariables);
1066-
return createUserMessage(userName, imageContent, audioContent, pdfFileContent, prompt.text());
1089+
return createUserMessage(userName, imageContent, audioContent, pdfFileContent, videoContent, prompt.text());
10671090

10681091
} else if (userMessageInfo.paramPosition().isPresent()) {
10691092
Integer paramIndex = userMessageInfo.paramPosition().get();
@@ -1077,8 +1100,8 @@ private static UserMessage prepareUserMessage(AiServiceContext context, AiServic
10771100

10781101
String text = toString(argValue);
10791102
return createUserMessage(userName, imageContent,
1080-
audioContent,
1081-
pdfFileContent, text.concat(supportsJsonSchema || !createInfo.getResponseSchemaInfo().enabled() ? ""
1103+
audioContent, pdfFileContent, videoContent,
1104+
text.concat(supportsJsonSchema || !createInfo.getResponseSchemaInfo().enabled() ? ""
10821105
: createInfo.getResponseSchemaInfo().outputFormatInstructions()));
10831106
} else {
10841107
// create a user message that instructs the model to ignore it's content
@@ -1106,7 +1129,7 @@ private static Map<String, Object> getTemplateVariables(Object[] methodArgs,
11061129
}
11071130

11081131
private static UserMessage createUserMessage(String name, ImageContent imageContent, AudioContent audioContent,
1109-
PdfFileContent pdfFileContent,
1132+
PdfFileContent pdfFileContent, VideoContent videoContent,
11101133
String text) {
11111134
List<dev.langchain4j.data.message.Content> contents = new ArrayList<>();
11121135
contents.add(TextContent.from(text));
@@ -1119,6 +1142,9 @@ private static UserMessage createUserMessage(String name, ImageContent imageCont
11191142
if (pdfFileContent != null) {
11201143
contents.add(pdfFileContent);
11211144
}
1145+
if (videoContent != null) {
1146+
contents.add(videoContent);
1147+
}
11221148
if (name == null) {
11231149
return UserMessage.userMessage(contents);
11241150
} else {

0 commit comments

Comments
 (0)