Skip to content

Commit 76f79cc

Browse files
michael-amerimichael-ameri
committed
text and image example
1 parent 62b2c0d commit 76f79cc

File tree

4 files changed

+161
-8
lines changed

4 files changed

+161
-8
lines changed

‎gemini-api/src/main/java/swiss/ameri/gemini/api/Content.java‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package swiss.ameri.gemini.api;
22

3+
import java.util.ArrayList;
34
import java.util.List;
45
import java.util.Locale;
56

@@ -8,6 +9,47 @@
89
*/
910
public sealed interface Content {
1011

12+
/**
13+
* Create a {@link TextContent}.
14+
*
15+
* @param role belonging to this turn in the conversation.
16+
* @param text by the role
17+
* @return a {@link TextContent}
18+
*/
19+
static TextContent textContent(
20+
Role role,
21+
String text
22+
) {
23+
return new TextContent(role == null ? null : role.roleName(), text);
24+
}
25+
26+
/**
27+
* Create a {@link MediaContent}.
28+
*
29+
* @param role belonging to this turn in the conversation.
30+
* @param mimeType see {@link MediaData}
31+
* @param mediaBase64 see {@link MediaData}
32+
* @return a {@link MediaContent}
33+
*/
34+
static MediaContent mediaContent(
35+
Role role,
36+
String mimeType,
37+
String mediaBase64
38+
39+
) {
40+
return new MediaContent(role == null ? null : role.roleName(), new MediaData(mimeType, mediaBase64));
41+
}
42+
43+
/**
44+
* For combined text and media content in one turn
45+
*
46+
* @return a {@link TextAndMediaContent.TextAndMediaContentBuilder}
47+
*/
48+
static TextAndMediaContent.TextAndMediaContentBuilder textAndMediaContentBuilder() {
49+
return TextAndMediaContent.builder();
50+
}
51+
52+
1153
/**
1254
* A part of a conversation that contains text.
1355
*
@@ -44,6 +86,77 @@ record TextAndMediaContent(
4486
List<MediaData> media
4587
) implements Content {
4688
// todo test
89+
90+
public static TextAndMediaContentBuilder builder() {
91+
return new TextAndMediaContentBuilder();
92+
}
93+
94+
95+
/**
96+
* Builder for {@link TextAndMediaContent}.
97+
*/
98+
public static class TextAndMediaContentBuilder {
99+
private String role;
100+
private String text;
101+
private final List<MediaData> media = new ArrayList<>();
102+
103+
private TextAndMediaContentBuilder() {
104+
}
105+
106+
/**
107+
* Set the role
108+
*
109+
* @param role to set
110+
* @return this
111+
* @see #role(Role)
112+
*/
113+
public TextAndMediaContentBuilder role(String role) {
114+
this.role = role;
115+
return this;
116+
}
117+
118+
/**
119+
* Set the role
120+
*
121+
* @param role to set
122+
* @return this
123+
*/
124+
public TextAndMediaContentBuilder role(Role role) {
125+
return role(role == null ? null : role.roleName());
126+
}
127+
128+
/**
129+
* Set the text
130+
*
131+
* @param text to set
132+
* @return this
133+
*/
134+
public TextAndMediaContentBuilder text(String text) {
135+
this.text = text;
136+
return this;
137+
}
138+
139+
/**
140+
* Add media
141+
*
142+
* @param media to add
143+
* @return this
144+
*/
145+
public TextAndMediaContentBuilder addMedia(MediaData media) {
146+
this.media.add(media);
147+
return this;
148+
}
149+
150+
/**
151+
* Create the result. It is not validated.
152+
*
153+
* @return a newly built {@link TextAndMediaContent}
154+
*/
155+
public TextAndMediaContent build() {
156+
return new TextAndMediaContent(role, text, media);
157+
}
158+
}
159+
47160
}
48161

49162
/**

‎gemini-api/src/main/java/swiss/ameri/gemini/api/GenAi.java‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,6 @@ private static GenerateContentRequest convert(GenerativeModel model) {
176176
List.of(
177177
new GenerationPart(
178178
textContent.text(),
179-
null,
180179
null
181180
)
182181
)
@@ -187,8 +186,10 @@ private static GenerateContentRequest convert(GenerativeModel model) {
187186
List.of(
188187
new GenerationPart(
189188
null,
190-
imageContent.media().mimeType(),
191-
imageContent.media().mediaBase64()
189+
new InlineData(
190+
imageContent.media().mimeType(),
191+
imageContent.media().mediaBase64()
192+
)
192193
)
193194
)
194195
);
@@ -199,15 +200,16 @@ private static GenerateContentRequest convert(GenerativeModel model) {
199200
Stream.of(
200201
new GenerationPart(
201202
textAndImagesContent.text(),
202-
null,
203203
null
204204
)
205205
),
206206
textAndImagesContent.media().stream()
207207
.map(imageData -> new GenerationPart(
208208
null,
209-
imageData.mimeType(),
210-
imageData.mediaBase64()
209+
new InlineData(
210+
imageData.mimeType(),
211+
imageData.mediaBase64()
212+
)
211213
))
212214
).toList()
213215
);
@@ -279,7 +281,13 @@ private record GenerationContent(
279281
}
280282

281283
private record GenerationPart(
284+
// contains one or the other
282285
String text,
286+
InlineData inline_data
287+
) {
288+
}
289+
290+
private record InlineData(
283291
String mime_type,
284292
String data
285293
) {

‎gemini-tester/src/main/java/swiss/ameri/gemini/tester/GeminiTester.java‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
import swiss.ameri.gemini.gson.GsonJsonParser;
55
import swiss.ameri.gemini.spi.JsonParser;
66

7+
import java.io.IOException;
8+
import java.io.InputStream;
9+
import java.util.Base64;
710
import java.util.concurrent.ExecutionException;
811
import java.util.concurrent.TimeUnit;
912
import java.util.concurrent.TimeoutException;
@@ -38,6 +41,7 @@ public static void main(String[] args) throws Exception {
3841
generateContent(genAi);
3942
generateContentStream(genAi);
4043
multiChatTurn(genAi);
44+
textAndImage(genAi);
4145
}
4246

4347
private static void multiChatTurn(GenAi genAi) {
@@ -79,8 +83,8 @@ private static void generateContent(GenAi genAi) throws InterruptedException, Ex
7983
private static GenerativeModel createStoryModel() {
8084
return GenerativeModel.builder()
8185
.modelName(ModelVariant.GEMINI_1_0_PRO)
82-
.addContent(new Content.TextContent(
83-
Content.Role.USER.roleName(),
86+
.addContent(Content.textContent(
87+
Content.Role.USER,
8488
"Write a 50 word story about a magic backpack."
8589
))
8690
.addSafetySetting(SafetySetting.of(
@@ -111,4 +115,32 @@ private static void listModels(GenAi genAi) {
111115
genAi.listModels()
112116
.forEach(System.out::println);
113117
}
118+
119+
private static void textAndImage(GenAi genAi) throws IOException {
120+
System.out.println("----- text and image");
121+
var model = GenerativeModel.builder()
122+
.modelName(ModelVariant.GEMINI_1_0_PRO_VISION)
123+
.addContent(
124+
Content.textAndMediaContentBuilder()
125+
.role(Content.Role.USER)
126+
.text("What is in this image?")
127+
.addMedia(new Content.MediaData(
128+
"image/png",
129+
loadSconesImage()
130+
))
131+
.build()
132+
).build();
133+
genAi.generateContent(model)
134+
.thenAccept(System.out::println)
135+
.join();
136+
}
137+
138+
public static String loadSconesImage() throws IOException {
139+
try (InputStream is = GeminiTester.class.getClassLoader().getResourceAsStream("scones.png")) {
140+
if (is == null) {
141+
throw new IllegalStateException("Image not found! ");
142+
}
143+
return Base64.getEncoder().encodeToString(is.readAllBytes());
144+
}
145+
}
114146
}

‎gemini-tester/src/main/resources/scones.png‎

349 KB
Loading

0 commit comments

Comments
 (0)