Skip to content

Commit 1b6d286

Browse files
rpanackalrpanackal
authored
test: [Orchestration] Extend Grounding Unit Test (#281)
* Extend unit test and add sample request json * Minor - check false report * Fix typo * Extend test to use additional model classes --------- Co-authored-by: Roshin Rajan Panackal <[email protected]>
1 parent 44700cf commit 1b6d286

File tree

3 files changed

+139
-43
lines changed

3 files changed

+139
-43
lines changed

‎orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java‎

Lines changed: 65 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,23 @@
4141
import com.sap.ai.sdk.orchestration.model.CompletionPostRequest;
4242
import com.sap.ai.sdk.orchestration.model.CompletionPostResponse;
4343
import com.sap.ai.sdk.orchestration.model.DPIEntities;
44+
import com.sap.ai.sdk.orchestration.model.DataRepositoryType;
45+
import com.sap.ai.sdk.orchestration.model.DocumentGroundingFilter;
4446
import com.sap.ai.sdk.orchestration.model.GenericModuleResult;
47+
import com.sap.ai.sdk.orchestration.model.GroundingFilterSearchConfiguration;
48+
import com.sap.ai.sdk.orchestration.model.GroundingModuleConfig;
49+
import com.sap.ai.sdk.orchestration.model.GroundingModuleConfigConfig;
4550
import com.sap.ai.sdk.orchestration.model.ImageContent;
4651
import com.sap.ai.sdk.orchestration.model.ImageContentImageUrl;
52+
import com.sap.ai.sdk.orchestration.model.KeyValueListPair;
4753
import com.sap.ai.sdk.orchestration.model.LLMModuleConfig;
4854
import com.sap.ai.sdk.orchestration.model.LLMModuleResult;
4955
import com.sap.ai.sdk.orchestration.model.LLMModuleResultSynchronous;
5056
import com.sap.ai.sdk.orchestration.model.ModuleConfigs;
5157
import com.sap.ai.sdk.orchestration.model.MultiChatMessage;
5258
import com.sap.ai.sdk.orchestration.model.OrchestrationConfig;
59+
import com.sap.ai.sdk.orchestration.model.SearchDocumentKeyValueListPair;
60+
import com.sap.ai.sdk.orchestration.model.SearchSelectOptionEnum;
5361
import com.sap.ai.sdk.orchestration.model.Template;
5462
import com.sap.ai.sdk.orchestration.model.TextContent;
5563
import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor;
@@ -129,34 +137,68 @@ void testCompletion() {
129137
}
130138

131139
@Test
132-
void testGrounding() {
140+
void testGrounding() throws IOException {
133141
stubFor(
134-
post(anyUrl())
142+
post(urlPathEqualTo("/completion"))
135143
.willReturn(
136144
aResponse()
137145
.withBodyFile("groundingResponse.json")
138146
.withHeader("Content-Type", "application/json")));
139-
final var response = client.chatCompletion(prompt, config);
140-
final var result = response.getOriginalResponse();
141-
var llmChoice =
142-
((LLMModuleResultSynchronous) result.getOrchestrationResult()).getChoices().get(0);
143-
144-
final var groundingData =
145-
(Map<String, Object>) result.getModuleResults().getGrounding().getData();
146-
assertThat(groundingData.get("grounding_query")).isEqualTo("grounding call");
147-
assertThat(groundingData.get("grounding_result").toString())
148-
.startsWith("Joule is the AI copilot that truly understands your business.");
149-
assertThat(result.getModuleResults().getGrounding().getMessage()).isEqualTo("grounding result");
150-
assertThat(((ChatMessage) result.getModuleResults().getTemplating().get(0)).getContent())
151-
.startsWith(
152-
"What does Joule do? Use the following information as additional context: Joule is the AI copilot that truly understands your business.");
153-
assertThat(llmChoice.getMessage().getContent())
154-
.startsWith(
155-
"Joule is an AI copilot that revolutionizes how users interact with their SAP business systems.");
156-
assertThat(llmChoice.getFinishReason()).isEqualTo("stop");
157-
assertThat(llmChoice.getMessage().getContent())
158-
.startsWith(
159-
"Joule is an AI copilot that revolutionizes how users interact with their SAP business systems.");
147+
148+
final var documentMetadata =
149+
SearchDocumentKeyValueListPair.create()
150+
.key("document metadata")
151+
.value("2")
152+
.selectMode(List.of(SearchSelectOptionEnum.IGNORE_IF_KEY_ABSENT));
153+
final var databaseFilter =
154+
DocumentGroundingFilter.create()
155+
.id("arbitrary-user-defined-id")
156+
.dataRepositoryType(DataRepositoryType.VECTOR)
157+
.searchConfig(GroundingFilterSearchConfiguration.create().maxChunkCount(3))
158+
.documentMetadata(List.of(documentMetadata))
159+
.chunkMetadata(List.of(KeyValueListPair.create().key("chunk metadata").value("1")));
160+
final var groundingConfigConfig =
161+
GroundingModuleConfigConfig.create()
162+
.inputParams(List.of("query"))
163+
.outputParam("results")
164+
.addFiltersItem(databaseFilter);
165+
final var groundingConfig =
166+
GroundingModuleConfig.create()
167+
.type(GroundingModuleConfig.TypeEnum.DOCUMENT_GROUNDING_SERVICE)
168+
.config(groundingConfigConfig);
169+
final var configWithGrounding = config.withGroundingConfig(groundingConfig);
170+
171+
final Map<String, String> inputParams =
172+
Map.of("query", "String used for similarity search in database");
173+
final var prompt =
174+
new OrchestrationPrompt(
175+
inputParams,
176+
Message.system("Context message with embedded grounding results. {{?results}}"));
177+
178+
final var response = client.chatCompletion(prompt, configWithGrounding);
179+
180+
assertThat(response).isNotNull();
181+
assertThat(response.getOriginalResponse().getRequestId())
182+
.isEqualTo("e5d2add4-408c-4da5-84ca-1d8b0fe350c8");
183+
184+
var moduleResults = response.getOriginalResponse().getModuleResults();
185+
assertThat(moduleResults).isNotNull();
186+
187+
var groundingModule = moduleResults.getGrounding();
188+
assertThat(groundingModule).isNotNull();
189+
assertThat(groundingModule.getMessage()).isEqualTo("grounding result");
190+
assertThat(groundingModule.getData()).isNotNull();
191+
var groundingData =
192+
Map.of(
193+
"grounding_query",
194+
"grounding call",
195+
"grounding_result",
196+
"First chunk```Second chunk```Last found chunk");
197+
assertThat(groundingModule.getData()).isEqualTo(groundingData);
198+
199+
final String requestBody = new String(fileLoader.apply("groundingRequest.json").readAllBytes());
200+
verify(
201+
postRequestedFor(urlPathEqualTo("/completion")).withRequestBody(equalToJson(requestBody)));
160202
}
161203

162204
@Test

‎orchestration/src/test/resources/__files/groundingResponse.json‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,62 @@
11
{
2-
"request_id": "0d9d7ce3-9ded-481f-80c6-977e78e2e905",
2+
"request_id": "e5d2add4-408c-4da5-84ca-1d8b0fe350c8",
33
"module_results": {
44
"grounding": {
55
"message": "grounding result",
66
"data": {
77
"grounding_query": "grounding call",
8-
"grounding_result": "Joule is the AI copilot that truly understands your business. Joule revolutionizes how you interact with your SAP business systems, making every touchpoint count and every task simpler.```It enables the companion of the Intelligent Enterprise, guiding you through content discovery within SAP Ecosystem, and giving a transparent role-based access to the relevant processes from everywhere. This is the one assistant experience, a unified and delightful user experience across SAP’s \u01ee solution portfolio."
8+
"grounding_result": "First chunk```Second chunk```Last found chunk"
99
}
1010
},
1111
"templating": [
1212
{
13-
"role": "user",
14-
"content": "What does Joule do? Use the following information as additional context: Joule is the AI copilot that truly understands your business. Joule revolutionizes how you interact with your SAP business systems, making every touchpoint count and every task simpler.```It enables the companion of the Intelligent Enterprise, guiding you through content discovery within SAP Ecosystem, and giving a transparent role-based access to the relevant processes from everywhere. This is the one assistant experience, a unified and delightful user experience across SAP’s \u01ee solution portfolio."
13+
"role": "system",
14+
"content": "Context message with embedded grounding results. First chunk```Second chunk```Last found chunk"
1515
}
1616
],
1717
"llm": {
18-
"id": "chatcmpl-AbRlNdsyQJfvBINnw3MOTP77WSE4X",
18+
"id": "chatcmpl-Apz5s3CMf99jkOnxvPshH1rGLwvvU",
1919
"object": "chat.completion",
20-
"created": 1733488221,
21-
"model": "gpt-35-turbo",
22-
"system_fingerprint": "fp_808245b034",
20+
"created": 1736952936,
21+
"model": "gpt-4o-2024-08-06",
22+
"system_fingerprint": "fp_f3927aa00d",
2323
"choices": [
2424
{
2525
"index": 0,
2626
"message": {
2727
"role": "assistant",
28-
"content": "Joule is an AI copilot that revolutionizes how users interact with their SAP business systems. It enables the companion of the Intelligent Enterprise, guiding users through content discovery within the SAP Ecosystem and providing transparent role-based access to relevant processes from anywhere. Joule aims to provide a unified and delightful user experience across SAP's solution portfolio."
28+
"content": "Response that makes uses of grounding results in the context message."
2929
},
3030
"finish_reason": "stop"
3131
}
3232
],
3333
"usage": {
34-
"completion_tokens": 68,
35-
"prompt_tokens": 113,
36-
"total_tokens": 181
34+
"completion_tokens": 163,
35+
"prompt_tokens": 217,
36+
"total_tokens": 380
3737
}
3838
}
3939
},
4040
"orchestration_result": {
41-
"id": "chatcmpl-AbRlNdsyQJfvBINnw3MOTP77WSE4X",
41+
"id": "chatcmpl-Apz5s3CMf99jkOnxvPshH1rGLwvvU",
4242
"object": "chat.completion",
43-
"created": 1733488221,
44-
"model": "gpt-35-turbo",
45-
"system_fingerprint": "fp_808245b034",
43+
"created": 1736952936,
44+
"model": "gpt-4o-2024-08-06",
45+
"system_fingerprint": "fp_f3927aa00d",
4646
"choices": [
4747
{
4848
"index": 0,
4949
"message": {
5050
"role": "assistant",
51-
"content": "Joule is an AI copilot that revolutionizes how users interact with their SAP business systems. It enables the companion of the Intelligent Enterprise, guiding users through content discovery within the SAP Ecosystem and providing transparent role-based access to relevant processes from anywhere. Joule aims to provide a unified and delightful user experience across SAP's solution portfolio."
51+
"content": "Response that makes uses of grounding results in the context message."
5252
},
5353
"finish_reason": "stop"
5454
}
5555
],
5656
"usage": {
57-
"completion_tokens": 68,
58-
"prompt_tokens": 113,
59-
"total_tokens": 181
57+
"completion_tokens": 163,
58+
"prompt_tokens": 217,
59+
"total_tokens": 380
6060
}
6161
}
6262
}

‎orchestration/src/test/resources/groundingRequest.json‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
{
2+
"orchestration_config" : {
3+
"module_configurations" : {
4+
"llm_module_config" : {
5+
"model_name" : "gpt-35-turbo-16k",
6+
"model_params" : {
7+
"max_tokens" : 50,
8+
"temperature" : 0.1,
9+
"frequency_penalty" : 0,
10+
"presence_penalty" : 0,
11+
"top_p" : 1,
12+
"n" : 1
13+
},
14+
"model_version" : "latest"
15+
},
16+
"templating_module_config" : {
17+
"template" : [ {
18+
"role" : "system",
19+
"content" : "Context message with embedded grounding results. {{?results}}"
20+
} ]
21+
},
22+
"grounding_module_config" : {
23+
"type" : "document_grounding_service",
24+
"config" : {
25+
"filters" : [ {
26+
"id" : "arbitrary-user-defined-id",
27+
"search_config" : {
28+
"max_chunk_count" : 3
29+
},
30+
"data_repositories" : [ "*" ],
31+
"data_repository_type" : "vector",
32+
"data_repository_metadata" : [ ],
33+
"document_metadata" : [ {
34+
"key" : "document metadata",
35+
"value" : [ "2" ],
36+
"select_mode" : [ "ignoreIfKeyAbsent" ]
37+
} ],
38+
"chunk_metadata" : [ {
39+
"key" : "chunk metadata",
40+
"value" : [ "1" ]
41+
} ]
42+
} ],
43+
"input_params" : [ "query" ],
44+
"output_param" : "results"
45+
}
46+
}
47+
},
48+
"stream" : false
49+
},
50+
"input_params" : {
51+
"query" : "String used for similarity search in database"
52+
},
53+
"messages_history" : [ ]
54+
}

0 commit comments

Comments
 (0)