Adapt tests in OpenAI module to run with OpenAI API

- Add a test profile for working with real LLM in cloud, without mocks
- Conditionally enable tests, which don't work with Mock GPT
- Adapt verification to verify both answers from Mock GPT and OpenAI
- Use the same model for all tests
- Use less tokens, especially for answers
- Link non-working tests to issues

Author: fedinskiy

integration-tests/openai/src/main/java/org/acme/example/openai/MessageUtil.java

@@ -2,15 +2,23 @@
 
 import java.util.Collections;
 
+import org.eclipse.microprofile.config.ConfigProvider;
+
 import dev.langchain4j.model.openai.internal.chat.ChatCompletionRequest;
 import dev.langchain4j.model.openai.internal.completion.CompletionRequest;
 import dev.langchain4j.model.openai.internal.embedding.EmbeddingRequest;
 
 public class MessageUtil {
 
+    static String model;
+
+    static {
+        model = ConfigProvider.getConfig().getValue("quarkus.langchain4j.openai.chat-model.model-name", String.class);
+    }
+
     public static CompletionRequest createCompletionRequest(String prompt) {
         return CompletionRequest.builder()
-                .model("gpt-3.5-turbo")
+                .model(model)
                 .logitBias(Collections.emptyMap())
                 .maxTokens(100)
                 .user("testing")
@@ -22,7 +30,7 @@ public static CompletionRequest createCompletionRequest(String prompt) {
 
     public static ChatCompletionRequest createChatCompletionRequest(String userMessage) {
         return ChatCompletionRequest.builder()
-                .model("gpt-3.5-turbo")
+                .model(model)
                 .logitBias(Collections.emptyMap())
                 .maxTokens(100)
                 .user("testing")

integration-tests/openai/src/main/java/org/acme/example/openai/QuarkusRestApiResource.java

@@ -64,7 +64,8 @@ public QuarkusRestApiResource(LangChain4jOpenAiConfig runtimeConfig)
     @Path("chat/sync")
     public String chatSync() {
         return restApi.blockingChatCompletion(
-                createChatCompletionRequest("Write a short 1 paragraph funny poem about segmentation fault"),
+                createChatCompletionRequest(
+                        "Which one of these languages is more susceptible to segmentation fault: Java, Go, C++ or Rust?"),
                 OpenAiRestApi.ApiMetadata.builder()
                         .openAiApiKey(token)
                         .organizationId(organizationId)
@@ -76,7 +77,7 @@ public String chatSync() {
     @Path("chat/async")
     public Uni<String> chatAsync() {
         return restApi
-                .createChatCompletion(createChatCompletionRequest("Write a short 1 paragraph funny poem about Unicode"),
+                .createChatCompletion(createChatCompletionRequest("Write a short definition of Unicode"),
                         OpenAiRestApi.ApiMetadata.builder()
                                 .openAiApiKey(token)
                                 .organizationId(organizationId)
@@ -122,7 +123,8 @@ public Multi<String> chatStreaming() {
     @Path("language/sync")
     public String languageSync() {
         return restApi.blockingCompletion(
-                createCompletionRequest("Write a short 1 paragraph funny poem about segmentation fault"),
+                createCompletionRequest(
+                        "Which one of these languages is more susceptible to segmentation fault: Java, Go, C++ or Rust?"),
                 OpenAiRestApi.ApiMetadata.builder()
                         .openAiApiKey(token)
                         .organizationId(organizationId)
@@ -134,7 +136,7 @@ public String languageSync() {
     @Path("language/async")
     public Uni<String> languageAsync() {
         return restApi
-                .completion(createCompletionRequest("Write a short 1 paragraph funny poem about Unicode"),
+                .completion(createCompletionRequest("Write a short definition of Unicode"),
                         OpenAiRestApi.ApiMetadata.builder()
                                 .openAiApiKey(token)
                                 .organizationId(organizationId)

integration-tests/openai/src/main/java/org/acme/example/openai/chat/ChatLanguageModelResource.java

@@ -14,6 +14,7 @@
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.core.MediaType;
 
+import org.eclipse.microprofile.config.inject.ConfigProperty;
 import org.jboss.resteasy.reactive.RestStreamElementType;
 
 import dev.langchain4j.data.message.AiMessage;
@@ -38,6 +39,9 @@ public class ChatLanguageModelResource {
     private final ChatModel chatLanguageModel;
     private final StreamingChatModel streamingChatLanguageModel;
 
+    @ConfigProperty(name = "quarkus.langchain4j.openai.chat-model.model-name")
+    String modelName;
+
     public ChatLanguageModelResource(ChatModel chatLanguageModel,
             StreamingChatModel streamingChatLanguageModel) {
         this.chatLanguageModel = chatLanguageModel;
@@ -96,14 +100,13 @@ public String template() {
     @Path("memory")
     public String memory() throws Exception {
 
-        TokenCountEstimator tokenizer = new OpenAiTokenCountEstimator("gpt-3.5-turbo");
+        TokenCountEstimator tokenizer = new OpenAiTokenCountEstimator(modelName);
         ChatMemory chatMemory = TokenWindowChatMemory.withMaxTokens(1000, tokenizer);
 
         StringBuffer sb = new StringBuffer();
 
         UserMessage userMessage1 = userMessage(
-                "How do I optimize database queries for a large-scale e-commerce platform? "
-                        + "Answer short in three to five lines maximum.");
+                "Who is the original creator of Linux operation system?");
         chatMemory.add(userMessage1);
 
         sb.append("[User]: ")
@@ -138,9 +141,7 @@ public void onError(Throwable throwable) {
         AiMessage firstAiMessage = futureRef.get().get(60, TimeUnit.SECONDS);
         chatMemory.add(firstAiMessage);
 
-        UserMessage userMessage2 = userMessage(
-                "Give a concrete example implementation of the first point? " +
-                        "Be short, 10 lines of code maximum.");
+        UserMessage userMessage2 = userMessage("In which country was he born?");
         chatMemory.add(userMessage2);
 
         sb.append("\n\n[User]: ")

integration-tests/openai/src/main/java/org/acme/example/openai/embedding/EmbeddingModelResource.java

@@ -26,6 +26,6 @@ public EmbeddingModelResource(EmbeddingModel embeddingModel) {
     @Path("blocking")
     @Produces("text/plain")
     public List<Float> blocking() {
-        return embeddingModel.embed("This is some text").content().vectorAsList();
+        return embeddingModel.embed("What is the last name of the author of Linux?").content().vectorAsList();
     }
 }

integration-tests/openai/src/main/resources/application.properties

@@ -2,6 +2,9 @@ openai.key=ADD_A_TOKEN
 %test.openai.key=ADD_A_TOKEN
 %test.quarkus.langchain4j.openai.base-url= https://mockgpt.wiremockapi.cloud/v1
 
+%cloud-llm.quarkus.langchain4j.openai.base-url=https://api.openai.com/v1
+%cloud-llm.quarkus.langchain4j.openai.chat-model.model-name=gpt-4o-mini
+
 quarkus.langchain4j.openai.api-key=${openai.key}
 quarkus.langchain4j.openai.chat-model.model-name=gpt-4o
 quarkus.langchain4j.timeout=60s

integration-tests/openai/src/test/java/org/acme/example/openai/OpenAiRestApiResourceTest.java

@@ -2,7 +2,6 @@
 
 import static io.restassured.RestAssured.given;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.hamcrest.Matchers.containsString;
 
 import java.net.URL;
 import java.util.ArrayList;
@@ -37,7 +36,7 @@ public void chatSync() {
                 .get("chat/sync")
                 .then()
                 .statusCode(200)
-                .body(containsString("MockGPT"));
+                .body(TestUtils.containsStringOrMock("C++"));
     }
 
     @Test
@@ -48,7 +47,7 @@ public void chatAsync() {
                 .get("chat/async")
                 .then()
                 .statusCode(200)
-                .body(containsString("MockGPT"));
+                .body(TestUtils.containsStringOrMock("Unicode"));
     }
 
     @Test

integration-tests/openai/src/test/java/org/acme/example/openai/TestUtils.java

@@ -0,0 +1,37 @@
+package org.acme.example.openai;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.eclipse.microprofile.config.ConfigProvider;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.hamcrest.Matcher;
+import org.hamcrest.Matchers;
+
+public class TestUtils {
+    private static final String MOCK_ANSWER = "MockGPT";
+
+    @ConfigProperty(name = "openai.key")
+    static String key;
+
+    static {
+        key = ConfigProvider.getConfig().getValue("openai.key", String.class);
+    }
+
+    private static boolean useMock() {
+        return key.equalsIgnoreCase("ADD_A_TOKEN");
+    }
+
+    public static Matcher<String> containsStringOrMock(String... expected) {
+        if (useMock()) {
+            return Matchers.containsString(MOCK_ANSWER);
+        } else {
+            // due to peculiarities of java generics, anyOf method would not accept List<Matcher<String>>
+            List<Matcher<? super String>> possibilities = new ArrayList<>(expected.length);
+            for (String value : expected) {
+                possibilities.add(Matchers.containsStringIgnoringCase(value));
+            }
+            return Matchers.anyOf(possibilities);
+        }
+    }
+}

integration-tests/openai/src/test/java/org/acme/example/openai/aiservices/AssistantResourceTest.java

@@ -1,10 +1,10 @@
 package org.acme.example.openai.aiservices;
 
 import static io.restassured.RestAssured.given;
-import static org.hamcrest.Matchers.containsString;
 
 import java.net.URL;
 
+import org.acme.example.openai.TestUtils;
 import org.junit.jupiter.api.Test;
 
 import io.quarkus.test.common.http.TestHTTPEndpoint;
@@ -22,10 +22,10 @@ public class AssistantResourceTest {
     public void get() {
         given()
                 .baseUri(url.toString())
-                .queryParam("message", "This is a test")
+                .queryParam("message", "Hello, my name is Quarkus")
                 .get()
                 .then()
                 .statusCode(200)
-                .body(containsString("MockGPT"));
+                .body(TestUtils.containsStringOrMock("Quarkus"));
     }
 }

integration-tests/openai/src/test/java/org/acme/example/openai/aiservices/AssistantResourceWithGuardrailsAndObservabilityTest.java

@@ -3,12 +3,12 @@
 import static io.restassured.RestAssured.get;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.awaitility.Awaitility.await;
-import static org.hamcrest.Matchers.containsString;
 
 import java.time.Duration;
 
 import jakarta.inject.Inject;
 
+import org.acme.example.openai.TestUtils;
 import org.acme.example.openai.aiservices.AssistantResourceWithGuardrailsAndObservability.AbstractIGImplementingValidateWithParams;
 import org.acme.example.openai.aiservices.AssistantResourceWithGuardrailsAndObservability.AbstractIGImplementingValidateWithUserMessage;
 import org.acme.example.openai.aiservices.AssistantResourceWithGuardrailsAndObservability.AbstractOGImplementingValidateWithAiMessage;
@@ -41,7 +41,7 @@ static void addSimpleRegistry() {
     void guardrailMetricsAvailable() {
         get("/assistant-with-guardrails-observability").then()
                 .statusCode(200)
-                .body(containsString("MockGPT"));
+                .body(TestUtils.containsStringOrMock("test"));
 
         await()
                 .atMost(Duration.ofSeconds(10))

integration-tests/openai/src/test/java/org/acme/example/openai/aiservices/AssistantResourceWithMetricsTest.java

@@ -1,14 +1,14 @@
 package org.acme.example.openai.aiservices;
 
 import static io.restassured.RestAssured.given;
-import static org.hamcrest.Matchers.containsString;
 import static org.junit.jupiter.api.Assertions.fail;
 
 import java.net.URL;
 import java.util.Collection;
 
 import jakarta.inject.Inject;
 
+import org.acme.example.openai.TestUtils;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
@@ -41,7 +41,7 @@ public void noMicrometerAnnotations() throws InterruptedException {
                 .get("a1")
                 .then()
                 .statusCode(200)
-                .body(containsString("MockGPT"));
+                .body(TestUtils.containsStringOrMock("test"));
 
         waitForMeters(
                 registry.find("langchain4j.aiservices.timed")
@@ -66,7 +66,7 @@ public void micrometerAnnotationOnClass() throws InterruptedException {
                 .get("a2")
                 .then()
                 .statusCode(200)
-                .body(containsString("MockGPT"));
+                .body(TestUtils.containsStringOrMock("test"));
 
         waitForMeters(
                 registry.find("langchain4j.aiservices.timed")
@@ -92,7 +92,7 @@ public void micrometerAnnotationOnMethod() throws InterruptedException {
                 .get("a2c2")
                 .then()
                 .statusCode(200)
-                .body(containsString("MockGPT"));
+                .body(TestUtils.containsStringOrMock("test"));
 
         waitForMeters(registry.find("a2c2-timed").timers(), 1);
         waitForMeters(registry.find("a2c2-counted")