fix: Update model lists and model availability test (#348)

TillK17 · I750911 · bot-sdk-js · web-flow · commit b194909e376f · 2025-02-28T11:09:31.000+01:00
* deprecated gpt-3.5-turbo-16k and added o1 and o3-mini to orchestration

* deprecated gpt-3.5-turbo-16k and added o1 and o3-mini to openAI

* adapted test to check for deprecation of models

* Formatting

* update of deprecated gpt-3.5-turbo-16k to gpt 4o

* Formatting

* Deleting o1 and o3-mini again

* Extend ScenarioTest

* Deprecate models

* fix 1 warning

* add missing deprecation warning in javadoc

* Fix merge and update test

* Formatting

---------

Co-authored-by: I750911 &lt;till.kurek@sap.com&gt;
Co-authored-by: SAP Cloud SDK Bot &lt;cloudsdk@sap.com&gt;
Co-authored-by: Jonas Israel &lt;jonas.israel@sap.com&gt;
Co-authored-by: Alexander Dümont &lt;alexander_duemont@web.de&gt;
Co-authored-by: Charles Dubois &lt;103174266+CharlesDuboisSAP@users.noreply.github.com&gt;
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java
@@ -19,20 +19,38 @@ public record OpenAiModel(@Nonnull String name, @Nullable String version) implem
   /** Azure OpenAI dall-e-3 image generate model */
   public static final OpenAiModel DALL_E_3 = new OpenAiModel("dall-e-3", null);
 
-  /** Azure OpenAI GPT-3.5 Turbo chat completions model */
-  public static final OpenAiModel GPT_35_TURBO = new OpenAiModel("gpt-35-turbo", null);
+  /**
+   * Azure OpenAI GPT-3.5 Turbo chat completions model
+   *
+   * @deprecated This model is not usable anymore. It is retired on AI Core since 2024-11-17.
+   */
+  @Deprecated public static final OpenAiModel GPT_35_TURBO = new OpenAiModel("gpt-35-turbo", null);
 
-  /** Azure OpenAI GPT-3.5 Turbo chat completions model */
+  /**
+   * Azure OpenAI GPT-3.5 Turbo chat completions model
+   *
+   * @deprecated This model is not usable anymore. It is retired on AI Core since 2025-02-22.
+   */
+  @Deprecated
   public static final OpenAiModel GPT_35_TURBO_1025 = new OpenAiModel("gpt-35-turbo-0125", null);
 
-  /** Azure OpenAI GPT-3.5 Turbo chat completions model */
+  /**
+   * Azure OpenAI GPT-3.5 Turbo chat completions model
+   *
+   * @deprecated This model is not usable anymore. It is retired on AI Core since 2025-02-13.
+   */
+  @Deprecated
   public static final OpenAiModel GPT_35_TURBO_16K = new OpenAiModel("gpt-35-turbo-16k", null);
 
   /** Azure OpenAI GPT-4 chat completions model */
   public static final OpenAiModel GPT_4 = new OpenAiModel("gpt-4", null);
 
-  /** Azure OpenAI GPT-4-32k chat completions model */
-  public static final OpenAiModel GPT_4_32K = new OpenAiModel("gpt-4-32k", null);
+  /**
+   * Azure OpenAI GPT-4-32k chat completions model
+   *
+   * @deprecated This model will not be usable anymore on 2025-05-30.
+   */
+  @Deprecated public static final OpenAiModel GPT_4_32K = new OpenAiModel("gpt-4-32k", null);
 
   /** Azure OpenAI GPT-4o chat completions model */
   public static final OpenAiModel GPT_4O = new OpenAiModel("gpt-4o", null);
@@ -54,7 +72,12 @@ public record OpenAiModel(@Nonnull String name, @Nullable String version) implem
   public static final OpenAiModel TEXT_EMBEDDING_3_SMALL =
       new OpenAiModel("text-embedding-3-small", null);
 
-  /** Azure OpenAI Text Embedding ADA 002 model */
+  /**
+   * Azure OpenAI Text Embedding ADA 002 model
+   *
+   * @deprecated This model is deprecated on AI Core.
+   */
+  @Deprecated
   public static final OpenAiModel TEXT_EMBEDDING_ADA_002 =
       new OpenAiModel("text-embedding-ada-002", null);
 
diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java
@@ -102,7 +102,12 @@ public class OrchestrationAiModel {
   /** Azure OpenAI GPT-3.5 Turbo chat completions model */
   public static final OrchestrationAiModel GPT_35_TURBO = new OrchestrationAiModel("gpt-35-turbo");
 
-  /** Azure OpenAI GPT-3.5 Turbo chat completions model */
+  /**
+   * Azure OpenAI GPT-3.5 Turbo chat completions model
+   *
+   * @deprecated This model is not usable anymore. It is retired on AI Core since 2025-02-13.
+   */
+  @Deprecated
   public static final OrchestrationAiModel GPT_35_TURBO_16K =
       new OrchestrationAiModel("gpt-35-turbo-16k");
 
diff --git a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformerTest.java b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformerTest.java
@@ -1,6 +1,6 @@
 package com.sap.ai.sdk.orchestration;
 
-import static com.sap.ai.sdk.orchestration.OrchestrationUnitTest.CUSTOM_GPT_35;
+import static com.sap.ai.sdk.orchestration.OrchestrationUnitTest.CUSTOM_GPT_4O;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
@@ -80,7 +80,7 @@ void testMessagesHistory() {
     var prompt = new OrchestrationPrompt("bar").messageHistory(List.of(systemMessage));
     var actual =
         ConfigToRequestTransformer.toCompletionPostRequest(
-            prompt, new OrchestrationModuleConfig().withLlmConfig(CUSTOM_GPT_35));
+            prompt, new OrchestrationModuleConfig().withLlmConfig(CUSTOM_GPT_4O));
 
     assertThat(actual.getMessagesHistory()).containsExactly(systemMessage.createChatMessage());
   }
diff --git a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java
@@ -16,7 +16,7 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.verify;
 import static com.sap.ai.sdk.orchestration.AzureFilterThreshold.ALLOW_SAFE;
 import static com.sap.ai.sdk.orchestration.AzureFilterThreshold.ALLOW_SAFE_LOW_MEDIUM;
-import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.GPT_35_TURBO_16K;
+import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.GPT_4O;
 import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.GPT_4O_MINI;
 import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.Parameter.*;
 import static org.apache.hc.core5.http.HttpStatus.SC_BAD_REQUEST;
@@ -82,8 +82,8 @@
  */
 @WireMockTest
 class OrchestrationUnitTest {
-  static final OrchestrationAiModel CUSTOM_GPT_35 =
-      GPT_35_TURBO_16K
+  static final OrchestrationAiModel CUSTOM_GPT_4O =
+      GPT_4O
           .withParam(MAX_TOKENS, 50)
           .withParam(TEMPERATURE, 0.1)
           .withParam(FREQUENCY_PENALTY, 0)
@@ -103,7 +103,7 @@ void setup(WireMockRuntimeInfo server) {
     final DefaultHttpDestination destination =
         DefaultHttpDestination.builder(server.getHttpBaseUrl()).build();
     client = new OrchestrationClient(destination);
-    config = new OrchestrationModuleConfig().withLlmConfig(CUSTOM_GPT_35);
+    config = new OrchestrationModuleConfig().withLlmConfig(CUSTOM_GPT_4O);
     prompt = new OrchestrationPrompt("Hello World! Why is this phrase so famous?");
     ApacheHttpClient5Accessor.setHttpClientCache(ApacheHttpClient5Cache.DISABLED);
   }
diff --git a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModelTest.java b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModelTest.java
@@ -9,7 +9,7 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
 import static com.github.tomakehurst.wiremock.client.WireMock.verify;
 import static com.github.tomakehurst.wiremock.stubbing.Scenario.STARTED;
-import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.GPT_35_TURBO_16K;
+import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.GPT_4O;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.ArgumentMatchers.any;
@@ -61,8 +61,7 @@ void setup(WireMockRuntimeInfo server) {
         DefaultHttpDestination.builder(server.getHttpBaseUrl()).build();
     client = new OrchestrationChatModel(new OrchestrationClient(destination));
     defaultOptions =
-        new OrchestrationChatOptions(
-            new OrchestrationModuleConfig().withLlmConfig(GPT_35_TURBO_16K));
+        new OrchestrationChatOptions(new OrchestrationModuleConfig().withLlmConfig(GPT_4O));
     prompt = new Prompt("Hello World! Why is this phrase so famous?", defaultOptions);
     ApacheHttpClient5Accessor.setHttpClientCache(ApacheHttpClient5Cache.DISABLED);
   }
diff --git a/orchestration/src/test/resources/filteringLooseRequest.json b/orchestration/src/test/resources/filteringLooseRequest.json
@@ -2,7 +2,7 @@
   "orchestration_config": {
     "module_configurations": {
       "llm_module_config": {
-        "model_name": "gpt-35-turbo-16k",
+        "model_name": "gpt-4o",
         "model_params": {
           "temperature": 0.1,
           "max_tokens": 50,
diff --git a/orchestration/src/test/resources/groundingRequest.json b/orchestration/src/test/resources/groundingRequest.json
@@ -2,7 +2,7 @@
   "orchestration_config" : {
     "module_configurations" : {
       "llm_module_config" : {
-        "model_name" : "gpt-35-turbo-16k",
+        "model_name" : "gpt-4o",
         "model_params" : {
           "max_tokens" : 50,
           "temperature" : 0.1,
diff --git a/orchestration/src/test/resources/maskingRequest.json b/orchestration/src/test/resources/maskingRequest.json
@@ -2,7 +2,7 @@
   "orchestration_config": {
     "module_configurations": {
       "llm_module_config": {
-        "model_name": "gpt-35-turbo-16k",
+        "model_name": "gpt-4o",
         "model_params": {
           "presence_penalty": 0,
           "frequency_penalty": 0,
diff --git a/orchestration/src/test/resources/messagesHistoryRequest.json b/orchestration/src/test/resources/messagesHistoryRequest.json
@@ -2,7 +2,7 @@
   "orchestration_config": {
     "module_configurations": {
       "llm_module_config": {
-        "model_name": "gpt-35-turbo-16k",
+        "model_name": "gpt-4o",
         "model_params": {
           "presence_penalty": 0,
           "frequency_penalty": 0,
diff --git a/orchestration/src/test/resources/templatingRequest.json b/orchestration/src/test/resources/templatingRequest.json
@@ -12,7 +12,7 @@
         "tools" : [ ]
       },
       "llm_module_config": {
-        "model_name": "gpt-35-turbo-16k",
+        "model_name": "gpt-4o",
         "model_params": {
           "max_tokens": 50,
           "temperature": 0.1,
diff --git a/orchestration/src/test/resources/toolCallsRequest.json b/orchestration/src/test/resources/toolCallsRequest.json
@@ -2,7 +2,7 @@
   "orchestration_config": {
     "module_configurations": {
       "llm_module_config": {
-        "model_name": "gpt-35-turbo-16k",
+        "model_name": "gpt-4o",
         "model_params": {},
         "model_version": "latest"
       },
diff --git a/orchestration/src/test/resources/toolCallsRequest2.json b/orchestration/src/test/resources/toolCallsRequest2.json
@@ -2,7 +2,7 @@
   "orchestration_config": {
     "module_configurations": {
       "llm_module_config": {
-        "model_name": "gpt-35-turbo-16k",
+        "model_name": "gpt-4o",
         "model_params": {},
         "model_version": "latest"
       },
diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/ScenarioTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/ScenarioTest.java
@@ -3,11 +3,13 @@
 import static org.assertj.core.api.Assertions.assertThat;
 
 import com.sap.ai.sdk.core.model.AiModelBaseData;
+import com.sap.ai.sdk.core.model.AiModelVersion;
 import com.sap.ai.sdk.foundationmodels.openai.OpenAiModel;
 import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Optional;
 import lombok.SneakyThrows;
+import org.assertj.core.api.SoftAssertions;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 
@@ -25,21 +27,45 @@ void openAiModelAvailability() {
     final var availableOpenAiModels =
         aiModelList.stream()
             .filter(model -> model.getExecutableId().equals("azure-openai"))
-            .map(AiModelBaseData::getModel)
-            .toList();
+            .collect(
+                () -> new HashMap<String, Boolean>(),
+                (list, model) -> list.put(model.getModel(), isDeprecated(model)),
+                HashMap::putAll);
 
     // Gather our declared OpenAI models
     Field[] declaredFields = OpenAiModel.class.getFields();
 
     // get the models from the OpenAiModel class
-    List<String> declaredOpenAiModelList = new ArrayList<>();
+    HashMap<String, Boolean> declaredOpenAiModelList = new HashMap<>();
     for (Field field : declaredFields) {
       if (field.getType().equals(OpenAiModel.class)) {
-        declaredOpenAiModelList.add(((OpenAiModel) field.get(null)).name());
+        declaredOpenAiModelList.put(
+            ((OpenAiModel) field.get(null)).name(), field.isAnnotationPresent(Deprecated.class));
       }
     }
 
     // Assert that the declared OpenAI models match the expected list
-    assertThat(declaredOpenAiModelList).containsAll(availableOpenAiModels);
+    assertThat(declaredOpenAiModelList.keySet()).containsAll(availableOpenAiModels.keySet());
+
+    SoftAssertions softly = new SoftAssertions();
+    for (var model : availableOpenAiModels.entrySet()) {
+      Boolean declaredDeprecated = declaredOpenAiModelList.get(model.getKey());
+      softly
+          .assertThat(declaredDeprecated)
+          .withFailMessage(
+              "%s is deprecated:%s on AI Core but deprecated:%s in AI SDK",
+              model.getKey(), model.getValue(), declaredDeprecated)
+          .isEqualTo(model.getValue());
+    }
+    softly.assertAll();
+  }
+
+  private static boolean isDeprecated(AiModelBaseData model) {
+    Optional<AiModelVersion> version =
+        model.getVersions().stream().filter(AiModelVersion::isIsLatest).findFirst();
+    if (version.isEmpty()) {
+      throw new RuntimeException();
+    }
+    return version.get().isDeprecated();
   }
 }
