Validate Structured Output Config before execution (#3584)

mzegla · web-flow · commit 0adec4a941de · 2025-08-21T12:27:43.000+02:00
diff --git a/Makefile b/Makefile
@@ -161,11 +161,11 @@ ifeq ($(findstring ubuntu,$(BASE_OS)),ubuntu)
   ifeq ($(BASE_OS_TAG),24.04)
         OS=ubuntu24
 	INSTALL_DRIVER_VERSION ?= "24.52.32224"
-	DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.3.0-19757-b3c74fae04a/openvino_toolkit_ubuntu24_2025.3.0.dev20250812_x86_64.tgz
+	DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.3.0-19785-689ab8b0685/openvino_toolkit_ubuntu24_2025.3.0.dev20250814_x86_64.tgz
   else ifeq  ($(BASE_OS_TAG),22.04)
         OS=ubuntu22
 	INSTALL_DRIVER_VERSION ?= "24.39.31294"
-	DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.3.0-19757-b3c74fae04a/openvino_toolkit_ubuntu22_2025.3.0.dev20250812_x86_64.tgz
+	DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.3.0-19785-689ab8b0685/openvino_toolkit_ubuntu22_2025.3.0.dev20250814_x86_64.tgz
   endif
 endif
 ifeq ($(BASE_OS),redhat)
@@ -174,7 +174,7 @@ ifeq ($(BASE_OS),redhat)
   BASE_IMAGE ?= registry.access.redhat.com/ubi9/ubi:$(BASE_OS_TAG_REDHAT)
   BASE_IMAGE_RELEASE=registry.access.redhat.com/ubi9/ubi-minimal:$(BASE_OS_TAG_REDHAT)
   DIST_OS=redhat
-  DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.3.0-19757-b3c74fae04a/openvino_toolkit_rhel8_2025.3.0.dev20250812_x86_64.tgz
+  DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.3.0-19785-689ab8b0685/openvino_toolkit_rhel8_2025.3.0.dev20250814_x86_64.tgz
   INSTALL_DRIVER_VERSION ?= "24.52.32224"
 endif
 
diff --git a/demos/common/export_models/requirements.txt b/demos/common/export_models/requirements.txt
@@ -4,7 +4,7 @@
 --pre
 optimum-intel
 openvino-tokenizers<=2025.3.0.0.dev20250812
-openvino<=2025.3.0.dev20250812
+openvino<=2025.3.0.dev20250814
 nncf>=2.11.0 
 sentence_transformers
 sentencepiece==0.2.0
diff --git a/src/BUILD b/src/BUILD
@@ -2541,6 +2541,7 @@ cc_test(
         "test/llm/config.json",
         "test/llm/assisted_decoding_config.json",
         "test/llm/lm_cb_regular.pbtxt",
+        "test/llm/lm_cb_with_tool_parser.pbtxt",
         "test/llm/lm_legacy_regular.pbtxt",
         "test/llm/lm_cb_speculative.pbtxt",
         "test/llm/lm_cb_prompt_lookup.pbtxt",
diff --git a/src/llm/io_processing/base_generation_config_builder.cpp b/src/llm/io_processing/base_generation_config_builder.cpp
@@ -29,6 +29,12 @@ void BaseGenerationConfigBuilder::setStructuralTagsConfig(const ov::genai::Struc
     }
 }
 
+void BaseGenerationConfigBuilder::validateStructuredOutputConfig(ov::genai::Tokenizer& tokenizer) {
+    if (config.structured_output_config.has_value()) {
+        config.structured_output_config.value().validate(tokenizer);
+    }
+}
+
 void BaseGenerationConfigBuilder::parseConfigFromRequest(const OpenAIChatCompletionsRequest& request) {
     // Generic
     config.apply_chat_template = false;  // template is applied on the serving side
diff --git a/src/llm/io_processing/base_generation_config_builder.hpp b/src/llm/io_processing/base_generation_config_builder.hpp
@@ -15,6 +15,7 @@
 //*****************************************************************************
 #pragma once
 #include <openvino/genai/generation_config.hpp>
+#include <openvino/genai/tokenizer.hpp>
 #include "../apis/openai_request.hpp"
 
 namespace ovms {
@@ -39,6 +40,10 @@ class BaseGenerationConfigBuilder {
 
     ov::genai::GenerationConfig& getConfig() { return config; }
 
+    // Validates the structured output configuration, if exists.
+    // Throws exception if validation fails.
+    void validateStructuredOutputConfig(ov::genai::Tokenizer& tokenizer);
+
     /*
      * Fills generation config with values read from OpenAI request.
      * If extended, model specific implementation should call base class method first to fill in common configuration
diff --git a/src/llm/io_processing/generation_config_builder.hpp b/src/llm/io_processing/generation_config_builder.hpp
@@ -19,6 +19,7 @@
 #include <iostream>
 #include <string>
 #include <openvino/genai/generation_config.hpp>
+#include <openvino/genai/tokenizer.hpp>
 #include "base_generation_config_builder.hpp"
 #include "phi4/generation_config_builder.hpp"
 #include "llama3/generation_config_builder.hpp"
@@ -58,6 +59,12 @@ class GenerationConfigBuilder {
         return builder_impl->getConfig();
     }
 
+    // Validates the structured output configuration, if exists.
+    // Throws exception if validation fails.
+    void validateStructuredOutputConfig(ov::genai::Tokenizer& tokenizer) {
+        builder_impl->validateStructuredOutputConfig(tokenizer);
+    }
+
     // Fills generation config with values read from OpenAI request
     void parseConfigFromRequest(const OpenAIChatCompletionsRequest& request) {
         builder_impl->parseConfigFromRequest(request);
diff --git a/src/llm/io_processing/hermes3/generation_config_builder.cpp b/src/llm/io_processing/hermes3/generation_config_builder.cpp
@@ -26,6 +26,12 @@ void Hermes3GenerationConfigBuilder::parseConfigFromRequest(const OpenAIChatComp
     // Call the base class method to fill in common configuration
     BaseGenerationConfigBuilder::parseConfigFromRequest(request);
 
+    // For now the only specific part is related to tools, so if there are no tools provided in the request
+    // we can exit early
+    if (request.toolNameSchemaMap.empty()) {
+        return;
+    }
+
     // Set tool guided generation config specific to Hermes3 and Qwen3 models
     ov::genai::StructuralTagsConfig structuralTagsConfig;
     static const std::string toolCallTrigger = "<tool_call>";
diff --git a/src/llm/io_processing/llama3/generation_config_builder.cpp b/src/llm/io_processing/llama3/generation_config_builder.cpp
@@ -26,6 +26,12 @@ void Llama3GenerationConfigBuilder::parseConfigFromRequest(const OpenAIChatCompl
     // Call the base class method to fill in common configuration
     BaseGenerationConfigBuilder::parseConfigFromRequest(request);
 
+    // For now the only specific part is related to tools, so if there are no tools provided in the request
+    // we can exit early
+    if (request.toolNameSchemaMap.empty()) {
+        return;
+    }
+
     // Set tool guided generation config specific to Llama-3 model
     ov::genai::StructuralTagsConfig structuralTagsConfig;
     static const std::string beginOfToolsString = "<|python_tag|>";
diff --git a/src/llm/io_processing/phi4/generation_config_builder.cpp b/src/llm/io_processing/phi4/generation_config_builder.cpp
@@ -26,6 +26,12 @@ void Phi4GenerationConfigBuilder::parseConfigFromRequest(const OpenAIChatComplet
     // Call the base class method to fill in common configuration
     BaseGenerationConfigBuilder::parseConfigFromRequest(request);
 
+    // For now the only specific part is related to tools, so if there are no tools provided in the request
+    // we can exit early
+    if (request.toolNameSchemaMap.empty()) {
+        return;
+    }
+
     // Set tool guided generation config specific to Phi-4 model as described in template from:
     // https://github.com/vllm-project/vllm/blob/v0.9.2/examples/tool_chat_template_phi4_mini.jinja
     ov::genai::StructuralTagsConfig structuralTagsConfig;
diff --git a/src/llm/servable.cpp b/src/llm/servable.cpp
@@ -81,6 +81,12 @@ absl::Status GenAiServable::parseRequest(std::shared_ptr<GenAiServableExecutionC
     }
     executionContext->generationConfigBuilder = std::make_shared<GenerationConfigBuilder>(getProperties()->baseGenerationConfig, getProperties()->toolParserName, getProperties()->enableToolGuidedGeneration);
     executionContext->generationConfigBuilder->parseConfigFromRequest(executionContext->apiHandler->getRequest());
+    try {
+        executionContext->generationConfigBuilder->validateStructuredOutputConfig(getProperties()->tokenizer);
+    } catch (const std::exception& e) {
+        SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Failed to validate structured output config: {}", e.what());
+        return absl::Status(absl::StatusCode::kInvalidArgument, "Invalid structured output config. Check if JSON schemas in your request are correct.");
+    }
     return absl::OkStatus();
 }
 
diff --git a/src/test/llm/config.json b/src/test/llm/config.json
@@ -13,6 +13,10 @@
         "name":"lm_cb_regular",
         "graph_path":"/ovms/src/test/llm/lm_cb_regular.pbtxt"
     },
+    {
+        "name":"lm_cb_with_tool_parser",
+        "graph_path":"/ovms/src/test/llm/lm_cb_with_tool_parser.pbtxt"
+    },
     {
         "name":"lm_legacy_regular",
         "graph_path":"/ovms/src/test/llm/lm_legacy_regular.pbtxt"
diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp
@@ -1080,6 +1080,84 @@ TEST_P(LLMFlowHttpTestParameterized, unaryStructuredOutput) {
     }
 }
 
+TEST_P(LLMFlowHttpTestParameterized, unaryStructuredOutputBadSchema) {
+    auto params = GetParam();
+    std::string requestBody = R"(
+        {
+            "model": ")" + params.modelName +
+                              R"(",
+            "stream": false,
+            "seed" : 1,
+            "temperature": 0.0,
+            "messages": [
+            {"role": "user",  "content": "Extract the name and age of the person from the text and structure the output in JSON format. Margaret is 20 years old."}
+            ],
+                "response_format": {
+                    "type": "json_schema",
+                    "json_schema": {
+                        "schema": {
+                        "type": "object",
+                        "properties": {
+                            "name": {
+                            "type": "string"
+                            },
+                            "age": {
+                            "type": "my_integer"
+                            }
+                        },
+                        "required": ["name", "age"]
+                        }
+                    }
+                }
+        }
+    )";
+
+    ASSERT_EQ(
+        handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser),
+        ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR);
+}
+
+TEST_P(LLMFlowHttpTestParameterized, unaryToolBadSchema) {
+    std::string requestBody = R"(
+    {
+        "model": "lm_cb_with_tool_parser",
+        "stream": false,
+        "temperature": 0,
+        "tools": [
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_horoscope",
+                    "description": "Get today's horoscope for an astrological sign.",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "sign": {
+                                "type": "string_with_a_suffix",
+                                "description": "An astrological sign like Taurus or Aquarius"
+                            }
+                        },
+                        "required": [
+                            "sign"
+                        ]
+                    }
+                }
+            }
+        ],
+        "messages": [
+            {
+                "role": "user",
+                "content": "What is my horoscope? I am an Aquarius."
+            }
+        ]
+    }
+    )";
+
+    ASSERT_EQ(
+        handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser),
+        ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR);
+}
+
 TEST_P(LLMFlowHttpTestParameterized, unaryCompletionsJsonLogprobs) {
     auto params = GetParam();
     // TODO: In the next step we should break this suite into smaller ones, use proper configuration instead of skipping
diff --git a/src/test/llm/lm_cb_with_tool_parser.pbtxt b/src/test/llm/lm_cb_with_tool_parser.pbtxt
@@ -0,0 +1,47 @@
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+input_stream: "HTTP_REQUEST_PAYLOAD:input"
+output_stream: "HTTP_RESPONSE_PAYLOAD:output"
+node {
+    name: "llmNode1"
+    calculator: "HttpLLMCalculator"
+    input_side_packet: "LLM_NODE_RESOURCES:llm"
+    input_stream: "LOOPBACK:loopback"
+    input_stream: "HTTP_REQUEST_PAYLOAD:input"
+    output_stream: "LOOPBACK:loopback"
+    output_stream: "HTTP_RESPONSE_PAYLOAD:output"
+    input_stream_info: {
+      tag_index: 'LOOPBACK:0',
+      back_edge: true
+    }
+    node_options: {
+        [type.googleapis.com/mediapipe.LLMCalculatorOptions]: {
+          models_path: "/ovms/src/test/llm_testing/facebook/opt-125m"
+          cache_size: 1,
+          tool_parser: "phi4" # fake tool parser to enable negative testing for tools
+          enable_tool_guided_generation: true
+        }
+    }
+    input_stream_handler {
+      input_stream_handler: "SyncSetInputStreamHandler",
+      options {
+        [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
+          sync_set {
+            tag_index: "LOOPBACK:0"
+          }
+        }
+      }
+    }
+}
diff --git a/third_party/llm_engine/llm_engine.bzl b/third_party/llm_engine/llm_engine.bzl
@@ -24,7 +24,7 @@ def llm_engine():
     new_git_repository(
         name = "llm_engine",
         remote = "https://github.com/openvinotoolkit/openvino.genai",
-        commit = "cf847adc3f21a09e171874466ba2619615a6e92e", # master 2025-08-13
+        commit = "f7c138cd9407c8c41b7191bb758ff7c280162b2e", # releases/2025/3 2025-08-14
         build_file = "@_llm_engine//:BUILD",
         init_submodules = True,
         recursive_init_submodules = True,
diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat
@@ -137,7 +137,7 @@ IF /I EXIST %bash_path% (
 ::::::::::::::::::::::: GENAI/OPENVINO - reinstalled per build trigger
 :: Set default GENAI_PACKAGE_URL if not set
 if "%GENAI_PACKAGE_URL%"=="" (
-    set "GENAI_PACKAGE_URL=https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.3.0.0.dev20250812/openvino_genai_windows_2025.3.0.0.dev20250812_x86_64.zip"
+    set "GENAI_PACKAGE_URL=https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.3.0.0.dev20250814/openvino_genai_windows_2025.3.0.0.dev20250814_x86_64.zip"
 )
 
 :: Extract genai_ver from GENAI_PACKAGE_URL (filename)

Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,12 @@ void BaseGenerationConfigBuilder::setStructuralTagsConfig(const ov::genai::Struc`
`29`	`29`	`}`
`30`	`30`	`}`
`31`	`31`
	`32`	`+void BaseGenerationConfigBuilder::validateStructuredOutputConfig(ov::genai::Tokenizer& tokenizer) {`
	`33`	`+ if (config.structured_output_config.has_value()) {`
	`34`	`+ config.structured_output_config.value().validate(tokenizer);`
	`35`	`+ }`
	`36`	`+}`
	`37`	`+`
`32`	`38`	`void BaseGenerationConfigBuilder::parseConfigFromRequest(const OpenAIChatCompletionsRequest& request) {`
`33`	`39`	`// Generic`
`34`	`40`	`config.apply_chat_template = false; // template is applied on the serving side`
Original file line number	Diff line number	Diff line change
`@@ -81,6 +81,12 @@ absl::Status GenAiServable::parseRequest(std::shared_ptr<GenAiServableExecutionC`
`81`	`81`	`}`
`82`	`82`	`executionContext->generationConfigBuilder = std::make_shared<GenerationConfigBuilder>(getProperties()->baseGenerationConfig, getProperties()->toolParserName, getProperties()->enableToolGuidedGeneration);`
`83`	`83`	`executionContext->generationConfigBuilder->parseConfigFromRequest(executionContext->apiHandler->getRequest());`
	`84`	`+ try {`
	`85`	`+ executionContext->generationConfigBuilder->validateStructuredOutputConfig(getProperties()->tokenizer);`
	`86`	`+ } catch (const std::exception& e) {`
	`87`	`+ SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Failed to validate structured output config: {}", e.what());`
	`88`	`+ return absl::Status(absl::StatusCode::kInvalidArgument, "Invalid structured output config. Check if JSON schemas in your request are correct.");`
	`89`	`+ }`
`84`	`90`	`return absl::OkStatus();`
`85`	`91`	`}`
`86`	`92`