Adding template validation prior to request flow

Author: jonathan-buttner

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/ValidatingSubstitutor.java

@@ -51,7 +51,12 @@ private static void ensureNoMorePlaceholdersExist(String substitutedString, Stri
         Matcher matcher = VARIABLE_PLACEHOLDER_PATTERN.matcher(substitutedString);
         if (matcher.find()) {
             throw new IllegalStateException(
-                Strings.format("Found placeholder [%s] in field [%s] after replacement call", matcher.group(), field)
+                Strings.format(
+                    "Found placeholder [%s] in field [%s] after replacement call, "
+                        + "please check that all templates have a corresponding field definition.",
+                    matcher.group(),
+                    field
+                )
             );
         }
     }

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/custom/CustomService.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.xpack.inference.services.SenderService;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
 import org.elasticsearch.xpack.inference.services.ServiceUtils;
+import org.elasticsearch.xpack.inference.services.custom.request.CustomRequest;
 
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -94,12 +95,32 @@ public void parseRequestConfig(
             throwIfNotEmptyMap(serviceSettingsMap, NAME);
             throwIfNotEmptyMap(taskSettingsMap, NAME);
 
+            validateConfiguration(model);
+
             parsedModelListener.onResponse(model);
         } catch (Exception e) {
             parsedModelListener.onFailure(e);
         }
     }
 
+    /**
+     * This does some initial validation with mock inputs to determine if any templates are missing a field to fill them.
+     */
+    private static void validateConfiguration(CustomModel model) {
+        String query = null;
+        if (model.getTaskType() == TaskType.RERANK) {
+            query = "test query";
+        }
+
+        try {
+            new CustomRequest(query, List.of("test input"), model).createHttpRequest();
+        } catch (Exception e) {
+            var validationException = new ValidationException();
+            validationException.addValidationError(Strings.format("Failed to validate model configuration: %s", e.getMessage()));
+            throw validationException;
+        }
+    }
+
     @Override
     public InferenceServiceConfiguration getConfiguration() {
         return Configuration.get();

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/custom/CustomServiceTests.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference.services.custom;
 
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
@@ -44,6 +45,7 @@
 import java.util.Map;
 
 import static org.elasticsearch.xpack.inference.Utils.TIMEOUT;
+import static org.elasticsearch.xpack.inference.Utils.getRequestConfigMap;
 import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
 import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
 import static org.elasticsearch.xpack.inference.services.custom.response.RerankResponseParser.RERANK_PARSER_DOCUMENT_TEXT;
@@ -546,4 +548,47 @@ public void testInfer_HandlesSparseEmbeddingRequest_Alibaba_Format() throws IOEx
             );
         }
     }
+
+    public void testParseRequestConfig_ThrowsAValidationError_WhenReplacementDoesNotFillTemplate() throws Exception {
+        try (var service = createService(threadPool, clientManager)) {
+
+            var settingsMap = new HashMap<>(
+                Map.of(
+                    CustomServiceSettings.URL,
+                    "http://www.abc.com",
+                    CustomServiceSettings.HEADERS,
+                    Map.of("key", "value"),
+                    QueryParameters.QUERY_PARAMETERS,
+                    List.of(List.of("key", "value")),
+                    CustomServiceSettings.REQUEST,
+                    "request body ${some_template}",
+                    CustomServiceSettings.RESPONSE,
+                    new HashMap<>(
+                        Map.of(
+                            CustomServiceSettings.JSON_PARSER,
+                            createResponseParserMap(TaskType.COMPLETION),
+                            CustomServiceSettings.ERROR_PARSER,
+                            new HashMap<>(Map.of(ErrorResponseParser.MESSAGE_PATH, "$.error.message"))
+                        )
+                    )
+                )
+            );
+
+            var config = getRequestConfigMap(settingsMap, createTaskSettingsMap(), createSecretSettingsMap());
+
+            var listener = new PlainActionFuture<Model>();
+            service.parseRequestConfig("id", TaskType.COMPLETION, config, listener);
+
+            var exception = expectThrows(ValidationException.class, () -> listener.actionGet(TIMEOUT));
+
+            assertThat(
+                exception.getMessage(),
+                is(
+                    "Validation Failed: 1: Failed to validate model configuration: Found placeholder "
+                        + "[${some_template}] in field [request] after replacement call, please check that all "
+                        + "templates have a corresponding field definition.;"
+                )
+            );
+        }
+    }
 }