Implement new syntax for RERANK and COMPLETION

Author: afoucret

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestRerankTestCase.java

@@ -88,7 +88,7 @@ public void testRerankWithSingleField() throws IOException {
         String query = """
             FROM rerank-test-index
             | WHERE match(title, "exploration")
-            | RERANK "exploration" ON title WITH test_reranker
+            | RERANK "exploration" ON title OPTIONS inferenceId=test_reranker
             | EVAL _score = ROUND(_score, 5)
             """;
 
@@ -107,7 +107,7 @@ public void testRerankWithMultipleFields() throws IOException {
         String query = """
             FROM rerank-test-index
             | WHERE match(title, "exploration")
-            | RERANK "exploration" ON title, author WITH test_reranker
+            | RERANK "exploration" ON title, author OPTIONS inferenceId=test_reranker
             | EVAL _score = ROUND(_score, 5)
             """;
 
@@ -126,7 +126,7 @@ public void testRerankWithPositionalParams() throws IOException {
         String query = """
             FROM rerank-test-index
             | WHERE match(title, "exploration")
-            | RERANK ? ON title WITH ?
+            | RERANK ? ON title OPTIONS inferenceId=?
             | EVAL _score = ROUND(_score, 5)
             """;
 
@@ -145,7 +145,7 @@ public void testRerankWithNamedParams() throws IOException {
         String query = """
             FROM rerank-test-index
             | WHERE match(title, ?queryText)
-            | RERANK ?queryText ON title WITH ?inferenceId
+            | RERANK ?queryText ON title OPTIONS inferenceId=?inferenceId
             | EVAL _score = ROUND(_score, 5)
             """;
 
@@ -164,7 +164,7 @@ public void testRerankWithMissingInferenceId() {
         String query = """
             FROM rerank-test-index
             | WHERE match(title, "exploration")
-            | RERANK "exploration" ON title WITH test_missing
+            | RERANK "exploration" ON title OPTIONS inferenceId=test_missing
             | EVAL _score = ROUND(_score, 5)
             """;
 

x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec

@@ -6,7 +6,7 @@ completion using a ROW source operator
 required_capability: completion
 
 ROW prompt="Who is Victor Hugo?"
-| COMPLETION completion_output = prompt WITH test_completion
+| COMPLETION prompt INTO completion_output OPTIONS inferenceId=test_completion
 ;
 
 prompt:keyword      | completion_output:keyword
@@ -18,7 +18,7 @@ completion using a ROW source operator and prompt is a multi-valued field
 required_capability: completion
 
 ROW prompt=["Answer the following question:", "Who is Victor Hugo?"]
-| COMPLETION completion_output = prompt WITH test_completion
+| COMPLETION prompt INTO completion_output OPTIONS inferenceId=test_completion
 ;
 
 prompt:keyword                                        | completion_output:keyword
@@ -34,7 +34,7 @@ FROM books METADATA _score
 | WHERE title:"war and peace" AND author:"Tolstoy"
 | SORT _score DESC
 | LIMIT 2
-| COMPLETION title WITH test_completion
+| COMPLETION title OPTIONS inferenceId=test_completion
 | KEEP title, completion
 ;
 
@@ -51,7 +51,7 @@ FROM books METADATA _score
 | WHERE title:"war and peace" AND author:"Tolstoy"
 | SORT _score DESC
 | LIMIT 2
-| COMPLETION CONCAT("This is a prompt: ", title) WITH test_completion
+| COMPLETION CONCAT("This is a prompt: ", title) OPTIONS inferenceId=test_completion
 | KEEP title, completion
 ;
 

x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec

@@ -781,7 +781,7 @@ FROM employees
 | KEEP emp_no, first_name, last_name
 | FORK (WHERE emp_no == 10048 OR emp_no == 10081)
        (WHERE emp_no == 10081 OR emp_no == 10087)
-| COMPLETION x = CONCAT(first_name, " ", last_name) WITH test_completion
+| COMPLETION CONCAT(first_name, " ", last_name) INTO x OPTIONS inferenceId=test_completion
 | SORT _fork, emp_no
 ;
 
@@ -799,7 +799,7 @@ required_capability: completion
 FROM employees
 | KEEP emp_no, first_name, last_name
 | FORK (WHERE emp_no == 10048 OR emp_no == 10081
-       | COMPLETION x = CONCAT(first_name, " ", last_name) WITH test_completion)
+       | COMPLETION CONCAT(first_name, " ", last_name) INTO x OPTIONS inferenceId=test_completion)
        (WHERE emp_no == 10081 OR emp_no == 10087)
 | SORT _fork, emp_no
 ;
@@ -817,7 +817,7 @@ required_capability: completion
 
 FROM employees
 | KEEP emp_no, first_name, last_name
-| COMPLETION x = CONCAT(first_name, " ", last_name) WITH test_completion
+| COMPLETION CONCAT(first_name, " ", last_name) INTO x OPTIONS inferenceId=test_completion
 | FORK (WHERE emp_no == 10048 OR emp_no == 10081)
        (WHERE emp_no == 10081 OR emp_no == 10087)
 | SORT _fork, emp_no

x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec

@@ -10,7 +10,7 @@ required_capability: match_operator_colon
 FROM books METADATA _score
 | WHERE title:"war and peace" AND author:"Tolstoy"
 | SORT _score DESC, book_no ASC
-| RERANK "war and peace" ON title WITH inferenceId=test_reranker
+| RERANK "war and peace" ON title OPTIONS inferenceId=test_reranker
 | EVAL _score=ROUND(_score, 2)
 | KEEP book_no, title, author, _score
 ;
@@ -29,7 +29,7 @@ required_capability: match_operator_colon
 FROM books METADATA _score
 | WHERE title:"war and peace" AND author:"Tolstoy"
 | SORT _score DESC, book_no ASC
-| RERANK "war and peace" ON title WITH inferenceId=test_reranker, scoreColumn=rerank_score
+| RERANK "war and peace" ON title INTO rerank_score OPTIONS inferenceId=test_reranker
 | EVAL _score=ROUND(_score, 2), rerank_score=ROUND(rerank_score, 2)
 | KEEP book_no, title, author, rerank_score
 ;
@@ -48,7 +48,7 @@ required_capability: match_operator_colon
 FROM books METADATA _score
 | WHERE title:"war and peace" AND author:"Tolstoy"
 | SORT _score DESC
-| RERANK "war and peace" ON title WITH inferenceId=test_reranker, scoreColumn=rerank_score
+| RERANK "war and peace" ON title INTO rerank_score OPTIONS inferenceId=test_reranker
 | EVAL _score=ROUND(_score, 2), rerank_score=ROUND(rerank_score, 2)
 | SORT rerank_score, _score ASC, book_no ASC
 | KEEP book_no, title, author, rerank_score
@@ -68,7 +68,7 @@ required_capability: match_operator_colon
 
 FROM books METADATA _score
 | WHERE title:"war and peace" AND author:"Tolstoy"
-| RERANK "war and peace" ON title, author WITH inferenceId=test_reranker
+| RERANK "war and peace" ON title, author OPTIONS inferenceId=test_reranker
 | EVAL _score=ROUND(_score, 2)
 | SORT _score DESC, book_no ASC
 | KEEP book_no, title, author, _score
@@ -90,7 +90,7 @@ FROM books METADATA _score
 | WHERE title:"war and peace" AND author:"Tolstoy"
 | SORT _score DESC, book_no ASC
 | LIMIT 3
-| RERANK "war and peace" ON title WITH inferenceId=test_reranker
+| RERANK "war and peace" ON title OPTIONS inferenceId=test_reranker
 | EVAL _score=ROUND(_score, 2)
 | SORT _score DESC, book_no ASC
 | KEEP book_no, title, author, _score
@@ -109,7 +109,7 @@ required_capability: match_operator_colon
 
 FROM books METADATA _score
 | WHERE title:"war and peace" AND author:"Tolstoy"
-| RERANK "war and peace" ON title WITH inferenceId=test_reranker
+| RERANK "war and peace" ON title OPTIONS inferenceId=test_reranker
 | EVAL _score=ROUND(_score, 2)
 | SORT _score DESC, book_no ASC
 | KEEP book_no, title, author, _score
@@ -129,7 +129,7 @@ required_capability: match_operator_colon
 
 FROM books
 | WHERE title:"war and peace" AND author:"Tolstoy"
-| RERANK "war and peace" ON title WITH inferenceId=test_reranker
+| RERANK "war and peace" ON title OPTIONS inferenceId=test_reranker
 | EVAL _score=ROUND(_score, 2)
 | KEEP book_no, title, author, _score
 | SORT author, title 
@@ -153,7 +153,7 @@ FROM books METADATA _id, _index, _score
 | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 )
        ( WHERE author:"Tolkien" | SORT _score, _id DESC | LIMIT 3 )
 | RRF
-| RERANK "Tolkien" ON title WITH inferenceId=test_reranker
+| RERANK "Tolkien" ON title OPTIONS inferenceId=test_reranker
 | EVAL _score=ROUND(_score, 2)
 | SORT _score DESC, book_no ASC
 | LIMIT 2