[v3.0.0] Deprecate OpenSearch DSL format (#3367)

seankao31 · web-flow · commit f750bc67267e · 2025-03-10T13:12:31.000-07:00
* mark places to change Signed-off-by: Sean Kao <seankao@amazon.com> * remove dsl format from doc Signed-off-by: Sean Kao <seankao@amazon.com> * remove Format.JSON Signed-off-by: Sean Kao <seankao@amazon.com> * fix test case wip Signed-off-by: Sean Kao <seankao@amazon.com> * Revert "fix test case wip" This reverts commit e3157aa. Signed-off-by: Sean Kao <seankao@amazon.com> * ignore legacy test using json response format Signed-off-by: Sean Kao <seankao@amazon.com> * spotlessApply Signed-off-by: Sean Kao <seankao@amazon.com> * remove json param check in executeRequest Signed-off-by: Sean Kao <seankao@amazon.com> * fix getScriptFieldFromQuery can't get requestBuilder from QueryPlan Signed-off-by: Sean Kao <seankao@amazon.com> * spotlessApply Signed-off-by: Sean Kao <seankao@amazon.com> * ignore test using json Signed-off-by: Sean Kao <seankao@amazon.com> * fix test case Signed-off-by: Sean Kao <seankao@amazon.com> * ignore legacy tests affected Signed-off-by: Sean Kao <seankao@amazon.com> * fix json call Signed-off-by: Sean Kao <seankao@amazon.com> * remove patch file Signed-off-by: Sean Kao <seankao@amazon.com> * comment Signed-off-by: Sean Kao <seankao@amazon.com> * remove unneeded file Signed-off-by: Sean Kao <seankao@amazon.com> * delete unused class Signed-off-by: Sean Kao <seankao@amazon.com> * revert change in prod code for test purpose; fix test Signed-off-by: Sean Kao <seankao@amazon.com> * re-evaluate affected tests Signed-off-by: Sean Kao <seankao@amazon.com> * spotlessApply Signed-off-by: Sean Kao <seankao@amazon.com> --------- Signed-off-by: Sean Kao <seankao@amazon.com>
diff --git a/docs/user/interfaces/protocol.rst b/docs/user/interfaces/protocol.rst
@@ -208,73 +208,6 @@ Result set::
 	  "status" : 400
 	}
 
-OpenSearch DSL
-==============
-
-Description
------------
-
-The plugin returns original response from OpenSearch in JSON. Because this is the native response from OpenSearch, extra efforts are needed to parse and interpret it.
-
-Example
--------
-
-SQL query::
-
-	>> curl -H 'Content-Type: application/json' -X POST localhost:9200/_plugins/_sql?format=json -d '{
-	  "query" : "SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2"
-	}'
-
-Result set::
-
-	{
-	  "_shards" : {
-	    "total" : 5,
-	    "failed" : 0,
-	    "successful" : 5,
-	    "skipped" : 0
-	  },
-	  "hits" : {
-	    "hits" : [
-	      {
-	        "_index" : "accounts",
-	        "_type" : "_doc",
-	        "_source" : {
-	          "firstname" : "Nanette",
-	          "age" : 28,
-	          "lastname" : "Bates"
-	        },
-	        "_id" : "13",
-	        "sort" : [
-	          28
-	        ],
-	        "_score" : null
-	      },
-	      {
-	        "_index" : "accounts",
-	        "_type" : "_doc",
-	        "_source" : {
-	          "firstname" : "Amber",
-	          "age" : 32,
-	          "lastname" : "Duke"
-	        },
-	        "_id" : "1",
-	        "sort" : [
-	          32
-	        ],
-	        "_score" : null
-	      }
-	    ],
-	    "total" : {
-	      "value" : 4,
-	      "relation" : "eq"
-	    },
-	    "max_score" : null
-	  },
-	  "took" : 100,
-	  "timed_out" : false
-	}
-
 CSV Format
 ==========
 
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java
@@ -38,6 +38,9 @@
 import org.junit.Ignore;
 import org.junit.Test;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class AggregationIT extends SQLIntegTestCase {
 
   @Override
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/DateFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/DateFormatIT.java
@@ -29,6 +29,9 @@
 import org.junit.Test;
 import org.opensearch.sql.legacy.exception.SqlParseException;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class DateFormatIT extends SQLIntegTestCase {
 
   private static final String SELECT_FROM =
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/DateFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/DateFunctionsIT.java
@@ -17,6 +17,7 @@
 import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
 import org.json.JSONObject;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
@@ -25,6 +26,9 @@
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.search.SearchHit;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class DateFunctionsIT extends SQLIntegTestCase {
 
   private static final String FROM = "FROM " + TestsConstants.TEST_INDEX_ONLINE;
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/GetEndpointQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/GetEndpointQueryIT.java
@@ -13,7 +13,6 @@
 import org.junit.rules.ExpectedException;
 import org.opensearch.client.ResponseException;
 
-/** Tests to cover requests with "?format=csv" parameter */
 public class GetEndpointQueryIT extends SQLIntegTestCase {
 
   @Rule public final ExpectedException rule = ExpectedException.none();
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/HashJoinIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/HashJoinIT.java
@@ -19,9 +19,13 @@
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
 /** Test new hash join algorithm by comparison with old implementation. */
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class HashJoinIT extends SQLIntegTestCase {
 
   /** Hint to use old join algorithm */
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/HavingIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/HavingIT.java
@@ -20,8 +20,12 @@
 import org.hamcrest.Matcher;
 import org.json.JSONArray;
 import org.json.JSONObject;
+import org.junit.Ignore;
 import org.junit.Test;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class HavingIT extends SQLIntegTestCase {
 
   private static final String SELECT_FROM_WHERE_GROUP_BY =
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JSONRequestIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JSONRequestIT.java
@@ -14,6 +14,7 @@
 import java.io.IOException;
 import java.util.Map;
 import org.json.JSONObject;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
@@ -23,6 +24,9 @@
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.SearchHits;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class JSONRequestIT extends SQLIntegTestCase {
 
   @Override
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JoinIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JoinIT.java
@@ -31,6 +31,9 @@
 import org.junit.Ignore;
 import org.junit.Test;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class JoinIT extends SQLIntegTestCase {
 
   private static final String USE_NL_HINT = " /*! USE_NL*/";
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MathFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MathFunctionsIT.java
@@ -12,6 +12,7 @@
 
 import com.fasterxml.jackson.core.JsonFactory;
 import java.io.IOException;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
@@ -20,6 +21,9 @@
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.search.SearchHit;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class MathFunctionsIT extends SQLIntegTestCase {
 
   private static final String FROM = "FROM " + TestsConstants.TEST_INDEX_ACCOUNT;
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MultiQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MultiQueryIT.java
@@ -14,8 +14,12 @@
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class MultiQueryIT extends SQLIntegTestCase {
 
   private static final String MINUS_SCROLL_DEFAULT_HINT =
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/NestedFieldQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/NestedFieldQueryIT.java
@@ -60,6 +60,9 @@
  * 6) Verification for conditions mixed with regular and nested fields
  * </pre>
  */
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class NestedFieldQueryIT extends SQLIntegTestCase {
 
   private static final String FROM =
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OrderIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OrderIT.java
@@ -10,8 +10,12 @@
 import java.io.IOException;
 import org.json.JSONArray;
 import org.json.JSONObject;
+import org.junit.Ignore;
 import org.junit.Test;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class OrderIT extends SQLIntegTestCase {
 
   @Override
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PluginIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PluginIT.java
@@ -12,6 +12,7 @@
 import java.io.IOException;
 import java.util.Locale;
 import org.json.JSONObject;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.opensearch.client.Request;
 import org.opensearch.client.RequestOptions;
@@ -26,6 +27,9 @@ protected void init() throws Exception {
     wipeAllClusterSettings();
   }
 
+  @Ignore(
+      "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+          + " response for now. Need to decide what to do with these test cases.")
   @Test
   public void sqlEnableSettingsTest() throws IOException {
     loadIndex(Index.ACCOUNT);
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PreparedStatementIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PreparedStatementIT.java
@@ -9,10 +9,14 @@
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
 // Refer to https://www.elastic.co/guide/en/elasticsearch/reference/6.5/integration-tests.html
 // for detailed OpenSearchIntegTestCase usages doc.
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class PreparedStatementIT extends SQLIntegTestCase {
 
   @Override
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryFunctionsIT.java
@@ -26,6 +26,7 @@
 import org.hamcrest.FeatureMatcher;
 import org.hamcrest.Matcher;
 import org.json.JSONObject;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
@@ -35,6 +36,9 @@
 import org.opensearch.search.SearchHit;
 import org.opensearch.sql.legacy.utils.StringUtils;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class QueryFunctionsIT extends SQLIntegTestCase {
 
   private static final String SELECT_ALL = "SELECT *";
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java
@@ -40,6 +40,9 @@
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.sql.legacy.utils.StringUtils;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class QueryIT extends SQLIntegTestCase {
 
   /**
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLFunctionsIT.java
@@ -44,6 +44,9 @@
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.SearchHits;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 /** Created by allwefantasy on 8/25/16. */
 public class SQLFunctionsIT extends SQLIntegTestCase {
 
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java
@@ -229,7 +229,7 @@ protected synchronized void loadIndex(Index index) throws IOException {
   }
 
   protected Request getSqlRequest(String request, boolean explain) {
-    return getSqlRequest(request, explain, "json");
+    return getSqlRequest(request, explain, "jdbc");
   }
 
   protected Request getSqlRequest(String request, boolean explain, String requestType) {
@@ -344,7 +344,7 @@ protected Request buildGetEndpointRequest(final String sqlQuery) {
 
     final String requestUrl =
         String.format(
-            Locale.ROOT, "%s?sql=%s&format=%s", QUERY_API_ENDPOINT, urlEncodedQuery, "json");
+            Locale.ROOT, "%s?sql=%s&format=%s", QUERY_API_ENDPOINT, urlEncodedQuery, "jdbc");
     return new Request("GET", requestUrl);
   }
 
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/ShowIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/ShowIT.java
@@ -13,8 +13,12 @@
 import java.io.IOException;
 import org.json.JSONObject;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class ShowIT extends SQLIntegTestCase {
 
   @Override
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SourceFieldIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SourceFieldIT.java
@@ -12,6 +12,7 @@
 import java.util.Set;
 import org.json.JSONObject;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
@@ -21,6 +22,9 @@
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.SearchHits;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class SourceFieldIT extends SQLIntegTestCase {
 
   @Override
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java
@@ -35,6 +35,9 @@
 import org.opensearch.client.ResponseException;
 import org.opensearch.sql.legacy.utils.StringUtils;
 
+@Ignore(
+    "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+        + " response for now. Need to decide what to do with these test cases.")
 public class SubqueryIT extends SQLIntegTestCase {
 
   @Rule public final ExpectedException exceptionRule = ExpectedException.none();
diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java
@@ -24,6 +24,7 @@
 import java.util.List;
 import org.json.JSONArray;
 import org.json.JSONObject;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
@@ -157,6 +158,9 @@ public void isnullShouldPassJDBC() throws IOException {
     assertEquals("boolean", response.query("/schema/0/type"));
   }
 
+  @Ignore(
+      "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+          + " response for now. Need to decide what to do with these test cases.")
   @Test
   public void isnullWithNotNullInputTest() throws IOException {
     assertThat(
@@ -188,6 +192,9 @@ public void isnullWithNullInputTest() {
         rows(LITERAL_TRUE.value(), LITERAL_FALSE.value()));
   }
 
+  @Ignore(
+      "OpenSearch DSL format is deprecated in 3.0.0. Ignore legacy IT that relies on json format"
+          + " response for now. Need to decide what to do with these test cases.")
   @Test
   public void isnullWithMathExpr() throws IOException {
     assertThat(
diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/ActionRequestRestExecutorFactory.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/ActionRequestRestExecutorFactory.java
diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticDefaultRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticDefaultRestExecutor.java
diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/Format.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/Format.java
diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java
diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/OpenSearchActionFactory.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/OpenSearchActionFactory.java
diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/SqlRequestParamTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/SqlRequestParamTest.java
diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/StringOperatorsTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/StringOperatorsTest.java
diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/OpenSearchActionFactoryTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/OpenSearchActionFactoryTest.java
diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java
