Support script pushdown in sort-on-measure pushdown rewriting (#4749)

* Support script pushdown in sort-on-measure pushdown rewriting

Signed-off-by: Lantao Jin <[email protected]>

* fix IT

Signed-off-by: Lantao Jin <[email protected]>

* Merge remote-tracking branch 'upstream/main' into issues/4738

* remove system.out.print

Signed-off-by: Lantao Jin <[email protected]>

* Fix IT

Signed-off-by: Lantao Jin <[email protected]>

Author: LantaoJin

common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java

@@ -185,7 +185,6 @@ public Grok compile(final String pattern, ZoneId defaultTimeZone, boolean namedO
               (group.get("subname") != null ? group.get("subname") : group.get("name")));
           namedRegex =
               StringUtils.replace(namedRegex, "%{" + group.get("name") + "}", replacement, 1);
-          // System.out.println(_expanded_pattern);
           index++;
         }
       }

common/src/test/java/org/opensearch/sql/common/grok/ApacheDataTypeTest.java

@@ -49,7 +49,6 @@ public void test002_httpd_access_semi() throws GrokException {
                 + " HTTP/%{NUMBER:httpversion;float})?|%{DATA:rawrequest})\" %{NUMBER:response;int}"
                 + " (?:%{NUMBER:bytes;long}|-)");
 
-    System.out.println(line);
     Match gm = grok.match(line);
     Map<String, Object> map = gm.capture();
 

common/src/test/java/org/opensearch/sql/common/grok/ApacheTest.java

@@ -38,7 +38,6 @@ public void test001_httpd_access() throws GrokException, IOException {
 
     BufferedReader br = new BufferedReader(new FileReader(LOG_FILE));
     String line;
-    System.out.println("Starting test with httpd log");
     while ((line = br.readLine()) != null) {
       Match gm = grok.match(line);
       final Map<String, Object> capture = gm.capture();
@@ -50,7 +49,6 @@ public void test001_httpd_access() throws GrokException, IOException {
   @Test
   public void test002_nasa_httpd_access() throws GrokException, IOException {
     Grok grok = compiler.compile("%{COMMONAPACHELOG}");
-    System.out.println("Starting test with nasa log -- may take a while");
     BufferedReader br;
     String line;
     File dir = new File(LOG_DIR_NASA);

common/src/test/java/org/opensearch/sql/common/grok/GrokTest.java

@@ -317,7 +317,6 @@ public void test013_IpSet() throws Throwable {
     try (FileReader fr = new FileReader(Resources.getResource(ResourceManager.IP).getFile());
         BufferedReader br = new BufferedReader(fr)) {
       String line;
-      System.out.println("Starting test with ip");
       while ((line = br.readLine()) != null) {
         Match gm = grok.match(line);
         final Map<String, Object> map = gm.capture();

common/src/test/java/org/opensearch/sql/common/grok/MessagesTest.java

@@ -29,7 +29,6 @@ public void test001_linux_messages() throws GrokException, IOException {
         new BufferedReader(
             new FileReader(Resources.getResource(ResourceManager.MESSAGES).getFile()));
     String line;
-    System.out.println("Starting test with linux messages log -- may take a while");
     while ((line = br.readLine()) != null) {
       Match gm = grok.match(line);
       Map<String, Object> map = gm.capture();

core/src/test/java/org/opensearch/sql/calcite/utils/BinUtilsTest.java

@@ -106,21 +106,13 @@ public void testParseSpanStringWithExtendedTimeUnits() {
   public void testMonthUnitDetection() {
     // Specifically test that 1mon is detected as "mon" not "m"
     String result = SpanParser.extractTimeUnit("1mon");
-    System.out.println("Result for '1mon': " + result);
     assertEquals("mon", result);
   }
 
   @Test
   public void testFullMonthParsingChain() {
     // Test the full parsing chain for 1mon
     var spanInfo = SpanParser.parse("1mon");
-    System.out.println(
-        "SpanInfo: type="
-            + spanInfo.getType()
-            + ", value="
-            + spanInfo.getValue()
-            + ", unit="
-            + spanInfo.getUnit());
     assertEquals(SpanType.TIME, spanInfo.getType());
     assertEquals(1.0, spanInfo.getValue(), 0.001);
     assertEquals("mon", spanInfo.getUnit());

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java

@@ -1149,6 +1149,17 @@ public void testExplainSortOnMeasure() throws IOException {
                 TEST_INDEX_BANK)));
   }
 
+  @Test
+  public void testExplainSortOnMeasureWithScript() throws IOException {
+    enabledOnlyWhenPushdownIsEnabled();
+    String expected = loadExpectedPlan("explain_agg_sort_on_measure_script.yaml");
+    assertYamlEqualsIgnoreId(
+        expected,
+        explainQueryYaml(
+            "source=opensearch-sql_test_index_account | eval new_state = lower(state) | "
+                + "stats bucket_nullable=false count() by new_state | sort `count()`"));
+  }
+
   @Test
   public void testExplainSortOnMeasureMultiTerms() throws IOException {
     enabledOnlyWhenPushdownIsEnabled();
@@ -1160,6 +1171,18 @@ public void testExplainSortOnMeasureMultiTerms() throws IOException {
                 + " gender, state | sort `count()`"));
   }
 
+  @Test
+  public void testExplainSortOnMeasureMultiTermsWithScript() throws IOException {
+    enabledOnlyWhenPushdownIsEnabled();
+    String expected = loadExpectedPlan("explain_agg_sort_on_measure_multi_terms_script.yaml");
+    assertYamlEqualsIgnoreId(
+        expected,
+        explainQueryYaml(
+            "source=opensearch-sql_test_index_account | eval new_gender = lower(gender), new_state"
+                + " = lower(state) | stats bucket_nullable=false count() by new_gender, new_state |"
+                + " sort `count()`"));
+  }
+
   @Test
   public void testExplainCompositeMultiBucketsAutoDateThenSortOnMeasureNotPushdown()
       throws IOException {

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFillNullCommandIT.java

@@ -137,7 +137,6 @@ public void testFillNullWithStringOnNumericAndStringMixedFields() {
                         "source=%s | fields num0, str2 | fillnull value='test' num0 str2",
                         TEST_INDEX_CALCS)));
 
-    System.out.println("Debugging error message: " + t);
     verifyErrorMessageContains(
         t,
         "fillnull failed: replacement value type VARCHAR is not compatible with field 'num0' "

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java

@@ -97,8 +97,6 @@ public void testMvappendWithIntAndDouble() throws IOException {
         executeQuery(
             source(TEST_INDEX_BANK, "eval result = mvappend(1, 2.5) | head 1 | fields result"));
 
-    System.out.println(actual);
-
     verifySchema(actual, schema("result", "array"));
     verifyDataRows(actual, rows(List.of(1, 2.5)));
   }

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java

@@ -428,7 +428,6 @@ public void testBrainAggregationModeWithGroupByClause_ShowNumberedToken() throws
                     + " mode=aggregation show_numbered_token=true max_sample_count=5"
                     + " variable_count_threshold=2 frequency_threshold_percentage=0.2",
                 TEST_INDEX_HDFS_LOGS));
-    System.out.println(result);
     verifySchema(
         result,
         schema("level", "string"),