SQL: Add CCS SQL documentation (#81545) (#82282)

bpintea · jrodewig · web-flow · commit 7f9895dbe940 · 2022-01-05T15:55:15.000-05:00
This adds the documentation for CCS SQL. Co-authored-by: James Rodewig <james.rodewig@elastic.co> (cherry picked from commit 13a0e42)
diff --git a/docs/reference/search/search-your-data/search-across-clusters.asciidoc b/docs/reference/search/search-your-data/search-across-clusters.asciidoc
@@ -19,8 +19,8 @@ The following APIs support {ccs}:
 * <<search-template,Search template>>
 * <<multi-search-template,Multi search template>>
 * <<search-field-caps,Field capabilities>>
-* <<sql-search-api,SQL search>>
 * experimental:[] <<eql-search-api,EQL search>>
+* experimental:[] <<sql-search-api,SQL search>>
 * <<search-vector-tile-api,Vector tile search>>
 
 [discrete]
diff --git a/docs/reference/sql/apis/sql-search-api.asciidoc b/docs/reference/sql/apis/sql-search-api.asciidoc
@@ -54,6 +54,12 @@ precedence.
 [[sql-search-api-request-body]]
 ==== {api-request-body-title}
 
+`catalog`::
+(Optional, string) Default catalog (cluster) for queries. If unspecified, the
+queries execute on the data in the local cluster only.
++
+experimental:[] See <<modules-cross-cluster-search,{ccs}>>.
+
 `columnar`::
 (Optional, Boolean) If `true`, returns results in a columnar format. Defaults to
 `false`. The API only supports this parameter for CBOR, JSON, SMILE, and YAML
diff --git a/docs/reference/sql/endpoints/jdbc.asciidoc b/docs/reference/sql/endpoints/jdbc.asciidoc
@@ -148,6 +148,13 @@ will be - typically the first in natural ascending order) for fields with multip
 ==== Index
 `index.include.frozen` (default `false`):: Whether to include <<freeze-index-api, frozen-indices>> in the query execution or not (default).
 
+[discrete]
+==== Cluster
+`catalog`:: Default catalog (cluster) for queries. If unspecified, the
+queries execute on the data in the local cluster only.
++
+experimental:[] See <<modules-cross-cluster-search,{ccs}>>.
+
 [discrete]
 ==== Additional
 
diff --git a/docs/reference/sql/language/indices.asciidoc b/docs/reference/sql/language/indices.asciidoc
@@ -34,6 +34,19 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[fromTablePatternQuoted]
 
 NOTE: There is the restriction that all resolved concrete tables have the exact same mapping.
 
+experimental:[] To run a <<modules-cross-cluster-search,{ccs}>>, specify a
+cluster name using the `<remote_cluster>:<target>` syntax, where
+`<remote_cluster>` maps to a SQL catalog (cluster) and `<target>` to a table
+(index or data stream). The `<remote_cluster>` supports wildcards (`*`)
+and `<target>` can be an index pattern.
+
+For example:
+
+[source, sql]
+----
+include-tagged::{sql-specs}/multi-cluster-with-security/multi-cluster-docs.csv-spec[fromQualifiedTableQuoted]
+----
+
 [[sql-index-patterns-like]]
 [discrete]
 ==== SQL `LIKE` notation
diff --git a/docs/reference/sql/language/syntax/commands/describe-table.asciidoc b/docs/reference/sql/language/syntax/commands/describe-table.asciidoc
@@ -5,25 +5,17 @@
 .Synopsis:
 [source, sql]
 ----
-DESCRIBE
-    [table identifier | <1>
-    [LIKE pattern]]     <2>
+DESCRIBE | DESC
+    [CATALOG identifier]? <1>
+    [INCLUDE FROZEN]?     <2>
+    [table_identifier |   <3>
+     LIKE pattern]        <4>
 ----
 
-<1> single table identifier or double quoted es multi index
-<2> SQL LIKE pattern
-
-or 
-
-[source, sql]
-----
-DESC 
-    [table identifier | <1>
-    [LIKE pattern]]     <2>
-----
-
-<1> single table identifier or double quoted es multi index
-<2> SQL LIKE pattern
+<1> Catalog (cluster) identifier. Supports wildcards (`*`).
+<2> Whether or not to include frozen indices.
+<3> Single table (index or data stream) identifier or double-quoted multi-target pattern.
+<4> SQL LIKE pattern matching table names.
 
 *Description*: `DESC` and `DESCRIBE` are aliases to <<sql-syntax-show-columns>>.
 
diff --git a/docs/reference/sql/language/syntax/commands/index.asciidoc b/docs/reference/sql/language/syntax/commands/index.asciidoc
@@ -6,12 +6,14 @@ This section contains the list of SQL commands supported by {es-sql} along with
 
 <<sql-syntax-describe-table>>:: Describe a table.
 <<sql-syntax-select>>:: Retrieve rows from zero or more tables.
+<<sql-syntax-show-catalogs>>:: List available catalogs.
 <<sql-syntax-show-columns>>:: List columns in table.
 <<sql-syntax-show-functions>>:: List supported functions.
 <<sql-syntax-show-tables>>:: List tables available.
 
 include::describe-table.asciidoc[]
 include::select.asciidoc[]
+include::show-catalogs.asciidoc[]
 include::show-columns.asciidoc[]
 include::show-functions.asciidoc[]
 include::show-tables.asciidoc[]
diff --git a/docs/reference/sql/language/syntax/commands/select.asciidoc b/docs/reference/sql/language/syntax/commands/select.asciidoc
@@ -123,6 +123,17 @@ The name can be a <<multi-index, pattern>> pointing to multiple indices (likely
 include-tagged::{sql-specs}/docs/docs.csv-spec[fromTablePatternQuoted]
 ----
 
+experimental:[] To run a <<modules-cross-cluster-search,{ccs}>>, specify a
+cluster name using the `<remote_cluster>:<target>` syntax, where
+`<remote_cluster>` maps to a SQL catalog (cluster) and `<target>` to a table
+(index or data stream). The `<remote_cluster>` supports wildcards (`*`)
+and `<target>` can be an <<sql-index-patterns, index pattern>>.
+
+[source, sql]
+----
+include-tagged::{sql-specs}/multi-cluster-with-security/multi-cluster-docs.csv-spec[fromQualifiedTableQuoted]
+----
+
 `alias`::
 A substitute name for the `FROM` item containing the alias. An alias is used for brevity or to eliminate ambiguity. When an alias is provided, it completely hides the actual name of the table and must be used in its place.
 
@@ -334,7 +345,7 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[orderByBasic]
 [[sql-syntax-order-by-grouping]]
 ==== Order By and Grouping
 
-For queries that perform grouping, ordering can be applied either on the grouping columns (by default ascending) or on aggregate functions. 
+For queries that perform grouping, ordering can be applied either on the grouping columns (by default ascending) or on aggregate functions.
 
 NOTE: With `GROUP BY`, make sure the ordering targets the resulting group - applying it to individual elements inside the group will have no impact on the results since regardless of the order, values inside the group are aggregated.
 
@@ -370,7 +381,7 @@ When doing full-text queries in the `WHERE` clause, results can be returned base
 
 NOTE: When doing multiple text queries in the `WHERE` clause then, their scores will be
 combined using the same rules as {es}'s
-<<query-dsl-bool-query,bool query>>. 
+<<query-dsl-bool-query,bool query>>.
 
 To sort based on the `score`, use the special function `SCORE()`:
 
diff --git a/docs/reference/sql/language/syntax/commands/show-catalogs.asciidoc b/docs/reference/sql/language/syntax/commands/show-catalogs.asciidoc
@@ -0,0 +1,17 @@
+[role="xpack"]
+[[sql-syntax-show-catalogs]]
+=== SHOW CATALOGS
+
+.Synopsis:
+[source, sql]
+----
+SHOW CATALOGS
+----
+
+*Description*: List the available catalogs and their types.
+
+[source, sql]
+----
+include-tagged::{sql-specs}/multi-cluster-with-security/multi-cluster-command.csv-spec[showCatalogs]
+----
+
diff --git a/docs/reference/sql/language/syntax/commands/show-columns.asciidoc b/docs/reference/sql/language/syntax/commands/show-columns.asciidoc
@@ -5,13 +5,18 @@
 .Synopsis:
 [source, sql]
 ----
-SHOW COLUMNS [ FROM | IN ]?
-    [table identifier | <1>
-    [LIKE pattern] ]    <2>
+SHOW COLUMNS
+    [CATALOG identifier]? <1>
+    [INCLUDE FROZEN]?     <2>
+    [FROM | IN]
+    [table_identifier |   <3>
+     LIKE pattern]        <4>
 ----
 
-<1> single table identifier or double quoted es multi index
-<2> SQL LIKE pattern
+<1> Catalog (cluster) identifier. Supports wildcards (`*`).
+<2> Whether or not to include frozen indices.
+<3> Single table (index or data stream) identifier or double-quoted multi-target pattern.
+<4> SQL LIKE pattern matching table names.
 
 See <<sql-index-patterns, index patterns>> for more information about
 patterns.
diff --git a/docs/reference/sql/language/syntax/commands/show-functions.asciidoc b/docs/reference/sql/language/syntax/commands/show-functions.asciidoc
@@ -5,7 +5,7 @@
 .Synopsis:
 [source, sql]
 ----
-SHOW FUNCTIONS [LIKE pattern?]? <1>
+SHOW FUNCTIONS [LIKE pattern]? <1>
 ----
 
 <1> SQL match pattern
diff --git a/docs/reference/sql/language/syntax/commands/show-tables.asciidoc b/docs/reference/sql/language/syntax/commands/show-tables.asciidoc
@@ -6,14 +6,18 @@
 [source, sql]
 ----
 SHOW TABLES
-    [INCLUDE FROZEN]?   <1>
-    [table identifier | <2>
-    [LIKE pattern ]]?   <3>
+    [CATALOG [catalog_identifier | <1>
+              LIKE pattern]]?      <2>
+    [INCLUDE FROZEN]?              <3>
+    [table_identifier |            <4>
+     LIKE pattern]?                <5>
 ----
 
-<1> Whether or not to include frozen indices
-<2> single table identifier or double quoted es multi index
-<3> SQL LIKE pattern
+<1> Catalog (cluster) identifier. Supports wildcards (`*`).
+<2> SQL LIKE pattern matching catalog names.
+<3> Whether or not to include frozen indices.
+<4> Single table (index or data stream) identifier or double-quoted multi-target pattern.
+<5> SQL LIKE pattern matching table names.
 
 See <<sql-index-patterns, index patterns>> for more information about
 patterns.
@@ -60,3 +64,9 @@ Or a mixture of single and multiple chars:
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesLikeMixed]
 ----
+
+List tables within remote clusters whose names are matched by a wildcard:
+[source, sql]
+----
+include-tagged::{sql-specs}/multi-cluster-with-security/multi-cluster-command.csv-spec[showTablesCatalogPatternMultiLike]
+----
diff --git a/x-pack/plugin/sql/qa/server/multi-cluster-with-security/src/test/java/org/elasticsearch/xpack/sql/qa/multi_cluster_with_security/JdbcCsvSpecIT.java b/x-pack/plugin/sql/qa/server/multi-cluster-with-security/src/test/java/org/elasticsearch/xpack/sql/qa/multi_cluster_with_security/JdbcCsvSpecIT.java
@@ -28,7 +28,7 @@ public class JdbcCsvSpecIT extends CsvSpecTestCase {
     public static final String EXTRACT_FN_NAME = "EXTRACT";
 
     private static final Pattern DESCRIBE_OR_SHOW = Pattern.compile("(?i)\\s*(DESCRIBE|SHOW).*");
-    private static final Pattern FROM_QUALIFIED = Pattern.compile("(?i)FROM\\w+" + REMOTE_CLUSTER_NAME + ":");
+    private static final Pattern FROM_QUALIFIED = Pattern.compile(".*(?i)FROM\\s+[^\\s:]+:[^\\s:]+\\s.*");
 
     @ParametersFactory(argumentFormatting = PARAM_FORMATTING)
     public static List<Object[]> readScriptSpec() throws Exception {
@@ -39,7 +39,13 @@ public static List<Object[]> readScriptSpec() throws Exception {
     }
 
     public JdbcCsvSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
-        super(fileName, groupName, testName, lineNumber, randomBoolean() ? qualifyFromClause(testCase) : testCase);
+        super(
+            fileName,
+            groupName,
+            testName,
+            lineNumber,
+            randomBoolean() && isFromQualified(testCase.query) == false ? qualifyFromClause(testCase) : testCase
+        );
     }
 
     // qualify the query FROM clause with the cluster name, but (crudely) skip `EXTRACT(a FROM b)` calls.
@@ -94,7 +100,9 @@ private static CsvTestCase qualifyFromClause(CsvTestCase testCase) {
     @Override
     public Connection esJdbc() throws SQLException {
         Connection connection = esJdbc(connectionProperties());
-        if (FROM_QUALIFIED.matcher(csvTestCase().query).matches() == false) {
+        // Only set the default catalog if the query index isn't yet qualified with the catalog, which can happen if query has been written
+        // qualified from the start (for the documentation) or edited in qualifyFromClause() above.
+        if (isFromQualified(csvTestCase().query) == false) {
             connection.setCatalog(REMOTE_CLUSTER_NAME);
         }
         return connection;
@@ -113,4 +121,10 @@ protected int fetchSize() {
         // similar to https://github.com/elastic/elasticsearch/issues/35176 quicker
         return fileName.startsWith("nested") && randomBoolean() ? randomIntBetween(1, 5) : super.fetchSize();
     }
+
+    // Simple check if the FROM clause of a query contains a cluster-qualified index (pattern).
+    // Note: it won't work reliably with multiple clauses (subselects) or queries embedded in strings.
+    private static boolean isFromQualified(String query) {
+        return FROM_QUALIFIED.matcher(query).matches();
+    }
 }
diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/multi-cluster-with-security/multi-cluster-command.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/multi-cluster-with-security/multi-cluster-command.csv-spec
@@ -7,12 +7,15 @@
 // Note: remote index discovery (for CCS) won't list aliases.
 
 showCatalogs
+// tag::showCatalogs
 SHOW CATALOGS;
 
      name         |     type
 ------------------+---------------
 integTest         |local
 my_remote_cluster |remote
+
+// end::showCatalogs
 ;
 
 showTables
@@ -61,13 +64,15 @@ my_remote_cluster|test_emp       |TABLE          |INDEX
 ;
 
 showTablesCatalogPatternMultiLike
+// tag::showTablesCatalogPatternMultiLike
 SHOW TABLES CATALOG 'my_*' LIKE 'test_emp%';
 
      catalog     |     name      |     type      |     kind
 -----------------+---------------+---------------+---------------
 my_remote_cluster|test_emp       |TABLE          |INDEX
 my_remote_cluster|test_emp_copy  |TABLE          |INDEX
 
+// end::showTablesCatalogPatternMultiLike
 ;
 
 showTablesList
diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/multi-cluster-with-security/multi-cluster-docs.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/multi-cluster-with-security/multi-cluster-docs.csv-spec
@@ -0,0 +1,17 @@
+// To mute tests follow example in file: example.csv-spec
+
+//
+// CSV spec used by the docs
+//
+
+fromQualifiedTableQuoted
+// tag::fromQualifiedTableQuoted
+SELECT emp_no FROM "my*cluster:*emp" LIMIT 1;
+
+    emp_no
+---------------
+10001
+
+// end::fromQualifiedTableQuoted
+;
+
