Skip to content

Commit 35fbea7

Browse files
craig[bot]normanchennyuzefovich
committed
144658: jsonpath: add support for `like_regex` flags r=yuzefovich a=normanchenn This commit adds support for regex flags for JSONPath's `like_regex` predicate. The supported flags are: - 'i': Case-insensitive matching - 's': Dot matches newline - 'm': Multiline mode (^\$ match at newlines) - 'x': Ignore whitespace mode (Not implemented in Postgres, we return the same error) - 'q': No special characters (treat pattern as a literal) Epic: None Release note (sql change): Add support for `like_regex` flags in JSONPath queries. For example, `SELECT jsonb_path_query('{}', '"a" like_regex ".*" flag "i"');`. 145255: colbuilder: don't use direct scans in an invalid case r=yuzefovich a=yuzefovich We have an experimental feature to build `coldata.Batch`es directly on the KV server side when handling Scan and ReverseScan requests. It assumes that each request corresponds to any number of full SQL rows, but previously we didn't disable usage of this feature when a single SQL row is populated via multiple requests. This is the case when we have multiple column families, and since we don't support Gets in the direct scans machinery right now, we need at least 5 column families - so that we get Scan for 0-1 and 3-4 CFs. This commit fixes this oversight to ensure that if we have two consecutive key spans that come from the same SQL row, we don't use the direct scans even if they are enabled. There is no release note since this feature is disabled by default. Fixes: #145232. Release note: None Co-authored-by: Norman Chen <[email protected]> Co-authored-by: Yahor Yuzefovich <[email protected]>
3 parents 5506b40 + c96c5a4 + 18a3fc3 commit 35fbea7

File tree

24 files changed

+390
-27
lines changed

24 files changed

+390
-27
lines changed

‎pkg/ccl/logictestccl/tests/3node-tenant/generated_test.go‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/ccl/logictestccl/tests/local-read-committed/generated_test.go‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/ccl/logictestccl/tests/local-repeatable-read/generated_test.go‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/sql/colexec/colbuilder/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ go_library(
1212
"//pkg/col/coldata",
1313
"//pkg/col/coldataext",
1414
"//pkg/col/typeconv",
15+
"//pkg/keys",
1516
"//pkg/kv/kvserver/concurrency/lock",
1617
"//pkg/settings",
1718
"//pkg/sql/catalog/descpb",

‎pkg/sql/colexec/colbuilder/execplan.go‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,15 @@
66
package colbuilder
77

88
import (
9+
"bytes"
910
"context"
1011
"reflect"
1112
"strings"
1213

1314
"github.com/cockroachdb/cockroach/pkg/col/coldata"
1415
"github.com/cockroachdb/cockroach/pkg/col/coldataext"
1516
"github.com/cockroachdb/cockroach/pkg/col/typeconv"
17+
"github.com/cockroachdb/cockroach/pkg/keys"
1618
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock"
1719
"github.com/cockroachdb/cockroach/pkg/settings"
1820
"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
@@ -876,17 +878,38 @@ func NewColOperator(
876878
core.TableReader.LockingWaitPolicy == descpb.ScanLockingWaitPolicy_SKIP_LOCKED {
877879
return false
878880
}
879-
// At the moment, the ColBatchDirectScan cannot handle Gets
880-
// (it's not clear whether it is worth to handle them via
881-
// the same path as for Scans and ReverseScans (which could
882-
// have too large of an overhead) or by teaching the
883-
// operator to also decode a single KV (similar to what
884-
// regular ColBatchScan does)).
885-
// TODO(yuzefovich, 23.1): explore supporting Gets somehow.
886-
for i := range core.TableReader.Spans {
887-
if len(core.TableReader.Spans[i].EndKey) == 0 {
881+
var prevRowPrefix []byte
882+
for i, sp := range core.TableReader.Spans {
883+
if len(sp.EndKey) == 0 {
884+
// At the moment, the ColBatchDirectScan cannot
885+
// handle Gets (it's not clear whether it is worth
886+
// to handle them via the same path as for Scans and
887+
// ReverseScans (which could have too large of an
888+
// overhead) or by teaching the operator to also
889+
// decode a single KV (similar to what regular
890+
// ColBatchScan does)).
891+
// TODO(yuzefovich, 23.1): explore supporting Gets
892+
// somehow.
888893
return false
889894
}
895+
l, err := keys.GetRowPrefixLength(sp.Key)
896+
if err != nil {
897+
// This should rarely happen since an error here
898+
// indicates that we're dealing with a non-SQL key,
899+
// so we'll be conservative and simply disable
900+
// direct columnar scans. (One example where we can
901+
// get an error if we had manually split the range.)
902+
return false
903+
}
904+
curRowPrefix := sp.Key[:l]
905+
if i > 0 && bytes.Equal(prevRowPrefix, curRowPrefix) {
906+
// Two consecutive requests are part of the
907+
// same SQL row in which case we cannot use direct
908+
// columnar scans since we'd create a separate
909+
// coldata.Batch for each.
910+
return false
911+
}
912+
prevRowPrefix = curRowPrefix
890913
}
891914
fetchSpec := core.TableReader.FetchSpec
892915
// Handling user-defined types requires type hydration which

‎pkg/sql/logictest/testdata/logic_test/direct_columnar_scans‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
statement ok
2+
SET direct_columnar_scans_enabled = true
3+
4+
statement ok
5+
CREATE TABLE t145232 (
6+
k INT PRIMARY KEY,
7+
a INT NOT NULL,
8+
b INT NOT NULL,
9+
c INT NOT NULL,
10+
v INT NOT NULL DEFAULT 5,
11+
FAMILY (c),
12+
FAMILY (v),
13+
FAMILY (k),
14+
FAMILY (a),
15+
FAMILY (b)
16+
);
17+
18+
statement ok
19+
INSERT INTO t145232 VALUES (2,2,2,2);
20+
21+
# Regression test for #145232 where we tried to use direct columnar scans even
22+
# though a single SQL row is handled by multiple Scan requests.
23+
query IIIII
24+
SELECT * FROM t145232 WHERE k = 2
25+
----
26+
2 2 2 2 5

‎pkg/sql/logictest/testdata/logic_test/distsql_enum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,11 +88,11 @@ SELECT t1.x from t1 INNER LOOKUP JOIN t2 ON t1.x=t2.x WHERE t2.y='hello'
8888
├ Node 2
8989
│ └ *colrpc.Outbox
9090
│ └ *rowexec.joinReader
91-
│ └ *colfetcher.ColBatchDirectScan
91+
│ └ *colfetcher.ColBatchScan
9292
└ Node 3
9393
└ *colrpc.Outbox
9494
└ *rowexec.joinReader
95-
└ *colfetcher.ColBatchDirectScan
95+
└ *colfetcher.ColBatchScan
9696

9797
query I
9898
SELECT t1.x from t1 INNER LOOKUP JOIN t2 ON t1.x=t2.x WHERE t2.y='hello'

‎pkg/sql/logictest/testdata/logic_test/jsonb_path_query‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1812,3 +1812,91 @@ SELECT jsonb_path_query('"1"', '$.abs()');
18121812

18131813
statement error pgcode 22036 pq: jsonpath item method .floor\(\) can only be applied to a numeric value
18141814
SELECT jsonb_path_query('{}', '(null).floor()');
1815+
1816+
query T
1817+
SELECT jsonb_path_query('"Hello"', '$ like_regex "hello" flag "i"');
1818+
----
1819+
true
1820+
1821+
query T
1822+
SELECT jsonb_path_query('"HELLO"', '$ like_regex "hello" flag "i"');
1823+
----
1824+
true
1825+
1826+
# Use the same pattern but different flags to ensure that ReCache recognizes
1827+
# the change of flags.
1828+
query T
1829+
SELECT jsonb_path_query('"HELLO"', '$ like_regex "hello" flag ""');
1830+
----
1831+
false
1832+
1833+
query T
1834+
SELECT jsonb_path_query('"Hello\nWorld"', '$ like_regex "Hello.World" flag "s"');
1835+
----
1836+
true
1837+
1838+
query T
1839+
SELECT jsonb_path_query('"Hello\nWorld"', '$ like_regex "Hello.World" flag ""');
1840+
----
1841+
false
1842+
1843+
query T
1844+
SELECT jsonb_path_query('"Line1\nLine2"', '$ like_regex "^Line2$" flag "m"');
1845+
----
1846+
true
1847+
1848+
query T
1849+
SELECT jsonb_path_query('"Line1\nLine2"', '$ like_regex "^Line2$" flag ""');
1850+
----
1851+
false
1852+
1853+
query T
1854+
SELECT jsonb_path_query('"Hello123World"', '$ like_regex "Hello.*World" flag "q"');
1855+
----
1856+
false
1857+
1858+
query T
1859+
SELECT jsonb_path_query('"Hello123World"', '$ like_regex "Hello.*World" flag ""');
1860+
----
1861+
true
1862+
1863+
# Case insensitive and dot matches newline
1864+
query T
1865+
SELECT jsonb_path_query('"Hello\nWorld"', '$ like_regex "hello.world" flag "is"');
1866+
----
1867+
true
1868+
1869+
# Case insensitive and multiline
1870+
query T
1871+
SELECT jsonb_path_query('"Line1\nline2"', '$ like_regex "^LINE2$" flag "im"');
1872+
----
1873+
true
1874+
1875+
# Case insensitive and literal matching
1876+
query T
1877+
SELECT jsonb_path_query('"Hello123World"', '$ like_regex "HELLO.*WORLD" flag "iq"');
1878+
----
1879+
false
1880+
1881+
# Dot matches newline and multiline
1882+
query T
1883+
SELECT jsonb_path_query('"Line1\nLine2\nLine3"', '$ like_regex "^Line1.Line2.Line3$" flag "ms"');
1884+
----
1885+
true
1886+
1887+
query T
1888+
SELECT jsonb_path_query('"Line1\nLine2\nLine3"', '$ like_regex "^Line1.Line2.Line3$" flag "m"');
1889+
----
1890+
false
1891+
1892+
# Dot matches newline and literal matching
1893+
query T
1894+
SELECT jsonb_path_query('"Hello\nWorld"', '$ like_regex "Hello.World" flag "sq"');
1895+
----
1896+
false
1897+
1898+
# Multiline and literal matching
1899+
query T
1900+
SELECT jsonb_path_query('"Line1\nLine2"', '$ like_regex "^Line1\nLine2$" flag "mq"');
1901+
----
1902+
false

‎pkg/sql/logictest/testdata/logic_test/jsonpath‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -159,9 +159,6 @@ SELECT 'last'::JSONPATH
159159
statement error pgcode 42601 pq: could not parse "@" as type jsonpath: @ is not allowed in root expressions
160160
SELECT '@'::JSONPATH
161161

162-
statement error unimplemented
163-
SELECT '$ ? (@ like_regex ".*" flag "i")'::JSONPATH;
164-
165162
statement error unimplemented
166163
SELECT '$.keyvalue()'::JSONPATH;
167164

‎pkg/sql/logictest/tests/fakedist-disk/generated_test.go‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)