Skip to content

Commit 7372262

Browse files
jeffswensonjeffswenson
committed
randgen: add schema filters for ldr
Add options to filter the columns, primary key, and secondary indexes generated by randgen. This will be used when testing LDR to constraint schema generation to schemas that are compatible with LDR. Release note: none Informs: CRDB-44094
1 parent f8f9e95 commit 7372262

File tree

7 files changed

+324
-65
lines changed

7 files changed

+324
-65
lines changed

‎pkg/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ ALL_TESTS = [
180180
"//pkg/crosscluster/physical:physical_test",
181181
"//pkg/crosscluster/producer:producer_disallowed_imports_test",
182182
"//pkg/crosscluster/producer:producer_test",
183+
"//pkg/crosscluster/replicationtestutils:replicationtestutils_test",
183184
"//pkg/crosscluster/replicationutils:replicationutils_test",
184185
"//pkg/crosscluster/streamclient/randclient:randclient_test",
185186
"//pkg/crosscluster/streamclient:streamclient_test",
@@ -1321,6 +1322,7 @@ GO_TARGETS = [
13211322
"//pkg/crosscluster/producer:producer",
13221323
"//pkg/crosscluster/producer:producer_test",
13231324
"//pkg/crosscluster/replicationtestutils:replicationtestutils",
1325+
"//pkg/crosscluster/replicationtestutils:replicationtestutils_test",
13241326
"//pkg/crosscluster/replicationutils:replicationutils",
13251327
"//pkg/crosscluster/replicationutils:replicationutils_test",
13261328
"//pkg/crosscluster/streamclient/randclient:randclient",

‎pkg/crosscluster/replicationtestutils/BUILD.bazel‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
load("@io_bazel_rules_go//go:def.bzl", "go_library")
1+
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
22

33
go_library(
44
name = "replicationtestutils",
@@ -36,14 +36,17 @@ go_library(
3636
"//pkg/spanconfig/spanconfigkvaccessor",
3737
"//pkg/sql",
3838
"//pkg/sql/catalog",
39+
"//pkg/sql/catalog/colinfo",
3940
"//pkg/sql/catalog/descpb",
4041
"//pkg/sql/catalog/descs",
4142
"//pkg/sql/catalog/desctestutils",
4243
"//pkg/sql/execinfra",
4344
"//pkg/sql/isql",
45+
"//pkg/sql/randgen",
4446
"//pkg/sql/rowenc",
4547
"//pkg/sql/sem/catconstants",
4648
"//pkg/sql/sem/tree",
49+
"//pkg/sql/types",
4750
"//pkg/storage",
4851
"//pkg/testutils",
4952
"//pkg/testutils/jobutils",
@@ -68,3 +71,28 @@ go_library(
6871
"@com_github_stretchr_testify//require",
6972
],
7073
)
74+
75+
go_test(
76+
name = "replicationtestutils_test",
77+
srcs = [
78+
"logical_test.go",
79+
"main_test.go",
80+
],
81+
data = ["//c-deps:libgeos"],
82+
embed = [":replicationtestutils"],
83+
deps = [
84+
"//pkg/base",
85+
"//pkg/ccl",
86+
"//pkg/ccl/storageccl",
87+
"//pkg/security/securityassets",
88+
"//pkg/security/securitytest",
89+
"//pkg/server",
90+
"//pkg/testutils/serverutils",
91+
"//pkg/testutils/sqlutils",
92+
"//pkg/testutils/testcluster",
93+
"//pkg/util/leaktest",
94+
"//pkg/util/log",
95+
"//pkg/util/randutil",
96+
"@com_github_stretchr_testify//require",
97+
],
98+
)

‎pkg/crosscluster/replicationtestutils/logical.go‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,12 @@ package replicationtestutils
88
import (
99
"context"
1010
"fmt"
11+
"math/rand"
1112

13+
"github.com/cockroachdb/cockroach/pkg/sql/catalog/colinfo"
14+
"github.com/cockroachdb/cockroach/pkg/sql/randgen"
15+
"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
16+
"github.com/cockroachdb/cockroach/pkg/sql/types"
1217
"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
1318
"github.com/cockroachdb/errors"
1419
)
@@ -39,3 +44,75 @@ func CheckEmptyDLQs(ctx context.Context, db sqlutils.DBHandle, dbName string) er
3944
}
4045
return nil
4146
}
47+
48+
func GenerateLDRTable(
49+
ctx context.Context, rng *rand.Rand, tableName string, supportKVWriter bool,
50+
) string {
51+
columnByName := func(name tree.Name, columnDefs []*tree.ColumnTableDef) *tree.ColumnTableDef {
52+
for _, col := range columnDefs {
53+
if col.Name == name {
54+
return col
55+
}
56+
}
57+
return nil
58+
}
59+
60+
tableDef := randgen.RandCreateTableWithName(ctx, rng, tableName, 0, []randgen.TableOption{
61+
randgen.WithPrimaryIndexRequired(),
62+
randgen.WithSkipColumnFamilyMutations(),
63+
randgen.WithPrimaryIndexFilter(func(indexDef *tree.IndexTableDef, columnDefs []*tree.ColumnTableDef) bool {
64+
for _, col := range indexDef.Columns {
65+
columnDef := columnByName(col.Column, columnDefs)
66+
// TODO(127315): types with composite encoding are not supported in the
67+
// primary key by LDR.
68+
if colinfo.CanHaveCompositeKeyEncoding(columnDef.Type.(*types.T)) {
69+
return false
70+
}
71+
// Do not allow computed columns in the primary key. Non-virtual computed columns in the primary key is
72+
// allowed by LDR in general, but its not compatible with the conflict workload.
73+
// TODO(jeffswenson): support computed columns in the primary key in the conflict workload.
74+
if columnDef.IsComputed() {
75+
return false
76+
}
77+
}
78+
if supportKVWriter && indexDef.Sharded != nil {
79+
// The KV writer does not support hash sharded indexes.
80+
return false
81+
}
82+
return true
83+
}),
84+
randgen.WithIndexFilter(func(indexDef tree.TableDef, columnDefs []*tree.ColumnTableDef) bool {
85+
switch indexDef := indexDef.(type) {
86+
case *tree.UniqueConstraintTableDef:
87+
// Do not allow unique indexes. The random data may cause
88+
// spurious unique constraint violations.
89+
// TODO(jeffswenson): extend the conflict workload to support unique indexes on fields
90+
// that can randomly generate exclusively unique values for each row. E.g. UUIDs could be unique, but
91+
// BOOLs are too limiting.
92+
return false
93+
case *tree.IndexTableDef:
94+
for _, col := range indexDef.Columns {
95+
if supportKVWriter && col.Expr != nil {
96+
// Do not allow expression indexes. These cause SQL to generate a hidden computed column, which is not
97+
// supported by the kv writer.
98+
if col.Expr != nil {
99+
return false
100+
}
101+
}
102+
columnDef := columnByName(col.Column, columnDefs)
103+
if columnDef.IsVirtual() {
104+
// Virtual computed columns are not supported in indexes by the classic sql writer or the kv writer.
105+
// TODO(jeffswenson): remove this restriction once the crud writer is the only writer.
106+
return false
107+
}
108+
}
109+
if supportKVWriter && indexDef.Sharded != nil {
110+
// The KV writer does not support hash sharded indexes.
111+
return false
112+
}
113+
}
114+
return true
115+
}),
116+
})
117+
return tree.AsStringWithFlags(tableDef, tree.FmtParsable)
118+
}

‎pkg/crosscluster/replicationtestutils/logical_test.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Copyright 2025 The Cockroach Authors.
2+
//
3+
// Use of this software is governed by the CockroachDB Software License
4+
// included in the /LICENSE file.
5+
6+
package replicationtestutils
7+
8+
import (
9+
"context"
10+
"testing"
11+
"time"
12+
13+
"github.com/cockroachdb/cockroach/pkg/base"
14+
"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
15+
"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
16+
"github.com/cockroachdb/cockroach/pkg/util/leaktest"
17+
"github.com/cockroachdb/cockroach/pkg/util/log"
18+
"github.com/cockroachdb/cockroach/pkg/util/randutil"
19+
"github.com/stretchr/testify/require"
20+
)
21+
22+
func TestGenerateLDRTable(t *testing.T) {
23+
defer leaktest.AfterTest(t)()
24+
defer log.Scope(t).Close(t)
25+
26+
ctx := context.Background()
27+
server, db, _ := serverutils.StartServer(t, base.TestServerArgs{})
28+
defer server.Stopper().Stop(ctx)
29+
30+
sqlDB := sqlutils.MakeSQLRunner(db)
31+
sqlDB.Exec(t, "CREATE DATABASE a")
32+
sqlDB.Exec(t, "CREATE DATABASE b")
33+
34+
_, err := server.SystemLayer().SQLConn(t).Exec("SET CLUSTER SETTING kv.rangefeed.enabled = true")
35+
require.NoError(t, err)
36+
37+
dbA := sqlutils.MakeSQLRunner(server.SQLConn(t, serverutils.DBName("a")))
38+
dbB := sqlutils.MakeSQLRunner(server.SQLConn(t, serverutils.DBName("b")))
39+
40+
// Create a random table in database A
41+
rndSrc, _ := randutil.NewTestRand()
42+
rndSrc.Seed(time.Now().UnixNano())
43+
44+
stmt := GenerateLDRTable(ctx, rndSrc, "test_writer", true)
45+
t.Logf("creating table: %s", stmt)
46+
dbA.Exec(t, stmt)
47+
48+
dbAURL := GetExternalConnectionURI(t, server, server, serverutils.DBName("a"))
49+
dbBURL := GetExternalConnectionURI(t, server, server, serverutils.DBName("b"))
50+
dbB.Exec(t,
51+
"CREATE LOGICALLY REPLICATED TABLE b.test_writer FROM TABLE a.test_writer ON $1 WITH BIDIRECTIONAL ON $2",
52+
dbAURL.String(),
53+
dbBURL.String(),
54+
)
55+
}

‎pkg/crosscluster/replicationtestutils/main_test.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Copyright 2024 The Cockroach Authors.
2+
//
3+
// Use of this software is governed by the CockroachDB Software License
4+
// included in the /LICENSE file.
5+
6+
package replicationtestutils
7+
8+
import (
9+
"os"
10+
"testing"
11+
12+
"github.com/cockroachdb/cockroach/pkg/ccl"
13+
_ "github.com/cockroachdb/cockroach/pkg/ccl/storageccl"
14+
"github.com/cockroachdb/cockroach/pkg/security/securityassets"
15+
"github.com/cockroachdb/cockroach/pkg/security/securitytest"
16+
"github.com/cockroachdb/cockroach/pkg/server"
17+
"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
18+
"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
19+
"github.com/cockroachdb/cockroach/pkg/util/randutil"
20+
)
21+
22+
func TestMain(m *testing.M) {
23+
defer ccl.TestingEnableEnterprise()()
24+
securityassets.SetLoader(securitytest.EmbeddedAssets)
25+
randutil.SeedForTests()
26+
serverutils.InitTestServerFactory(server.TestServerFactory)
27+
serverutils.InitTestClusterFactory(testcluster.TestClusterFactory)
28+
os.Exit(m.Run())
29+
}
30+
31+
//go:generate ../util/leaktest/add-leaktest.sh *_test.go

0 commit comments

Comments
 (0)