Skip to content

Commit e0ea92b

Browse files
asmyasnikovasmyasnikov
authored
Merge pull request #1580 from ydb-platform/xsql
database/sql over query service client
2 parents 9a4b4f8 + 8f69ba5 commit e0ea92b

File tree

102 files changed

+3223
-2107
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

102 files changed

+3223
-2107
lines changed

‎.github/workflows/examples.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
strategy:
1616
fail-fast: false
1717
matrix:
18-
ydb-version: [ 23.3, 24.1 ]
18+
ydb-version: [ 24.1, 24.2, 24.3 ]
1919
application: [ native/table, native/query, database_sql, gorm, xorm ]
2020
services:
2121
ydb:

‎.github/workflows/tests.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
strategy:
2020
fail-fast: false
2121
matrix:
22-
go-version: [1.21.x, 1.22.x, 1.23.x]
22+
go-version: [1.22.x, 1.23.x]
2323
os: [ubuntu, windows, macOS]
2424
env:
2525
OS: ${{ matrix.os }}-latest
@@ -52,8 +52,8 @@ jobs:
5252
strategy:
5353
fail-fast: false
5454
matrix:
55-
go-version: [1.21.x, 1.22.x, 1.23.x]
56-
ydb-version: [23.3, 24.1, 24.2]
55+
go-version: [1.22.x, 1.23.x]
56+
ydb-version: [24.1, 24.2, 24.3]
5757
os: [ubuntu]
5858
services:
5959
ydb:
@@ -133,6 +133,7 @@ jobs:
133133
YDB_CONNECTION_STRING_SECURE: grpcs://localhost:2135/local
134134
YDB_SSL_ROOT_CERTIFICATES_FILE: /tmp/ydb_certs/ca.pem
135135
YDB_SESSIONS_SHUTDOWN_URLS: http://localhost:8765/actors/kqp_proxy?force_shutdown=all
136+
YDB_DATABASE_SQL_OVER_QUERY_SERVICE: 1
136137
HIDE_APPLICATION_OUTPUT: 1
137138
steps:
138139
- name: Checkout code

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
* Added implementation of `database/sql` driver over query service client
2+
* Added `ydb.WithQueryService(bool)` option to explicitly enable `database/sql` driver over query service client
3+
* Added environment parameter `YDB_DATABASE_SQL_OVER_QUERY_SERVICE` to enable `database/sql` driver over query service client without code rewriting
4+
15
## v3.94.0
26
* Refactored golang types mapping into ydb types using `ydb.ParamsFromMap` and `database/sql` query arguments
37
* Small breaking change: type mapping for `ydb.ParamsFromMap` and `database/sql` type `uuid.UUID` changed from ydb type `Text` to ydb type `UUID`

‎driver.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import (
1414
"github.com/ydb-platform/ydb-go-sdk/v3/discovery"
1515
"github.com/ydb-platform/ydb-go-sdk/v3/internal/balancer"
1616
"github.com/ydb-platform/ydb-go-sdk/v3/internal/conn"
17-
"github.com/ydb-platform/ydb-go-sdk/v3/internal/connector"
1817
internalCoordination "github.com/ydb-platform/ydb-go-sdk/v3/internal/coordination"
1918
coordinationConfig "github.com/ydb-platform/ydb-go-sdk/v3/internal/coordination/config"
2019
"github.com/ydb-platform/ydb-go-sdk/v3/internal/credentials"
@@ -37,6 +36,7 @@ import (
3736
"github.com/ydb-platform/ydb-go-sdk/v3/internal/topic/topicclientinternal"
3837
"github.com/ydb-platform/ydb-go-sdk/v3/internal/xcontext"
3938
"github.com/ydb-platform/ydb-go-sdk/v3/internal/xerrors"
39+
"github.com/ydb-platform/ydb-go-sdk/v3/internal/xsql"
4040
"github.com/ydb-platform/ydb-go-sdk/v3/internal/xsync"
4141
"github.com/ydb-platform/ydb-go-sdk/v3/log"
4242
"github.com/ydb-platform/ydb-go-sdk/v3/operation"
@@ -93,7 +93,7 @@ type (
9393
topic *xsync.Once[*topicclientinternal.Client]
9494
topicOptions []topicoptions.TopicOption
9595

96-
databaseSQLOptions []connector.Option
96+
databaseSQLOptions []xsql.Option
9797

9898
pool *conn.Pool
9999

‎dsn.go‎

Lines changed: 38 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,9 @@ import (
99
"github.com/ydb-platform/ydb-go-sdk/v3/balancers"
1010
"github.com/ydb-platform/ydb-go-sdk/v3/credentials"
1111
"github.com/ydb-platform/ydb-go-sdk/v3/internal/bind"
12-
"github.com/ydb-platform/ydb-go-sdk/v3/internal/connector"
1312
"github.com/ydb-platform/ydb-go-sdk/v3/internal/dsn"
14-
tableSql "github.com/ydb-platform/ydb-go-sdk/v3/internal/table/conn"
1513
"github.com/ydb-platform/ydb-go-sdk/v3/internal/xerrors"
14+
"github.com/ydb-platform/ydb-go-sdk/v3/internal/xsql"
1615
)
1716

1817
const tablePathPrefixTransformer = "table_path_prefix"
@@ -44,6 +43,22 @@ func UnregisterDsnParser(registrationID int) {
4443
dsnParsers[registrationID] = nil
4544
}
4645

46+
var stringToType = map[string]QueryMode{
47+
"data": DataQueryMode,
48+
"scan": ScanQueryMode,
49+
"scheme": SchemeQueryMode,
50+
"scripting": ScriptingQueryMode,
51+
"query": QueryExecuteQueryMode,
52+
}
53+
54+
func queryModeFromString(s string) QueryMode {
55+
if t, ok := stringToType[s]; ok {
56+
return t
57+
}
58+
59+
return unknownQueryMode
60+
}
61+
4762
//nolint:funlen
4863
func parseConnectionString(dataSourceName string) (opts []Option, _ error) {
4964
info, err := dsn.Parse(dataSourceName)
@@ -60,45 +75,52 @@ func parseConnectionString(dataSourceName string) (opts []Option, _ error) {
6075
opts = append(opts, WithBalancer(balancers.FromConfig(balancer)))
6176
}
6277
if queryMode := info.Params.Get("go_query_mode"); queryMode != "" {
63-
mode := tableSql.QueryModeFromString(queryMode)
64-
if mode == tableSql.UnknownQueryMode {
78+
switch mode := queryModeFromString(queryMode); mode {
79+
case QueryExecuteQueryMode:
80+
opts = append(opts, withConnectorOptions(xsql.WithQueryService(true)))
81+
case unknownQueryMode:
6582
return nil, xerrors.WithStackTrace(fmt.Errorf("unknown query mode: %s", queryMode))
83+
default:
84+
opts = append(opts, withConnectorOptions(xsql.WithDefaultQueryMode(modeToMode(mode))))
6685
}
67-
opts = append(opts, withConnectorOptions(connector.WithDefaultQueryMode(mode)))
6886
} else if queryMode := info.Params.Get("query_mode"); queryMode != "" {
69-
mode := tableSql.QueryModeFromString(queryMode)
70-
if mode == tableSql.UnknownQueryMode {
87+
switch mode := queryModeFromString(queryMode); mode {
88+
case QueryExecuteQueryMode:
89+
opts = append(opts, withConnectorOptions(xsql.WithQueryService(true)))
90+
case unknownQueryMode:
7191
return nil, xerrors.WithStackTrace(fmt.Errorf("unknown query mode: %s", queryMode))
92+
default:
93+
opts = append(opts, withConnectorOptions(xsql.WithDefaultQueryMode(modeToMode(mode))))
7294
}
73-
opts = append(opts, withConnectorOptions(connector.WithDefaultQueryMode(mode)))
7495
}
7596
if fakeTx := info.Params.Get("go_fake_tx"); fakeTx != "" {
7697
for _, queryMode := range strings.Split(fakeTx, ",") {
77-
mode := tableSql.QueryModeFromString(queryMode)
78-
if mode == tableSql.UnknownQueryMode {
98+
switch mode := queryModeFromString(queryMode); mode {
99+
case unknownQueryMode:
79100
return nil, xerrors.WithStackTrace(fmt.Errorf("unknown query mode: %s", queryMode))
101+
default:
102+
opts = append(opts, withConnectorOptions(WithFakeTx(mode)))
80103
}
81-
opts = append(opts, withConnectorOptions(connector.WithFakeTx(mode)))
82104
}
83105
}
84106
if info.Params.Has("go_query_bind") {
85-
var binders []connector.Option
107+
var binders []xsql.Option
86108
queryTransformers := strings.Split(info.Params.Get("go_query_bind"), ",")
87109
for _, transformer := range queryTransformers {
88110
switch transformer {
89111
case "declare":
90-
binders = append(binders, connector.WithQueryBind(bind.AutoDeclare{}))
112+
binders = append(binders, xsql.WithQueryBind(bind.AutoDeclare{}))
91113
case "positional":
92-
binders = append(binders, connector.WithQueryBind(bind.PositionalArgs{}))
114+
binders = append(binders, xsql.WithQueryBind(bind.PositionalArgs{}))
93115
case "numeric":
94-
binders = append(binders, connector.WithQueryBind(bind.NumericArgs{}))
116+
binders = append(binders, xsql.WithQueryBind(bind.NumericArgs{}))
95117
default:
96118
if strings.HasPrefix(transformer, tablePathPrefixTransformer) {
97119
prefix, err := extractTablePathPrefixFromBinderName(transformer)
98120
if err != nil {
99121
return nil, xerrors.WithStackTrace(err)
100122
}
101-
binders = append(binders, connector.WithQueryBind(bind.TablePathPrefix(prefix)))
123+
binders = append(binders, xsql.WithQueryBind(bind.TablePathPrefix(prefix)))
102124
} else {
103125
return nil, xerrors.WithStackTrace(
104126
fmt.Errorf("unknown query rewriter: %s", transformer),

‎dsn_test.go‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@ import (
88

99
"github.com/ydb-platform/ydb-go-sdk/v3/config"
1010
"github.com/ydb-platform/ydb-go-sdk/v3/internal/bind"
11-
"github.com/ydb-platform/ydb-go-sdk/v3/internal/connector"
12-
querySql "github.com/ydb-platform/ydb-go-sdk/v3/internal/query/conn"
13-
tableSql "github.com/ydb-platform/ydb-go-sdk/v3/internal/table/conn"
11+
"github.com/ydb-platform/ydb-go-sdk/v3/internal/xsql"
12+
"github.com/ydb-platform/ydb-go-sdk/v3/internal/xsql/legacy"
13+
"github.com/ydb-platform/ydb-go-sdk/v3/internal/xsql/propose"
1414
)
1515

1616
func TestParse(t *testing.T) {
17-
newConnector := func(opts ...connector.Option) *connector.Connector {
18-
c := &connector.Connector{}
17+
newConnector := func(opts ...xsql.Option) *xsql.Connector {
18+
c := &xsql.Connector{}
1919
for _, opt := range opts {
2020
if opt != nil {
2121
if err := opt.Apply(c); err != nil {
@@ -26,11 +26,11 @@ func TestParse(t *testing.T) {
2626

2727
return c
2828
}
29-
newTableConn := func(opts ...tableSql.Option) *tableSql.Conn {
30-
return tableSql.New(context.Background(), nil, nil, opts...)
29+
newLegacyConn := func(opts ...legacy.Option) *legacy.Conn {
30+
return legacy.New(context.Background(), nil, nil, opts...)
3131
}
32-
newQueryConn := func(opts ...querySql.Option) *querySql.Conn {
33-
return querySql.New(context.Background(), nil, nil, opts...)
32+
newQueryConn := func(opts ...propose.Option) *propose.Conn {
33+
return propose.New(context.Background(), nil, nil, opts...)
3434
}
3535
compareConfigs := func(t *testing.T, lhs, rhs *config.Config) {
3636
require.Equal(t, lhs.Secure(), rhs.Secure())
@@ -40,7 +40,7 @@ func TestParse(t *testing.T) {
4040
for _, tt := range []struct {
4141
dsn string
4242
opts []config.Option
43-
connectorOpts []connector.Option
43+
connectorOpts []xsql.Option
4444
err error
4545
}{
4646
{
@@ -70,8 +70,8 @@ func TestParse(t *testing.T) {
7070
config.WithEndpoint("localhost:2135"),
7171
config.WithDatabase("/local"),
7272
},
73-
connectorOpts: []connector.Option{
74-
connector.WithDefaultQueryMode(tableSql.ScriptingQueryMode),
73+
connectorOpts: []xsql.Option{
74+
xsql.WithDefaultQueryMode(legacy.ScriptingQueryMode),
7575
},
7676
err: nil,
7777
},
@@ -82,9 +82,9 @@ func TestParse(t *testing.T) {
8282
config.WithEndpoint("localhost:2135"),
8383
config.WithDatabase("/local"),
8484
},
85-
connectorOpts: []connector.Option{
86-
connector.WithDefaultQueryMode(tableSql.ScriptingQueryMode),
87-
connector.WithQueryBind(bind.TablePathPrefix("path/to/tables")),
85+
connectorOpts: []xsql.Option{
86+
xsql.WithDefaultQueryMode(legacy.ScriptingQueryMode),
87+
xsql.WithQueryBind(bind.TablePathPrefix("path/to/tables")),
8888
},
8989
err: nil,
9090
},
@@ -95,10 +95,10 @@ func TestParse(t *testing.T) {
9595
config.WithEndpoint("localhost:2135"),
9696
config.WithDatabase("/local"),
9797
},
98-
connectorOpts: []connector.Option{
99-
connector.WithDefaultQueryMode(tableSql.ScriptingQueryMode),
100-
connector.WithQueryBind(bind.TablePathPrefix("path/to/tables")),
101-
connector.WithQueryBind(bind.NumericArgs{}),
98+
connectorOpts: []xsql.Option{
99+
xsql.WithDefaultQueryMode(legacy.ScriptingQueryMode),
100+
xsql.WithQueryBind(bind.TablePathPrefix("path/to/tables")),
101+
xsql.WithQueryBind(bind.NumericArgs{}),
102102
},
103103
err: nil,
104104
},
@@ -109,10 +109,10 @@ func TestParse(t *testing.T) {
109109
config.WithEndpoint("localhost:2135"),
110110
config.WithDatabase("/local"),
111111
},
112-
connectorOpts: []connector.Option{
113-
connector.WithDefaultQueryMode(tableSql.ScriptingQueryMode),
114-
connector.WithQueryBind(bind.TablePathPrefix("path/to/tables")),
115-
connector.WithQueryBind(bind.PositionalArgs{}),
112+
connectorOpts: []xsql.Option{
113+
xsql.WithDefaultQueryMode(legacy.ScriptingQueryMode),
114+
xsql.WithQueryBind(bind.TablePathPrefix("path/to/tables")),
115+
xsql.WithQueryBind(bind.PositionalArgs{}),
116116
},
117117
err: nil,
118118
},
@@ -123,10 +123,10 @@ func TestParse(t *testing.T) {
123123
config.WithEndpoint("localhost:2135"),
124124
config.WithDatabase("/local"),
125125
},
126-
connectorOpts: []connector.Option{
127-
connector.WithDefaultQueryMode(tableSql.ScriptingQueryMode),
128-
connector.WithQueryBind(bind.TablePathPrefix("path/to/tables")),
129-
connector.WithQueryBind(bind.AutoDeclare{}),
126+
connectorOpts: []xsql.Option{
127+
xsql.WithDefaultQueryMode(legacy.ScriptingQueryMode),
128+
xsql.WithQueryBind(bind.TablePathPrefix("path/to/tables")),
129+
xsql.WithQueryBind(bind.AutoDeclare{}),
130130
},
131131
err: nil,
132132
},
@@ -137,9 +137,9 @@ func TestParse(t *testing.T) {
137137
config.WithEndpoint("localhost:2135"),
138138
config.WithDatabase("/local"),
139139
},
140-
connectorOpts: []connector.Option{
141-
connector.WithDefaultQueryMode(tableSql.ScriptingQueryMode),
142-
connector.WithQueryBind(bind.TablePathPrefix("path/to/tables")),
140+
connectorOpts: []xsql.Option{
141+
xsql.WithDefaultQueryMode(legacy.ScriptingQueryMode),
142+
xsql.WithQueryBind(bind.TablePathPrefix("path/to/tables")),
143143
},
144144
err: nil,
145145
},
@@ -150,11 +150,11 @@ func TestParse(t *testing.T) {
150150
config.WithEndpoint("localhost:2135"),
151151
config.WithDatabase("/local"),
152152
},
153-
connectorOpts: []connector.Option{
154-
connector.WithDefaultQueryMode(tableSql.ScriptingQueryMode),
155-
connector.WithQueryBind(bind.TablePathPrefix("path/to/tables")),
156-
connector.WithQueryBind(bind.PositionalArgs{}),
157-
connector.WithQueryBind(bind.AutoDeclare{}),
153+
connectorOpts: []xsql.Option{
154+
xsql.WithDefaultQueryMode(legacy.ScriptingQueryMode),
155+
xsql.WithQueryBind(bind.TablePathPrefix("path/to/tables")),
156+
xsql.WithQueryBind(bind.PositionalArgs{}),
157+
xsql.WithQueryBind(bind.AutoDeclare{}),
158158
},
159159
err: nil,
160160
},
@@ -165,9 +165,9 @@ func TestParse(t *testing.T) {
165165
config.WithEndpoint("localhost:2135"),
166166
config.WithDatabase("/local"),
167167
},
168-
connectorOpts: []connector.Option{
169-
connector.WithFakeTx(tableSql.ScriptingQueryMode),
170-
connector.WithFakeTx(tableSql.SchemeQueryMode),
168+
connectorOpts: []xsql.Option{
169+
WithFakeTx(ScriptingQueryMode),
170+
WithFakeTx(SchemeQueryMode),
171171
},
172172
err: nil,
173173
},
@@ -183,15 +183,15 @@ func TestParse(t *testing.T) {
183183
exp := newConnector(tt.connectorOpts...)
184184
act := newConnector(d.databaseSQLOptions...)
185185
t.Run("tableOptions", func(t *testing.T) {
186-
require.Equal(t, newTableConn(exp.TableOpts...), newTableConn(act.TableOpts...))
186+
require.Equal(t, newLegacyConn(exp.LegacyOpts...), newLegacyConn(act.LegacyOpts...))
187187
})
188188
t.Run("queryOptions", func(t *testing.T) {
189-
require.Equal(t, newQueryConn(exp.QueryOpts...), newQueryConn(act.QueryOpts...))
189+
require.Equal(t, newQueryConn(exp.Options...), newQueryConn(act.Options...))
190190
})
191-
exp.TableOpts = nil
192-
exp.QueryOpts = nil
193-
act.TableOpts = nil
194-
act.QueryOpts = nil
191+
exp.LegacyOpts = nil
192+
exp.Options = nil
193+
act.LegacyOpts = nil
194+
act.Options = nil
195195
require.Equal(t, exp.Bindings(), act.Bindings())
196196
require.Equal(t, exp, act)
197197
compareConfigs(t, config.New(tt.opts...), d.config)

‎examples/ddl/ddl.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,10 @@ ALTER TABLE small_table3 SET (TTL = Interval("PT3H") ON d);
6767
`
6868
)
6969

70-
func executeQuery(ctx context.Context, c table.Client, prefix, query string) (err error) {
70+
func executeQuery(ctx context.Context, c table.Client, prefix, sql string) (err error) {
7171
err = c.Do(ctx,
7272
func(ctx context.Context, s table.Session) error {
73-
err = s.ExecuteSchemeQuery(ctx, fmt.Sprintf(query, prefix))
73+
err = s.ExecuteSchemeQuery(ctx, fmt.Sprintf(sql, prefix))
7474

7575
return err
7676
},

‎internal/bind/auto_declare.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ func (m AutoDeclare) blockID() blockID {
1313
return blockDeclare
1414
}
1515

16-
func (m AutoDeclare) RewriteQuery(query string, args ...interface{}) (
16+
func (m AutoDeclare) ToYdb(sql string, args ...interface{}) (
1717
yql string, newArgs []interface{}, err error,
1818
) {
1919
params, err := Params(args...)
@@ -22,7 +22,7 @@ func (m AutoDeclare) RewriteQuery(query string, args ...interface{}) (
2222
}
2323

2424
if len(params) == 0 {
25-
return query, args, nil
25+
return sql, args, nil
2626
}
2727

2828
var (
@@ -46,7 +46,7 @@ func (m AutoDeclare) RewriteQuery(query string, args ...interface{}) (
4646

4747
buffer.WriteByte('\n')
4848

49-
buffer.WriteString(query)
49+
buffer.WriteString(sql)
5050

5151
for _, param := range params {
5252
newArgs = append(newArgs, param)

0 commit comments

Comments
 (0)