Skip to content

Commit ab32360

Browse files
vitalyisaev2vitalyisaev2
authored
PostgreSQL: use int64 instead of int32 when working with histrograms (#336)
* PostgreSQL: int32 bounds -> int64 bounds * regenerate protofiles * PostgreSQL: int32 bounds -> int64 bounds * fix tests * Fix PG container version
1 parent 43b2363 commit ab32360

File tree

7 files changed

+71
-66
lines changed

7 files changed

+71
-66
lines changed

‎app/server/datasource/rdbms/postgresql/split.pb.go‎

Lines changed: 33 additions & 33 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎app/server/datasource/rdbms/postgresql/split.proto‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@ package NYql.Connector.App.Server.DataSource.RDBMS.PostgreSQL;
66

77
option go_package = "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/postgresql/";
88

9-
// TInt32Bounds represents the bounds that can be found for an `INTEGER`, `SERIAL` column
9+
// TInt64Bounds represents the bounds that can be found for an `bigint`, `bigserial` column
1010
// An open interval can be represented by omitting one of the bounds.
11-
message TInt32Bounds {
12-
google.protobuf.Int32Value upper = 1;
13-
google.protobuf.Int32Value lower = 2;
11+
message TInt64Bounds {
12+
google.protobuf.Int64Value upper = 1;
13+
google.protobuf.Int64Value lower = 2;
1414
};
1515

1616
// TSplitDescription represents the description of PostgreSQL's table split.
@@ -25,7 +25,7 @@ message TSplitDescription {
2525
string column_name = 1;
2626

2727
oneof payload {
28-
TInt32Bounds int32_bounds = 2;
28+
TInt64Bounds int64_bounds = 2;
2929
}
3030
}
3131

‎app/server/datasource/rdbms/postgresql/split_provider.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,7 @@ WHERE
303303

304304
defer rows.Close()
305305

306-
var bounds []int32
306+
var bounds []int64
307307

308308
if !rows.Next() {
309309
logger.Warn(
@@ -327,10 +327,10 @@ WHERE
327327
// Add first open interval
328328
result = append(result, &TSplitDescription_THistogramBounds{
329329
ColumnName: pk.columnName,
330-
Payload: &TSplitDescription_THistogramBounds_Int32Bounds{
331-
Int32Bounds: &TInt32Bounds{
330+
Payload: &TSplitDescription_THistogramBounds_Int64Bounds{
331+
Int64Bounds: &TInt64Bounds{
332332
Lower: nil,
333-
Upper: wrapperspb.Int32(bounds[0]),
333+
Upper: wrapperspb.Int64(bounds[0]),
334334
},
335335
},
336336
})
@@ -339,10 +339,10 @@ WHERE
339339
for i := 0; i < len(bounds)-1; i++ {
340340
result = append(result, &TSplitDescription_THistogramBounds{
341341
ColumnName: pk.columnName,
342-
Payload: &TSplitDescription_THistogramBounds_Int32Bounds{
343-
Int32Bounds: &TInt32Bounds{
344-
Lower: wrapperspb.Int32(bounds[i]),
345-
Upper: wrapperspb.Int32(bounds[i+1]),
342+
Payload: &TSplitDescription_THistogramBounds_Int64Bounds{
343+
Int64Bounds: &TInt64Bounds{
344+
Lower: wrapperspb.Int64(bounds[i]),
345+
Upper: wrapperspb.Int64(bounds[i+1]),
346346
},
347347
},
348348
})
@@ -351,9 +351,9 @@ WHERE
351351
// Add last open interval
352352
result = append(result, &TSplitDescription_THistogramBounds{
353353
ColumnName: pk.columnName,
354-
Payload: &TSplitDescription_THistogramBounds_Int32Bounds{
355-
Int32Bounds: &TInt32Bounds{
356-
Lower: wrapperspb.Int32(bounds[len(bounds)-1]),
354+
Payload: &TSplitDescription_THistogramBounds_Int64Bounds{
355+
Int64Bounds: &TInt64Bounds{
356+
Lower: wrapperspb.Int64(bounds[len(bounds)-1]),
357357
Upper: nil,
358358
},
359359
},

‎app/server/datasource/rdbms/postgresql/sql_formatter.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -164,10 +164,10 @@ func (f sqlFormatter) renderSelectQueryTextWithHistogramBounds(
164164
}
165165

166166
switch t := (histogramBounds.Payload).(type) {
167-
case *TSplitDescription_THistogramBounds_Int32Bounds:
168-
out, err := f.renderSelectQueryTextWithInt32Bounds(sb, histogramBounds.ColumnName, t.Int32Bounds)
167+
case *TSplitDescription_THistogramBounds_Int64Bounds:
168+
out, err := f.renderSelectQueryTextWithInt64Bounds(sb, histogramBounds.ColumnName, t.Int64Bounds)
169169
if err != nil {
170-
return "", fmt.Errorf("render select query text with int32 bounds: %w", err)
170+
return "", fmt.Errorf("render select query text with int64 bounds: %w", err)
171171
}
172172

173173
return out, nil
@@ -176,10 +176,10 @@ func (f sqlFormatter) renderSelectQueryTextWithHistogramBounds(
176176
}
177177
}
178178

179-
func (f sqlFormatter) renderSelectQueryTextWithInt32Bounds(
179+
func (f sqlFormatter) renderSelectQueryTextWithInt64Bounds(
180180
sb *strings.Builder,
181181
columnName string,
182-
bounds *TInt32Bounds,
182+
bounds *TInt64Bounds,
183183
) (string, error) {
184184
if columnName == "" {
185185
return "", fmt.Errorf("column name is empty")

‎app/server/datasource/rdbms/postgresql/sql_formatter_test.go‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package postgresql
33
import (
44
"context"
55
"errors"
6+
"fmt"
67
"testing"
78

89
"github.com/golang/protobuf/proto"
@@ -485,10 +486,10 @@ func TestMakeSelectQuery(t *testing.T) {
485486
Payload: &TSplitDescription_HistogramBounds{
486487
HistogramBounds: &TSplitDescription_THistogramBounds{
487488
ColumnName: "id",
488-
Payload: &TSplitDescription_THistogramBounds_Int32Bounds{
489-
Int32Bounds: &TInt32Bounds{
490-
Lower: &wrapperspb.Int32Value{Value: 906},
491-
Upper: &wrapperspb.Int32Value{Value: 677197},
489+
Payload: &TSplitDescription_THistogramBounds_Int64Bounds{
490+
Int64Bounds: &TInt64Bounds{
491+
Lower: &wrapperspb.Int64Value{Value: 906},
492+
Upper: &wrapperspb.Int64Value{Value: 677197},
492493
},
493494
},
494495
},
@@ -530,10 +531,10 @@ func TestMakeSelectQuery(t *testing.T) {
530531
Payload: &TSplitDescription_HistogramBounds{
531532
HistogramBounds: &TSplitDescription_THistogramBounds{
532533
ColumnName: "id",
533-
Payload: &TSplitDescription_THistogramBounds_Int32Bounds{
534-
Int32Bounds: &TInt32Bounds{
535-
Lower: &wrapperspb.Int32Value{Value: 906},
536-
Upper: &wrapperspb.Int32Value{Value: 677197},
534+
Payload: &TSplitDescription_THistogramBounds_Int64Bounds{
535+
Int64Bounds: &TInt64Bounds{
536+
Lower: &wrapperspb.Int64Value{Value: 906},
537+
Upper: &wrapperspb.Int64Value{Value: 677197},
537538
},
538539
},
539540
},
@@ -598,7 +599,11 @@ func TestMakeSelectQuery(t *testing.T) {
598599
require.Equal(t, len(tc.outputYdbTypes), len(actualTypes))
599600

600601
for i := range tc.outputYdbTypes {
601-
require.True(t, proto.Equal(tc.outputYdbTypes[i], actualTypes[i]))
602+
require.True(
603+
t,
604+
proto.Equal(tc.outputYdbTypes[i], actualTypes[i]),
605+
fmt.Sprintf("type #%d: %v %v", i, tc.outputYdbTypes[i], actualTypes[i]),
606+
)
602607
}
603608
})
604609
}

‎scripts/debug/kqprun/script.ydb.local.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,13 @@ PRAGMA generic.UsePredicatePushdown="true";
1313
-- SELECT * FROM external_datasource.`yq-4224` WHERE hash = "6758ddf04f23be19dc7adf08356c697f21dc751aabc1c71b55d340ee920781ca";
1414
-- SELECT * FROM external_datasource.`yq-4224` WHERE hash LIKE "6758%";
1515

16-
-- SELECT * FROM external_datasource.primitives;
16+
SELECT * FROM external_datasource.simple WHERE id = (2 + 3);
1717

1818
-- SELECT * FROM external_datasource.primitives WHERE `col_13_utf8` LIKE Utf8("a%");
1919

2020
-- SELECT * FROM external_datasource.pushdown_regexp WHERE `col_01_string` REGEXP '\\d+';
2121

2222
-- SELECT * FROM external_datasource.pushdown_regexp WHERE `col_02_utf8` REGEXP '\\d+';
2323

24-
SELECT * FROM external_datasource.pushdown_regexp WHERE `col_01_string` LIKE 'a%b_c_%d';
24+
-- SELECT * FROM external_datasource.pushdown_regexp WHERE `col_01_string` LIKE 'a%b%d';
2525

‎tests/infra/datasource/docker-compose.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ services:
1919
hard: 262144
2020

2121
postgresql:
22-
image: mirror.gcr.io/postgres
22+
image: mirror.gcr.io/postgres:16
2323
container_name: ${USER}-fq-connector-go-tests-postgresql
2424
ports:
2525
- '5433:5432'

0 commit comments

Comments
 (0)