Skip to content

Commit 58dbb72

Browse files
vitalyisaev2vitalyisaev2
authored
PostgreSQL: support numeric type (#339)
1 parent c5b4833 commit 58dbb72

30 files changed

+669
-102
lines changed

‎app/server/datasource/interface.go‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,16 @@ type DataSource[T paging.Acceptor] interface {
5858
) error
5959
}
6060

61+
type ColumnDescription struct {
62+
Name string // mandatory
63+
Type string // mandatory
64+
Precision *uint8 // filled only for numeric columns
65+
Scale *uint8 // filled only for numeric columns
66+
}
67+
6168
type TypeMapper interface {
62-
SQLTypeToYDBColumn(columnName, typeName string, rules *api_service_protos.TTypeMappingSettings) (*Ydb.Column, error)
69+
SQLTypeToYDBColumn(
70+
columnDescription *ColumnDescription,
71+
rules *api_service_protos.TTypeMappingSettings,
72+
) (*Ydb.Column, error)
6373
}

‎app/server/datasource/rdbms/clickhouse/table_metadata_query.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
)
77

88
func TableMetadataQuery(request *api_service_protos.TDescribeTableRequest) (string, *rdbms_utils.QueryArgs) {
9-
query := "SELECT name, type FROM system.columns WHERE table = ? and database = ?"
9+
query := "SELECT name, type, numeric_precision, numeric_scale FROM system.columns WHERE table = ? and database = ?"
1010

1111
var args rdbms_utils.QueryArgs
1212

‎app/server/datasource/rdbms/clickhouse/type_mapping.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ type typeMapper struct {
2828

2929
//nolint:gocyclo
3030
func (tm typeMapper) SQLTypeToYDBColumn(
31-
columnName, typeName string,
31+
columnDescription *datasource.ColumnDescription,
3232
rules *api_service_protos.TTypeMappingSettings,
3333
) (*Ydb.Column, error) {
3434
var (
@@ -46,6 +46,8 @@ func (tm typeMapper) SQLTypeToYDBColumn(
4646
arrayContainer := false
4747
innerNullable := false
4848

49+
typeName := columnDescription.Type
50+
4951
if matches := tm.isNullable.FindStringSubmatch(typeName); len(matches) > 0 {
5052
nullable = true
5153
typeName = matches[1]
@@ -128,7 +130,7 @@ func (tm typeMapper) SQLTypeToYDBColumn(
128130
}
129131

130132
return &Ydb.Column{
131-
Name: columnName,
133+
Name: columnDescription.Name,
132134
Type: ydbType,
133135
}, nil
134136
}

‎app/server/datasource/rdbms/ms_sql_server/table_metadata_query.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ import (
99

1010
func TableMetadataQuery(request *api_service_protos.TDescribeTableRequest) (string, *rdbms_utils.QueryArgs) {
1111
// opts := request.GetDataSourceInstance().GetPgOptions().GetSchema()
12-
query := "SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = @p1;"
12+
query := `SELECT column_name, data_type, numeric_precision, numeric_scale
13+
FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = @p1;`
1314

1415
var args rdbms_utils.QueryArgs
1516

‎app/server/datasource/rdbms/ms_sql_server/type_mapping.go‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,10 @@ var _ datasource.TypeMapper = typeMapper{}
2121
type typeMapper struct{}
2222

2323
//nolint:gocyclo
24-
func (typeMapper) SQLTypeToYDBColumn(columnName, typeName string, rules *api_service_protos.TTypeMappingSettings) (*Ydb.Column, error) {
24+
func (typeMapper) SQLTypeToYDBColumn(
25+
columnDescription *datasource.ColumnDescription,
26+
rules *api_service_protos.TTypeMappingSettings,
27+
) (*Ydb.Column, error) {
2528
var (
2629
ydbType *Ydb.Type
2730
err error
@@ -31,7 +34,7 @@ func (typeMapper) SQLTypeToYDBColumn(columnName, typeName string, rules *api_ser
3134

3235
// MS SQL Server Data Types https://learn.microsoft.com/ru-ru/sql/t-sql/data-types/data-types-transact-sql?view=sql-server-ver16
3336
// Reference table: https://github.com/ydb-platform/fq-connector-go/blob/main/docs/type_mapping_table.md
34-
switch typeName {
37+
switch columnDescription.Type {
3538
case "bit":
3639
ydbType = common.MakePrimitiveType(Ydb.Type_BOOL)
3740
case "tinyint":
@@ -72,17 +75,17 @@ func (typeMapper) SQLTypeToYDBColumn(columnName, typeName string, rules *api_ser
7275
return nil, fmt.Errorf("make YDB date time type: %w", err)
7376
}
7477
default:
75-
return nil, fmt.Errorf("convert type '%s': %w", typeName, common.ErrDataTypeNotSupported)
78+
return nil, fmt.Errorf("convert type '%s': %w", columnDescription.Type, common.ErrDataTypeNotSupported)
7679
}
7780

7881
if err != nil {
79-
return nil, fmt.Errorf("convert type '%s': %w", typeName, err)
82+
return nil, fmt.Errorf("convert type '%s': %w", columnDescription.Type, err)
8083
}
8184

8285
ydbType = common.MakeOptionalType(ydbType)
8386

8487
return &Ydb.Column{
85-
Name: columnName,
88+
Name: columnDescription.Name,
8689
Type: ydbType,
8790
}, nil
8891
}

‎app/server/datasource/rdbms/mysql/table_metadata_query.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,16 @@ import (
77

88
func TableMetadataQuery(request *api_service_protos.TDescribeTableRequest) (string, *rdbms_utils.QueryArgs) {
99
// TODO: do not add 'unsigned' modifiers to column type and use the driver-provided fields instead.
10+
//
1011
// In MySQL schema and database are basically the same thing. So we can safely pass dbname as
1112
// `schema_name` when quering `information_schema`.
12-
query := `SELECT column_name, column_type FROM information_schema.columns
13-
WHERE table_name = ? AND table_schema = ?`
13+
//
14+
// TODO: learn how to extract numeric precision and scale from MySQL
15+
query := `SELECT
16+
column_name,
17+
column_type
18+
FROM information_schema.columns
19+
WHERE table_name = ? AND table_schema = ?`
1420

1521
var args rdbms_utils.QueryArgs
1622

‎app/server/datasource/rdbms/mysql/type_mapping.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ type typeMapper struct {
2828

2929
//nolint:gocyclo
3030
func (tm *typeMapper) SQLTypeToYDBColumn(
31-
columnName, columnType string,
31+
columnDescription *datasource.ColumnDescription,
3232
typeMapperSettings *api_service_protos.TTypeMappingSettings,
3333
) (*Ydb.Column, error) {
3434
var (
@@ -37,9 +37,9 @@ func (tm *typeMapper) SQLTypeToYDBColumn(
3737
err error
3838
)
3939

40-
typeNameWithoutModifier := strings.Split(columnType, " ")[0]
40+
typeNameWithoutModifier := strings.Split(columnDescription.Type, " ")[0]
4141

42-
if matches := tm.reType.FindStringSubmatch(columnType); len(matches) > 0 {
42+
if matches := tm.reType.FindStringSubmatch(columnDescription.Type); len(matches) > 0 {
4343
typeName = matches[tm.reType.SubexpIndex("type")]
4444
typeSize, err = strconv.ParseUint(matches[tm.reType.SubexpIndex("size")], 10, 64)
4545

@@ -50,9 +50,9 @@ func (tm *typeMapper) SQLTypeToYDBColumn(
5050
typeName = typeNameWithoutModifier
5151
}
5252

53-
unsigned := strings.Contains(columnType, "unsigned")
53+
unsigned := strings.Contains(columnDescription.Type, "unsigned")
5454

55-
ydbColumn := Ydb.Column{Name: columnName}
55+
ydbColumn := Ydb.Column{Name: columnDescription.Name}
5656

5757
switch typeName {
5858
case typeInt, typeMediumInt:

‎app/server/datasource/rdbms/oracle/table_metadata_query.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,10 @@ import (
88
func TableMetadataQuery(request *api_service_protos.TDescribeTableRequest) (string, *rdbms_utils.QueryArgs) {
99
// TODO YQ-3413: synonym tables and from other users.
1010
// TODO YQ-3454: all capitalize
11-
query := "SELECT column_name, data_type FROM user_tab_columns WHERE table_name = :1"
11+
// query := `SELECT column_name, data_type, DATA_PRECISION, DATA_SCALE
12+
// FROM user_tab_columns WHERE table_name = :1`
13+
query := `SELECT column_name, data_type
14+
FROM user_tab_columns WHERE table_name = :1`
1215

1316
var args rdbms_utils.QueryArgs
1417

‎app/server/datasource/rdbms/oracle/type_mapping.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,19 @@ type typeMapper struct {
2525
isTimestampWLTZ *regexp.Regexp
2626
}
2727

28-
func (tm typeMapper) SQLTypeToYDBColumn(columnName, typeName string, rules *api_service_protos.TTypeMappingSettings) (*Ydb.Column, error) {
28+
func (tm typeMapper) SQLTypeToYDBColumn(
29+
columnDescription *datasource.ColumnDescription,
30+
rules *api_service_protos.TTypeMappingSettings,
31+
) (*Ydb.Column, error) {
2932
var (
3033
ydbType *Ydb.Type
3134
err error
3235
)
3336

3437
_ = rules
3538

39+
typeName := columnDescription.Type
40+
3641
// Oracle Data Types
3742
// https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html#GUID-7B72E154-677A-4342-A1EA-C74C1EA928E6
3843
// Reference table: https://github.com/ydb-platform/fq-connector-go/blob/main/docs/type_mapping_table.md
@@ -77,7 +82,7 @@ func (tm typeMapper) SQLTypeToYDBColumn(columnName, typeName string, rules *api_
7782
ydbType = common.MakeOptionalType(ydbType)
7883

7984
return &Ydb.Column{
80-
Name: columnName,
85+
Name: columnDescription.Name,
8186
Type: ydbType,
8287
}, nil
8388
}

‎app/server/datasource/rdbms/postgresql/table_metadata_query.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ func TableMetadataQuery(
99
request *api_service_protos.TDescribeTableRequest,
1010
schema string,
1111
) (string, *rdbms_utils.QueryArgs) {
12-
query := "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = $1 AND table_schema = $2"
12+
query := "SELECT column_name, data_type, numeric_precision, numeric_scale " +
13+
"FROM information_schema.columns WHERE table_name = $1 AND table_schema = $2"
1314

1415
var args rdbms_utils.QueryArgs
1516

0 commit comments

Comments
 (0)