Skip to content

Commit 0054c3c

Browse files
killme2008killme2008
authored
feat: impl row-based protocol and custom time index column name (#51)
* feat: bump greptimedb gRPC protos Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * feat: replace column-based gRPC with row-based Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * feat: refactor encoder Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * feat: sort columns by semantic type and name Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * feat: supports custom ts column name Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * chore: tweak tests and apply suggestions Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * chore: comment Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * docs: remove version note Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * fix: log Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * chore: add benchmark info Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * fix: format Signed-off-by: Dennis Zhuang <killme2008@gmail.com> --------- Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
1 parent b99a314 commit 0054c3c

16 files changed

+49559
-24162
lines changed

‎README.md‎

Lines changed: 276 additions & 170 deletions
Large diffs are not rendered by default.

‎protos/column.proto‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,12 @@ syntax = "proto3";
1616

1717
package greptime.v1;
1818

19-
option java_package = "io.greptime.v1";
20-
option java_outer_classname = "Columns";
21-
option go_package = "github.com/GreptimeTeam/greptime-proto/go/greptime/v1";
22-
2319
import "protos/common.proto";
20+
import "protos/row.proto";
21+
22+
option go_package = "github.com/GreptimeTeam/greptime-proto/go/greptime/v1";
23+
option java_outer_classname = "Columns";
24+
option java_package = "io.greptime.v1";
2425

2526
message Column {
2627
string column_name = 1;
@@ -57,12 +58,12 @@ message Column {
5758
repeated int32 interval_year_month_values = 24;
5859
repeated int64 interval_day_time_values = 25;
5960
repeated IntervalMonthDayNano interval_month_day_nano_values = 26;
60-
repeated int64 duration_second_values = 27;
61-
repeated int64 duration_millisecond_values = 28;
62-
repeated int64 duration_microsecond_values = 29;
63-
repeated int64 duration_nanosecond_values = 30;
6461
repeated Decimal128 decimal128_values = 31;
62+
63+
repeated ListValue list_values = 40;
64+
repeated StructValue struct_values = 41;
6565
}
66+
6667
// The array of non-null values in this column.
6768
//
6869
// For example: suppose there is a column "foo" that contains some int32
@@ -84,4 +85,7 @@ message Column {
8485

8586
// Extension for ColumnDataType.
8687
ColumnDataTypeExtension datatype_extension = 6;
88+
89+
// Additional column options.
90+
ColumnOptions options = 7;
8791
}

‎protos/common.proto‎

Lines changed: 143 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,34 @@ syntax = "proto3";
1616

1717
package greptime.v1;
1818

19-
option java_package = "io.greptime.v1";
20-
option java_outer_classname = "Common";
2119
option go_package = "github.com/GreptimeTeam/greptime-proto/go/greptime/v1";
20+
option java_outer_classname = "Common";
21+
option java_package = "io.greptime.v1";
22+
23+
message QueryContext {
24+
string current_catalog = 1;
25+
string current_schema = 2;
26+
string timezone = 4;
27+
map<string, string> extensions = 5;
28+
uint32 channel = 6;
29+
SnapshotSequences snapshot_seqs = 7;
30+
// Explain options for the query.
31+
ExplainOptions explain = 8;
32+
}
33+
34+
message SnapshotSequences {
35+
// Mappings of the RegionId to the minimal sequence of SST file to scan.
36+
map<uint64, uint64> sst_min_sequences = 1;
37+
// mapping of RegionId to SequenceNumber, for snapshot read, meaning that the
38+
// read should only container data that was committed before (and include) the
39+
// given sequence number
40+
map<uint64, uint64> snapshot_seqs = 7;
41+
}
42+
43+
message ExplainOptions {
44+
// Whether to enable verbose explain output.
45+
bool verbose = 1;
46+
}
2247

2348
message RequestHeader {
2449
// The `catalog` that is selected to be used in this request.
@@ -32,9 +57,13 @@ message RequestHeader {
3257
// Encoded trace_id & span_id, follow the w3c Trace Context
3358
// https://www.w3.org/TR/trace-context/#header-name
3459
map<string, string> tracing_context = 5;
60+
// The `timezone` for the request
61+
string timezone = 6;
3562
}
3663

37-
message ResponseHeader { Status status = 1; }
64+
message ResponseHeader {
65+
Status status = 1;
66+
}
3867

3968
message Status {
4069
// Corresponding to the `StatusCode` definition of GreptimeDB
@@ -54,11 +83,37 @@ message Basic {
5483
string password = 2;
5584
}
5685

57-
message Token { string token = 1; }
86+
message Token {
87+
string token = 1;
88+
}
89+
90+
message TableName {
91+
string catalog_name = 1;
92+
string schema_name = 2;
93+
string table_name = 3;
94+
}
95+
96+
message AffectedRows {
97+
uint32 value = 1;
98+
}
5899

59-
message AffectedRows { uint32 value = 1; }
100+
message Metrics {
101+
bytes metrics = 1;
102+
}
103+
104+
message ExpireAfter {
105+
int64 value = 1;
106+
}
107+
108+
// Eval Interval in seconds
109+
message EvalInterval {
110+
int64 seconds = 1;
111+
}
60112

61-
message FlightMetadata { AffectedRows affected_rows = 1; }
113+
message FlightMetadata {
114+
AffectedRows affected_rows = 1;
115+
Metrics metrics = 2;
116+
}
62117

63118
enum SemanticType {
64119
TAG = 0;
@@ -93,11 +148,11 @@ enum ColumnDataType {
93148
INTERVAL_YEAR_MONTH = 23;
94149
INTERVAL_DAY_TIME = 24;
95150
INTERVAL_MONTH_DAY_NANO = 25;
96-
DURATION_SECOND = 26;
97-
DURATION_MILLISECOND = 27;
98-
DURATION_MICROSECOND = 28;
99-
DURATION_NANOSECOND = 29;
100151
DECIMAL128 = 30;
152+
JSON = 31;
153+
VECTOR = 32;
154+
LIST = 40;
155+
STRUCT = 41;
101156
}
102157

103158
message IntervalMonthDayNano {
@@ -106,20 +161,98 @@ message IntervalMonthDayNano {
106161
int64 nanoseconds = 3;
107162
}
108163

164+
/// Time Unit for timestamp
165+
enum TimeUnit {
166+
// The default time unit is MILLISECOND
167+
MILLISECOND = 0;
168+
SECOND = 1;
169+
MICROSECOND = 2;
170+
NANOSECOND = 3;
171+
}
172+
173+
message TimeRange {
174+
int64 start = 1;
175+
int64 end = 2;
176+
}
177+
178+
/// Truncate partially by time ranges
179+
message TimeRanges {
180+
TimeUnit time_unit = 1;
181+
repeated TimeRange time_ranges = 2;
182+
}
183+
109184
// (hi: high 64 bits, lo: low 64 bits) are used to keep the decimal128 value.
110185
message Decimal128 {
111186
int64 hi = 1;
112187
int64 lo = 2;
113188
}
114189

190+
message StructField {
191+
string name = 1;
192+
ColumnDataType datatype = 2;
193+
ColumnDataTypeExtension datatype_extension = 3;
194+
}
195+
115196
// Type extension for some complex types
116197
message ColumnDataTypeExtension {
117198
oneof type_ext {
118199
DecimalTypeExtension decimal_type = 1;
200+
// Marks the binary column in proto is actually a JSON column.
201+
JsonTypeExtension json_type = 2;
202+
VectorTypeExtension vector_type = 3;
203+
ListTypeExtension list_type = 4;
204+
StructTypeExtension struct_type = 5;
119205
}
120206
}
121207

122208
message DecimalTypeExtension {
123209
int32 precision = 1;
124210
int32 scale = 2;
125211
}
212+
213+
enum JsonTypeExtension {
214+
JSON_BINARY = 0;
215+
}
216+
217+
message VectorTypeExtension {
218+
uint32 dim = 1;
219+
}
220+
221+
message ListTypeExtension {
222+
ColumnDataType datatype = 1;
223+
ColumnDataTypeExtension datatype_extension = 2;
224+
}
225+
226+
message StructTypeExtension {
227+
repeated StructField fields = 1;
228+
}
229+
230+
// Additional options for the column.
231+
message ColumnOptions {
232+
// Supported keys:
233+
// "fulltext":
234+
// A JSON encoded string containing full-text search options for the column.
235+
//
236+
// The fulltext options JSON structure:
237+
// {
238+
// "enable": bool, // Indicates whether full-text search is
239+
// // enabled for the column.
240+
//
241+
// "analyzer": string, // The language-specific text analyzer to
242+
// // use for indexing and searching text.
243+
// // Supported values: ["English" (Default), "Chinese"].
244+
//
245+
// "case-sensitive": bool // Indicates whether the text should be treated
246+
// // as case-sensitive during full-text search.
247+
// }
248+
//
249+
// Example:
250+
// "fulltext": "{\"enable\": true, \"analyzer\": \"English\", \"case-sensitive\": false}"
251+
map<string, string> options = 1;
252+
}
253+
254+
message ArrowIpc {
255+
bytes schema = 1;
256+
bytes data_header = 2;
257+
bytes payload = 3;
258+
}

‎protos/database.proto‎

Lines changed: 46 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,14 @@ syntax = "proto3";
1616

1717
package greptime.v1;
1818

19-
option java_package = "io.greptime.v1";
20-
option java_outer_classname = "Database";
21-
option go_package = "github.com/GreptimeTeam/greptime-proto/go/greptime/v1";
22-
23-
import "protos/ddl.proto";
2419
import "protos/column.proto";
2520
import "protos/common.proto";
21+
import "protos/ddl.proto";
22+
import "protos/row.proto";
23+
24+
option go_package = "github.com/GreptimeTeam/greptime-proto/go/greptime/v1";
25+
option java_outer_classname = "Database";
26+
option java_package = "io.greptime.v1";
2627

2728
service GreptimeDatabase {
2829
rpc Handle(GreptimeRequest) returns (GreptimeResponse);
@@ -36,23 +37,33 @@ message GreptimeRequest {
3637
InsertRequests inserts = 2;
3738
QueryRequest query = 3;
3839
DdlRequest ddl = 4;
39-
DeleteRequest delete = 5;
40+
DeleteRequests deletes = 5;
41+
RowInsertRequests row_inserts = 6;
42+
RowDeleteRequests row_deletes = 7;
4043
}
4144
}
4245

4346
message GreptimeResponse {
4447
ResponseHeader header = 1;
45-
oneof response { AffectedRows affected_rows = 2; }
48+
oneof response {
49+
AffectedRows affected_rows = 2;
50+
}
4651
}
4752

4853
message QueryRequest {
4954
oneof query {
5055
string sql = 1;
5156
bytes logical_plan = 2;
52-
// PromRangeQuery prom_range_query = 3;
5357
}
5458
}
5559

60+
// A temporary solution for executing insert into table SELECT .. with logical plan
61+
// since substrait to logical plan doesn't support dml yet.
62+
message InsertIntoPlan {
63+
TableName table_name = 1;
64+
bytes logical_plan = 2;
65+
}
66+
5667
message InsertRequests {
5768
repeated InsertRequest inserts = 1;
5869
}
@@ -67,22 +78,44 @@ message InsertRequest {
6778
//
6879
// Note: the row_count of all columns in a InsertRequest must be same.
6980
uint32 row_count = 4;
81+
}
7082

71-
// The region number of current insert request.
72-
uint32 region_number = 5;
83+
message DeleteRequests {
84+
repeated DeleteRequest deletes = 1;
7385
}
7486

7587
message DeleteRequest {
7688
// The table name to delete from. Catalog name and schema name are in the
7789
// `RequestHeader`.
7890
string table_name = 1;
7991

80-
// The region number of current delete request.
81-
uint32 region_number = 2;
82-
8392
// The data to delete, indexed by key columns.
8493
repeated Column key_columns = 3;
8594

8695
// The row count of all columns above.
8796
uint32 row_count = 4;
8897
}
98+
99+
message RowInsertRequests {
100+
repeated RowInsertRequest inserts = 1;
101+
}
102+
103+
message RowInsertRequest {
104+
string table_name = 1;
105+
106+
// Data is represented here.
107+
Rows rows = 2;
108+
}
109+
110+
message RowDeleteRequests {
111+
repeated RowDeleteRequest deletes = 1;
112+
}
113+
114+
message RowDeleteRequest {
115+
// The table name to delete from. Catalog name and schema name are in the
116+
// `RequestHeader`.
117+
string table_name = 1;
118+
119+
// The data to delete.
120+
Rows rows = 2;
121+
}

0 commit comments

Comments
 (0)