Skip to content

Commit 71754ff

Browse files
dl239dl239
authored
feat: support compress (#3572)
1 parent c2b7817 commit 71754ff

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+489
-466
lines changed

‎cases/plan/create.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1035,3 +1035,40 @@ cases:
10351035
+-kind: HIVE
10361036
+-path: hdfs://path
10371037
+-table_option_list: []
1038+
1039+
- id: 34
1040+
desc: Create 指定压缩
1041+
sql: |
1042+
create table t1(
1043+
column1 int,
1044+
column2 timestamp,
1045+
index(key=column1, ts=column2)) OPTIONS (compress_type="snappy");
1046+
expect:
1047+
node_tree_str: |
1048+
+-node[CREATE]
1049+
+-table: t1
1050+
+-IF NOT EXIST: 0
1051+
+-column_desc_list[list]:
1052+
| +-0:
1053+
| | +-node[kColumnDesc]
1054+
| | +-column_name: column1
1055+
| | +-column_type: int32
1056+
| | +-NOT NULL: 0
1057+
| +-1:
1058+
| | +-node[kColumnDesc]
1059+
| | +-column_name: column2
1060+
| | +-column_type: timestamp
1061+
| | +-NOT NULL: 0
1062+
| +-2:
1063+
| +-node[kColumnIndex]
1064+
| +-keys: [column1]
1065+
| +-ts_col: column2
1066+
| +-abs_ttl: -2
1067+
| +-lat_ttl: -2
1068+
| +-ttl_type: <nil>
1069+
| +-version_column: <nil>
1070+
| +-version_count: 0
1071+
+-table_option_list[list]:
1072+
+-0:
1073+
+-node[kCompressType]
1074+
+-compress_type: snappy

‎docs/en/reference/sql/ddl/CREATE_TABLE_STATEMENT.md‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,11 @@ StorageMode
473473
::= 'Memory'
474474
| 'HDD'
475475
| 'SSD'
476+
CompressTypeOption
477+
::= 'COMPRESS_TYPE' '=' CompressType
478+
CompressType
479+
::= 'NoCompress'
480+
| 'Snappy
476481
```
477482
478483
@@ -484,6 +489,7 @@ StorageMode
484489
| `REPLICANUM` | It defines the number of replicas for the table. Note that the number of replicas is only configurable in Cluster version. | `OPTIONS (REPLICANUM=3)` |
485490
| `DISTRIBUTION` | It defines the distributed node endpoint configuration. Generally, it contains a Leader node and several followers. `(leader, [follower1, follower2, ..])`. Without explicit configuration, OpenMLDB will automatically configure `DISTRIBUTION` according to the environment and nodes. | `DISTRIBUTION = [ ('127.0.0.1:6527', [ '127.0.0.1:6528','127.0.0.1:6529' ])]` |
486491
| `STORAGE_MODE` | It defines the storage mode of the table. The supported modes are `Memory`, `HDD` and `SSD`. When not explicitly configured, it defaults to `Memory`. <br/>If you need to support a storage mode other than `Memory` mode, `tablet` requires additional configuration options. For details, please refer to [tablet configuration file **conf/tablet.flags**](../../../deploy/conf.md#the-configuration-file-for-apiserver:-conf/tablet.flags). | `OPTIONS (STORAGE_MODE='HDD')` |
492+
| `COMPRESS_TYPE` | It defines the compress types of the table. The supported compress type are `NoCompress` and `Snappy`. The default value is `NoCompress` | `OPTIONS (COMPRESS_TYPE='Snappy')`
487493
488494
489495
#### The Difference between Disk Table and Memory Table
@@ -515,11 +521,11 @@ DESC t1;
515521
--- -------------------- ------ ---------- ------ ---------------
516522
1 INDEX_0_1651143735 col1 std_time 0min kAbsoluteTime
517523
--- -------------------- ------ ---------- ------ ---------------
518-
--------------
519-
storage_mode
520-
--------------
521-
HDD
522-
--------------
524+
--------------- --------------
525+
compress_type storage_mode
526+
--------------- --------------
527+
NoCompress HDD
528+
--------------- --------------
523529
```
524530
The following sql command create a table with specified distribution.
525531
```sql

‎docs/en/reference/sql/ddl/DESC_STATEMENT.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,11 @@ desc t1;
5656
--- -------------------- ------ ---------- ---------- ---------------
5757
1 INDEX_0_1658136511 col1 std_time 43200min kAbsoluteTime
5858
--- -------------------- ------ ---------- ---------- ---------------
59-
--------------
60-
storage_mode
61-
--------------
62-
Memory
63-
--------------
59+
--------------- --------------
60+
compress_type storage_mode
61+
--------------- --------------
62+
NoCompress Memory
63+
--------------- --------------
6464

6565
```
6666

‎docs/en/reference/sql/ddl/SHOW_CREATE_TABLE_STATEMENT.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ show create table t1;
2121
`c3` bigInt,
2222
`c4` timestamp,
2323
INDEX (KEY=`c1`, TS=`c4`, TTL_TYPE=ABSOLUTE, TTL=0m)
24-
) OPTIONS (PARTITIONNUM=8, REPLICANUM=2, STORAGE_MODE='HDD');
24+
) OPTIONS (PARTITIONNUM=8, REPLICANUM=2, STORAGE_MODE='HDD', COMPRESS_TYPE='NoCompress');
2525
------- ---------------------------------------------------------------
2626

2727
1 rows in set

‎docs/zh/openmldb_sql/ddl/CREATE_TABLE_STATEMENT.md‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,11 @@ StorageMode
450450
::= 'Memory'
451451
| 'HDD'
452452
| 'SSD'
453+
CompressTypeOption
454+
::= 'COMPRESS_TYPE' '=' CompressType
455+
CompressType
456+
::= 'NoCompress'
457+
| 'Snappy'
453458
```
454459

455460

@@ -460,6 +465,7 @@ StorageMode
460465
| `REPLICANUM` | 配置表的副本数。请注意，副本数只有在集群版中才可以配置。 | `OPTIONS (REPLICANUM=3)` |
461466
| `DISTRIBUTION` | 配置分布式的节点endpoint。一般包含一个Leader节点和若干Follower节点。`(leader, [follower1, follower2, ..])`。不显式配置时，OpenMLDB会自动根据环境和节点来配置`DISTRIBUTION`。 | `DISTRIBUTION = [ ('127.0.0.1:6527', [ '127.0.0.1:6528','127.0.0.1:6529' ])]` |
462467
| `STORAGE_MODE` | 表的存储模式，支持的模式有`Memory`、`HDD`或`SSD`。不显式配置时，默认为`Memory`。<br/>如果需要支持非`Memory`模式的存储模式，`tablet`需要额外的配置选项，具体可参考[tablet配置文件 conf/tablet.flags](../../../deploy/conf.md)。 | `OPTIONS (STORAGE_MODE='HDD')` |
468+
| `COMPRESS_TYPE` | 指定表的压缩类型。目前只支持Snappy压缩, 。默认为 `NoCompress` 即不压缩。 | `OPTIONS (COMPRESS_TYPE='Snappy')`
463469

464470
#### 磁盘表与内存表区别
465471
- 磁盘表对应`STORAGE_MODE`的取值为`HDD`或`SSD`。内存表对应的`STORAGE_MODE`取值为`Memory`。
@@ -488,11 +494,11 @@ DESC t1;
488494
--- -------------------- ------ ---------- ------ ---------------
489495
1 INDEX_0_1651143735 col1 std_time 0min kAbsoluteTime
490496
--- -------------------- ------ ---------- ------ ---------------
491-
--------------
492-
storage_mode
493-
--------------
494-
HDD
495-
--------------
497+
--------------- --------------
498+
compress_type storage_mode
499+
--------------- --------------
500+
NoCompress HDD
501+
--------------- --------------
496502
```
497503
创建一张表，指定分片的分布状态
498504
```sql

‎docs/zh/openmldb_sql/ddl/DESC_STATEMENT.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,11 @@ desc t1;
5656
--- -------------------- ------ ---------- ---------- ---------------
5757
1 INDEX_0_1658136511 col1 std_time 43200min kAbsoluteTime
5858
--- -------------------- ------ ---------- ---------- ---------------
59-
--------------
60-
storage_mode
61-
--------------
62-
Memory
63-
--------------
59+
--------------- --------------
60+
compress_type storage_mode
61+
--------------- --------------
62+
NoCompress Memory
63+
--------------- --------------
6464

6565
```
6666

‎docs/zh/openmldb_sql/ddl/SHOW_CREATE_TABLE_STATEMENT.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ show create table t1;
2121
`c3` bigInt,
2222
`c4` timestamp,
2323
INDEX (KEY=`c1`, TS=`c4`, TTL_TYPE=ABSOLUTE, TTL=0m)
24-
) OPTIONS (PARTITIONNUM=8, REPLICANUM=2, STORAGE_MODE='HDD');
24+
) OPTIONS (PARTITIONNUM=8, REPLICANUM=2, STORAGE_MODE='HDD', COMPRESS_TYPE='NoCompress');
2525
------- ---------------------------------------------------------------
2626

2727
1 rows in set

‎hybridse/include/node/node_enum.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ enum SqlNodeType {
9797
kWithClauseEntry,
9898
kAlterTableStmt,
9999
kShowStmt,
100+
kCompressType,
100101
kSqlNodeTypeLast, // debug type
101102
};
102103

@@ -342,6 +343,11 @@ enum StorageMode {
342343
kHDD = 3,
343344
};
344345

346+
enum CompressType {
347+
kNoCompress = 0,
348+
kSnappy = 1,
349+
};
350+
345351
// batch plan node type
346352
enum BatchPlanNodeType { kBatchDataset, kBatchPartition, kBatchMap };
347353

‎hybridse/include/node/node_manager.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -399,8 +399,6 @@ class NodeManager {
399399

400400
SqlNode *MakeReplicaNumNode(int num);
401401

402-
SqlNode *MakeStorageModeNode(StorageMode storage_mode);
403-
404402
SqlNode *MakePartitionNumNode(int num);
405403

406404
SqlNode *MakeDistributionsNode(const NodePointVector& distribution_list);

‎hybridse/include/node/sql_node.h‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include <vector>
2626

2727
#include "absl/status/statusor.h"
28+
#include "absl/strings/match.h"
2829
#include "absl/strings/str_cat.h"
2930
#include "absl/strings/string_view.h"
3031
#include "boost/algorithm/string.hpp"
@@ -309,17 +310,26 @@ inline const std::string StorageModeName(StorageMode mode) {
309310
}
310311

311312
inline const StorageMode NameToStorageMode(const std::string& name) {
312-
if (boost::iequals(name, "memory")) {
313+
if (absl::EqualsIgnoreCase(name, "memory")) {
313314
return kMemory;
314-
} else if (boost::iequals(name, "hdd")) {
315+
} else if (absl::EqualsIgnoreCase(name, "hdd")) {
315316
return kHDD;
316-
} else if (boost::iequals(name, "ssd")) {
317+
} else if (absl::EqualsIgnoreCase(name, "ssd")) {
317318
return kSSD;
318319
} else {
319320
return kUnknown;
320321
}
321322
}
322323

324+
inline absl::StatusOr<CompressType> NameToCompressType(const std::string& name) {
325+
if (absl::EqualsIgnoreCase(name, "snappy")) {
326+
return CompressType::kSnappy;
327+
} else if (absl::EqualsIgnoreCase(name, "nocompress")) {
328+
return CompressType::kNoCompress;
329+
}
330+
return absl::Status(absl::StatusCode::kInvalidArgument, absl::StrCat("invalid compress type: ", name));
331+
}
332+
323333
inline const std::string RoleTypeName(RoleType type) {
324334
switch (type) {
325335
case kLeader:
@@ -1884,6 +1894,23 @@ class StorageModeNode : public SqlNode {
18841894
StorageMode storage_mode_;
18851895
};
18861896

1897+
class CompressTypeNode : public SqlNode {
1898+
public:
1899+
CompressTypeNode() : SqlNode(kCompressType, 0, 0), compress_type_(kNoCompress) {}
1900+
1901+
explicit CompressTypeNode(CompressType compress_type)
1902+
: SqlNode(kCompressType, 0, 0), compress_type_(compress_type) {}
1903+
1904+
~CompressTypeNode() {}
1905+
1906+
CompressType GetCompressType() const { return compress_type_; }
1907+
1908+
void Print(std::ostream &output, const std::string &org_tab) const;
1909+
1910+
private:
1911+
CompressType compress_type_;
1912+
};
1913+
18871914
class CreateTableLikeClause {
18881915
public:
18891916
CreateTableLikeClause() = default;

0 commit comments

Comments
 (0)