Skip to content

Commit 88ec576

Browse files
Enable fragment result cache and data cache (#12)
1 parent 72285c6 commit 88ec576

File tree

143 files changed

+2589
-26
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

143 files changed

+2589
-26
lines changed
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
connector.name=hive-hadoop2
2+
3+
hive.metastore.uri=thrift://a74b6e20f132e4eac840a591d5caf0ba-c3bc190f0aee9cb5.elb.us-east-1.amazonaws.com:9083
4+
hive.s3.ssl.enabled=true
5+
hive.s3.path-style-access=true
6+
hive.s3.iam-role=arn:aws:iam::093347738777:role/deploy-cluster-instance-s3-access-role
7+
8+
# metastore
9+
hive.metastore-cache-ttl=240m
10+
hive.metastore-refresh-interval=30m
11+
hive.metastore-timeout=3m
12+
hive.metastore-cache-maximum-size=1000000
13+
# writer properties
14+
hive.parquet.writer.version=PARQUET_1_0
15+
hive.max-partitions-per-writers=5000
16+
hive.collect-column-statistics-on-write=true
17+
# for parquet write 64bit timestamp rather than 96 bits https://github.com/prestodb/presto/issues/22605#issuecomment-2089544473
18+
hive.parquet.optimized-writer.enabled=true
19+
# split optimization
20+
hive.max-outstanding-splits=1800
21+
hive.max-initial-splits=600
22+
hive.max-initial-split-size=128MB
23+
hive.max-split-size=256MB
24+
hive.split-loader-concurrency=32
25+
# dml permission
26+
hive.non-managed-table-writes-enabled=true
27+
hive.non-managed-table-creates-enabled=true
28+
# pushdown
29+
hive.pushdown-filter-enabled=true
30+
hive.s3select-pushdown.enabled=true
31+
# Data content specific
32+
hive.recursive-directories=true
33+
# affinity scheduling
34+
hive.node-selection-strategy=SOFT_AFFINITY
35+
# metadata caching
36+
hive.parquet.use-column-names=true
37+
hive.parquet-batch-read-optimization-enabled=true
38+
hive.parquet.metadata-cache-enabled=true
39+
hive.parquet.metadata-cache-size=300MB
40+
hive.parquet.metadata-cache-ttl-since-last-access=12h
41+
# hive legacy config
42+
hive.allow-drop-table=true
43+
hive.allow-rename-table=true
44+
hive.allow-add-column=true
45+
hive.allow-drop-column=true
46+
hive.allow-rename-column=true
47+
48+
# alluxio data cache
49+
cache.enabled=true
50+
cache.type=ALLUXIO
51+
cache.alluxio.max-cache-size=1397GB
52+
cache.alluxio.config-validation-enabled=true
53+
cache.base-directory=file:///data/presto-cache/alluxiodatacache/hive
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
connector.name=iceberg
2+
3+
hive.metastore.uri=thrift://a74b6e20f132e4eac840a591d5caf0ba-c3bc190f0aee9cb5.elb.us-east-1.amazonaws.com:9083
4+
hive.s3.ssl.enabled=true
5+
hive.s3.path-style-access=true
6+
hive.s3.iam-role=arn:aws:iam::093347738777:role/deploy-cluster-instance-s3-access-role
7+
8+
# metastore
9+
hive.metastore-cache-ttl=240m
10+
hive.metastore-refresh-interval=30m
11+
hive.metastore-timeout=3m
12+
hive.metastore-cache-maximum-size=1000000
13+
# writer properties
14+
hive.parquet.writer.version=PARQUET_1_0
15+
iceberg.max-partitions-per-writer=5000
16+
# for parquet write 64bit timestamp rather than 96 bits https://github.com/prestodb/presto/issues/22605#issuecomment-2089544473
17+
hive.parquet.optimized-writer.enabled=true
18+
# dml permission
19+
hive.non-managed-table-writes-enabled=true
20+
hive.non-managed-table-creates-enabled=true
21+
# pushdown
22+
hive.pushdown-filter-enabled=true
23+
hive.s3select-pushdown.enabled=true
24+
# Data content specific
25+
hive.recursive-directories=true
26+
# affinity scheduling
27+
hive.node-selection-strategy=SOFT_AFFINITY
28+
# metadata caching
29+
hive.parquet.use-column-names=true
30+
hive.parquet-batch-read-optimization-enabled=true
31+
iceberg.parquet.metadata-cache-enabled=true
32+
iceberg.parquet.metadata-cache-size=300MB
33+
iceberg.parquet.metadata-cache-ttl-since-last-access=12h
34+
35+
# alluxio data cache
36+
cache.enabled=true
37+
cache.type=ALLUXIO
38+
cache.alluxio.max-cache-size=1397GB
39+
cache.alluxio.config-validation-enabled=true
40+
cache.base-directory=file:///data/presto-cache/alluxiodatacache/iceberg
File renamed without changes.
File renamed without changes.

0 commit comments

Comments
 (0)