Skip to content

Commit aaaecde

Browse files
gongxun0928gongxun0928
committed
feature: asynchronously update file statistics during vacuum
Previously, statistics (min-max, sum, count, etc.) were computed synchronously during data insertion, causing significant slowdowns due to heavy computational overhead. This change introduces an asynchronous approach to maintain statistics: - Add a GUC parameter to control statistics collection during writes (disabled by default) - Skip statistics computation during INSERT to ensure fast writes - Update statistics asynchronously during VACUUM on PAX tables by scanning file metadata - Re-read files and refresh statistics only when metadata indicates they are stale ``` create table t1(c1 int, c2 int, c3 int, c4 int, c5 int, c6 int) using pax with(minmax_columns='c1,c2,c3,c4,c5,c6'); set pax.enable_sync_collect_stats to on; -- collect stats synchronously insert into t1 select i,i,i,i,i,i from generate_series(1,1000000) i; INSERT 0 1000000 Time: 2733.731 ms (00:02.734) create table t2(c1 int, c2 int, c3 int, c4 int, c5 int, c6 int) using pax; insert into t2 select i,i,i,i,i,i from generate_series(1,1000000) i; INSERT 0 1000000 Time: 1816.836 ms (00:01.817) ```
1 parent 468b1e6 commit aaaecde

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+635
-104
lines changed

‎contrib/pax_storage/expected/cluster.out‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
set pax.max_tuples_per_file to 131072;
2+
set pax.enable_sync_collect_stats = on;
23
-- cluster table using index
34
-- start_ignore
45
drop table if EXISTS t_index_cluster;
@@ -295,3 +296,4 @@ select ptblockname,ptstatistics,ptisclustered from get_pax_aux_table('t_lexical_
295296
(6 rows)
296297

297298
drop table t_lexical_cluster;
299+
reset pax.enable_sync_collect_stats;

‎contrib/pax_storage/expected/disable_autovacuum.out‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
alter system set autovacuum = off;
2+
select gp_segment_id, pg_reload_conf() from gp_id union select gp_segment_id, pg_reload_conf() from gp_dist_random('gp_id');
3+
gp_segment_id | pg_reload_conf
4+
---------------+----------------
5+
2 | t
6+
1 | t
7+
0 | t
8+
-1 | t
9+
(4 rows)
10+

‎contrib/pax_storage/expected/enable_autovacuum.out‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
alter system set autovacuum = on;
2+
select gp_segment_id, pg_reload_conf() from gp_id union select gp_segment_id, pg_reload_conf() from gp_dist_random('gp_id');
3+
gp_segment_id | pg_reload_conf
4+
---------------+----------------
5+
2 | t
6+
1 | t
7+
0 | t
8+
-1 | t
9+
(4 rows)
10+

‎contrib/pax_storage/expected/filter.out‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
set pax.enable_debug to on;
22
set pax.enable_sparse_filter = on;
3+
set pax.enable_sync_collect_stats = on;
34
create table pax_test.null_test_t(a int, b int, c text) using pax;
45
insert into pax_test.null_test_t(a) select null from generate_series(1,2)i;
56
insert into pax_test.null_test_t select 1, i, 'cc_' || i from generate_series(1,2)i;
@@ -223,3 +224,4 @@ kind group, filter rate: 0 / 1
223224
reset client_min_messages;
224225
drop table pax_test.in_test_t;
225226
reset pax.enable_sparse_filter;
227+
reset pax.enable_sync_collect_stats;

‎contrib/pax_storage/expected/filter_tree.out‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
set default_table_access_method to pax;
88
set pax.enable_debug to on;
99
set pax.enable_sparse_filter to on;
10+
set pax.enable_sync_collect_stats = on;
1011
create or replace function intrc(iint int)
1112
returns int as $$
1213
begin return iint; end;
@@ -735,6 +736,7 @@ LOG: statement: select count(*) from t1 where coalesce(v1, 2) != 1;
735736

736737
reset client_min_messages;
737738
LOG: statement: reset client_min_messages;
739+
reset pax.enable_sync_collect_stats;
738740
drop table t1;
739741
drop table t2;
740742
drop table t_allnull;

‎contrib/pax_storage/expected/filter_tree_optimizer.out‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
set default_table_access_method to pax;
88
set pax.enable_debug to on;
99
set pax.enable_sparse_filter to on;
10+
set pax.enable_sync_collect_stats = on;
1011
create or replace function intrc(iint int)
1112
returns int as $$
1213
begin return iint; end;
@@ -737,6 +738,7 @@ LOG: statement: select count(*) from t1 where coalesce(v1, 2) != 1;
737738
(1 row)
738739

739740
reset client_min_messages;
741+
reset pax.enable_sync_collect_stats;
740742
LOG: statement: reset client_min_messages;
741743
drop table t1;
742744
drop table t2;

‎contrib/pax_storage/expected/filter_tree_root_quals.out‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
set default_table_access_method to pax;
88
set pax.enable_debug to on;
99
set pax.enable_sparse_filter to on;
10+
set pax.enable_sync_collect_stats = on;
1011
create table t1(same int, v1 int, v2 int, v3 int, v4 int) using pax with (minmax_columns='v1,v2,v3,v4');
1112
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'same' as the Apache Cloudberry data distribution key for this table.
1213
HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
@@ -55,5 +56,6 @@ LOG: kind file, filter rate: 1 / 2
5556
reset client_min_messages;
5657
LOG: statement: reset client_min_messages;
5758
reset optimizer;
59+
reset pax.enable_sync_collect_stats;
5860
drop table t1;
5961
drop table t2;

‎contrib/pax_storage/expected/filter_tree_root_quals_1.out‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
set default_table_access_method to pax;
88
set pax.enable_debug to on;
99
set pax.enable_sparse_filter to on;
10+
set pax.enable_sync_collect_stats = on;
1011
create table t1(same int, v1 int, v2 int, v3 int, v4 int) using pax with (minmax_columns='v1,v2,v3,v4');
1112
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'same' as the Apache Cloudberry data distribution key for this table.
1213
HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.

‎contrib/pax_storage/expected/pax_vacuum.out‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
show autovacuum;
2+
autovacuum
3+
------------
4+
off
5+
(1 row)
6+
7+
-- async collect stats in vacuum
8+
set pax.enable_sync_collect_stats to off;
9+
create table pax_vacuum (a int, b int) using pax with(minmax_columns='a,b') distributed by (a);
10+
insert into pax_vacuum select i, i from generate_series(1, 1000) i;
11+
select * from get_pax_aux_table('pax_vacuum');
12+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
13+
-------------+------------+-----------------------------------------------------------------+----------------+-----------------+---------------
14+
0 | 322 | [(false,false),(322),None,None],[(false,false),(322),None,None] | f | f | f
15+
0 | 340 | [(false,false),(340),None,None],[(false,false),(340),None,None] | f | f | f
16+
0 | 338 | [(false,false),(338),None,None],[(false,false),(338),None,None] | f | f | f
17+
(3 rows)
18+
19+
vacuum pax_vacuum;
20+
select * from get_pax_aux_table('pax_vacuum');
21+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
22+
-------------+------------+---------------------------------------------------------------------------------+----------------+-----------------+---------------
23+
0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)] | f | f | f
24+
0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)] | f | f | f
25+
0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)] | f | f | f
26+
(3 rows)
27+
28+
truncate pax_vacuum;
29+
select * from get_pax_aux_table('pax_vacuum');
30+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
31+
-------------+------------+--------------+----------------+-----------------+---------------
32+
(0 rows)
33+
34+
-- sync collect stats in vacuum
35+
set pax.enable_sync_collect_stats to on;
36+
insert into pax_vacuum select i, i from generate_series(1, 1000) i;
37+
select * from get_pax_aux_table('pax_vacuum');
38+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
39+
-------------+------------+---------------------------------------------------------------------------------+----------------+-----------------+---------------
40+
0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)] | f | f | f
41+
0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)] | f | f | f
42+
0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)] | f | f | f
43+
(3 rows)
44+
45+
vacuum pax_vacuum;
46+
select * from get_pax_aux_table('pax_vacuum');
47+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
48+
-------------+------------+---------------------------------------------------------------------------------+----------------+-----------------+---------------
49+
0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)] | f | f | f
50+
0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)] | f | f | f
51+
0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)] | f | f | f
52+
(3 rows)
53+
54+
truncate pax_vacuum;
55+
select * from get_pax_aux_table('pax_vacuum');
56+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
57+
-------------+------------+--------------+----------------+-----------------+---------------
58+
(0 rows)
59+
60+
-- update and vacuum
61+
set pax.enable_sync_collect_stats to off;
62+
insert into pax_vacuum select i, i from generate_series(1, 1000) i;
63+
select * from get_pax_aux_table('pax_vacuum');
64+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
65+
-------------+------------+-----------------------------------------------------------------+----------------+-----------------+---------------
66+
0 | 322 | [(false,false),(322),None,None],[(false,false),(322),None,None] | f | f | f
67+
0 | 338 | [(false,false),(338),None,None],[(false,false),(338),None,None] | f | f | f
68+
0 | 340 | [(false,false),(340),None,None],[(false,false),(340),None,None] | f | f | f
69+
(3 rows)
70+
71+
vacuum pax_vacuum;
72+
select * from get_pax_aux_table('pax_vacuum');
73+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
74+
-------------+------------+---------------------------------------------------------------------------------+----------------+-----------------+---------------
75+
0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)] | f | f | f
76+
0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)] | f | f | f
77+
0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)] | f | f | f
78+
(3 rows)
79+
80+
update pax_vacuum set b = b + 1;
81+
select * from get_pax_aux_table('pax_vacuum');
82+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
83+
-------------+------------+-----------------------------------------------------------------+----------------+-----------------+---------------
84+
0 | 338 | [(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f
85+
1 | 338 | [(false,false),(338),None,None],[(false,false),(338),None,None] | f | f | f
86+
0 | 322 | [(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f
87+
1 | 322 | [(false,false),(322),None,None],[(false,false),(322),None,None] | f | f | f
88+
0 | 340 | [(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f
89+
1 | 340 | [(false,false),(340),None,None],[(false,false),(340),None,None] | f | f | f
90+
(6 rows)
91+
92+
vacuum pax_vacuum;
93+
select * from get_pax_aux_table('pax_vacuum');
94+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
95+
-------------+------------+---------------------------------------------------------------------------------+----------------+-----------------+---------------
96+
0 | 338 | [(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f
97+
1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(3,1001),(167000)] | f | f | f
98+
0 | 322 | [(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f
99+
1 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(2,999),(161602)] | f | f | f
100+
0 | 340 | [(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f
101+
1 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(6,998),(172898)] | f | f | f
102+
(6 rows)
103+
104+
truncate pax_vacuum;
105+
select * from get_pax_aux_table('pax_vacuum');
106+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
107+
-------------+------------+--------------+----------------+-----------------+---------------
108+
(0 rows)
109+
110+
-- sync collect stats with vacuum
111+
set pax.enable_sync_collect_stats to on;
112+
insert into pax_vacuum select i, i from generate_series(1, 1000) i;
113+
select * from get_pax_aux_table('pax_vacuum');
114+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
115+
-------------+------------+---------------------------------------------------------------------------------+----------------+-----------------+---------------
116+
0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)] | f | f | f
117+
0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)] | f | f | f
118+
0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)] | f | f | f
119+
(3 rows)
120+
121+
vacuum pax_vacuum;
122+
select * from get_pax_aux_table('pax_vacuum');
123+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
124+
-------------+------------+---------------------------------------------------------------------------------+----------------+-----------------+---------------
125+
0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)] | f | f | f
126+
0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)] | f | f | f
127+
0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)] | f | f | f
128+
(3 rows)
129+
130+
update pax_vacuum set b = b + 1;
131+
select * from get_pax_aux_table('pax_vacuum');
132+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
133+
-------------+------------+---------------------------------------------------------------------------------+----------------+-----------------+---------------
134+
0 | 338 | [(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f
135+
1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(3,1001),(167000)] | f | f | f
136+
0 | 322 | [(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f
137+
1 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(2,999),(161602)] | f | f | f
138+
0 | 340 | [(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f
139+
1 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(6,998),(172898)] | f | f | f
140+
(6 rows)
141+
142+
vacuum pax_vacuum;
143+
select * from get_pax_aux_table('pax_vacuum');
144+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
145+
-------------+------------+---------------------------------------------------------------------------------+----------------+-----------------+---------------
146+
0 | 338 | [(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f
147+
1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(3,1001),(167000)] | f | f | f
148+
0 | 322 | [(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f
149+
1 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(2,999),(161602)] | f | f | f
150+
0 | 340 | [(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f
151+
1 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(6,998),(172898)] | f | f | f
152+
(6 rows)
153+
154+
truncate pax_vacuum;
155+
select * from get_pax_aux_table('pax_vacuum');
156+
ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered
157+
-------------+------------+--------------+----------------+-----------------+---------------
158+
(0 rows)
159+
160+
drop table pax_vacuum;

‎contrib/pax_storage/expected/statistics/min_max_bit_byte_types.out‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ set default_table_access_method to pax;
77
set pax.enable_debug to on;
88
set pax.enable_sparse_filter to on;
99
set pax.max_tuples_per_group to 5;
10+
set pax.enable_sync_collect_stats = on;
1011
--
1112
-- Test the bool min/max types support
1213
--
@@ -612,3 +613,4 @@ drop table t_varbit;
612613
reset pax.enable_debug;
613614
reset pax.enable_sparse_filter;
614615
reset pax.max_tuples_per_group;
616+
reset pax.enable_sync_collect_stats;

0 commit comments

Comments
 (0)