Skip to content

Commit 8b4a558

Browse files
youngsofunyoungsofun
authored
fix: fix memory_size of sliced string view. (#19014)
1 parent 43fda0d commit 8b4a558

26 files changed

+72
-61
lines changed

‎src/common/column/src/binview/mod.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,8 @@ impl<T: ViewType + ?Sized> BinaryViewColumnGeneric<T> {
297297
}
298298

299299
pub fn memory_size(&self) -> usize {
300-
self.total_buffer_len + self.len() * 16
300+
// when read back from parquet, the buffer is a bytes array without views
301+
self.total_bytes_len + self.len() * 20
301302
}
302303

303304
fn total_unshared_buffer_len(&self) -> usize {

‎src/common/column/tests/it/binview/mod.rs‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -151,15 +151,30 @@ fn test_slice() {
151151
];
152152

153153
let array: Utf8ViewColumn = data.into_iter().collect();
154-
assert_eq!(array.memory_size(), 156);
154+
assert_eq!(array.total_bytes_len(), 78);
155+
assert_eq!(array.total_buffer_len(), 60);
156+
assert_eq!(array.memory_size(), 198);
155157

156-
let a3 = array.sliced(2, 3);
157-
assert_eq!(a3.into_iter().collect::<Vec<_>>(), vec![
158+
let a0 = array.clone().sliced(0, 2);
159+
assert_eq!(a0.into_iter().collect::<Vec<_>>(), vec!["hello", "world",]);
160+
assert_eq!(a0.memory_size(), 50);
161+
assert_eq!(a0.total_bytes_len(), 10);
162+
163+
let a1 = array.clone().sliced(2, 3);
164+
assert_eq!(a1.into_iter().collect::<Vec<_>>(), vec![
158165
"databend",
159166
"yyyyyyyyyyyyyyyyyyyyy",
160167
"zzzzzzzzzzzzzzzzzzzzz",
161168
]);
162-
assert_eq!(a3.memory_size(), 108);
169+
assert_eq!(a1.memory_size(), 110);
170+
assert_eq!(a1.total_bytes_len(), 50);
171+
172+
let a2 = array.sliced(5, 1);
173+
assert_eq!(a2.into_iter().collect::<Vec<_>>(), vec![
174+
"abcabcabcabcabcabc",
175+
]);
176+
assert_eq!(a2.memory_size(), 38);
177+
assert_eq!(a2.total_bytes_len(), 18);
163178
}
164179

165180
#[test]

‎src/query/expression/src/values.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -892,7 +892,7 @@ impl ScalarRef<'_> {
892892
ScalarRef::Decimal(_) => n * self.memory_size(),
893893
ScalarRef::Boolean(_) => n.div_ceil(8),
894894
ScalarRef::Binary(s) => s.len() * n + (n + 1) * 8,
895-
ScalarRef::String(s) => n * 16 + if s.len() > 12 && n > 0 { s.len() } else { 0 },
895+
ScalarRef::String(s) => n * (20 + s.len()),
896896
ScalarRef::Timestamp(_) => n * 8,
897897
ScalarRef::TimestampTz(_) => n * 16,
898898
ScalarRef::Date(_) => n * 4,

‎src/query/expression/tests/it/block.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ fn test_block_entry_memory_size() {
9090
assert_eq!(3, entry.memory_size());
9191

9292
let col = StringType::from_data((0..10).map(|x| x.to_string()).collect::<Vec<_>>());
93-
assert_eq!(col.memory_size(), 10 * 16);
93+
assert_eq!(col.memory_size(), 210);
9494

9595
let array = ArrayColumn::<Int64Type>::new(
9696
Buffer::from_iter(0..10i64),

‎tests/nox/python_client/test_local.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ def test_stream_load():
6161
]
6262
progress = conn.stream_load("INSERT INTO test VALUES", values)
6363
assert progress.write_rows == 3, f"progress.write_rows: {progress.write_rows}"
64-
assert progress.write_bytes == 211, f"progress.write_bytes: {progress.write_bytes}"
64+
# assert progress.write_bytes >= 242, f"progress.write_bytes: {progress.write_bytes}"
6565

6666
rows = conn.query_iter("SELECT * FROM test")
6767
ret = [row.values() for row in rows]
@@ -104,9 +104,9 @@ def run_load_file(load_method):
104104
assert progress.write_rows == 3, (
105105
f"{load_method} progress.write_rows: {progress.write_rows}"
106106
)
107-
assert progress.write_bytes == 211, (
108-
f"{load_method}: progress.write_bytes: {progress.write_bytes}"
109-
)
107+
# assert progress.write_bytes == 211, (
108+
# f"{load_method}: progress.write_bytes: {progress.write_bytes}"
109+
# )
110110

111111
rows = conn.query_iter("SELECT * FROM test")
112112
ret = [row.values() for row in rows]

‎tests/sqllogictests/suites/base/09_fuse_engine/09_0020_analyze.test‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ query T
161161
select * from fuse_statistic('db_09_0020', 't_string') order by column_name asc;
162162
----
163163
id 10 0 4 [bucket id: 0, min: "1", max: "1", ndv: 1.0, count: 1.0], [bucket id: 1, min: "2", max: "2", ndv: 1.0, count: 1.0], [bucket id: 2, min: "3", max: "3", ndv: 1.0, count: 1.0], [bucket id: 3, min: "4", max: "4", ndv: 1.0, count: 1.0], [bucket id: 4, min: "5", max: "5", ndv: 1.0, count: 1.0], [bucket id: 5, min: "6", max: "6", ndv: 1.0, count: 1.0], [bucket id: 6, min: "7", max: "7", ndv: 1.0, count: 1.0], [bucket id: 7, min: "8", max: "8", ndv: 1.0, count: 1.0], [bucket id: 8, min: "9", max: "9", ndv: 1.0, count: 1.0], [bucket id: 9, min: "10", max: "10", ndv: 1.0, count: 1.0]
164-
str_val 10 0 16 [bucket id: 0, min: "1.0", max: "1.0", ndv: 1.0, count: 1.0], [bucket id: 1, min: "10.0", max: "10.0", ndv: 1.0, count: 1.0], [bucket id: 2, min: "2.0", max: "2.0", ndv: 1.0, count: 1.0], [bucket id: 3, min: "3.0", max: "3.0", ndv: 1.0, count: 1.0], [bucket id: 4, min: "4.0", max: "4.0", ndv: 1.0, count: 1.0], [bucket id: 5, min: "5.0", max: "5.0", ndv: 1.0, count: 1.0], [bucket id: 6, min: "6.0", max: "6.0", ndv: 1.0, count: 1.0], [bucket id: 7, min: "7.0", max: "7.0", ndv: 1.0, count: 1.0], [bucket id: 8, min: "8.0", max: "8.0", ndv: 1.0, count: 1.0], [bucket id: 9, min: "9.0", max: "9.0", ndv: 1.0, count: 1.0]
164+
str_val 10 0 23 [bucket id: 0, min: "1.0", max: "1.0", ndv: 1.0, count: 1.0], [bucket id: 1, min: "10.0", max: "10.0", ndv: 1.0, count: 1.0], [bucket id: 2, min: "2.0", max: "2.0", ndv: 1.0, count: 1.0], [bucket id: 3, min: "3.0", max: "3.0", ndv: 1.0, count: 1.0], [bucket id: 4, min: "4.0", max: "4.0", ndv: 1.0, count: 1.0], [bucket id: 5, min: "5.0", max: "5.0", ndv: 1.0, count: 1.0], [bucket id: 6, min: "6.0", max: "6.0", ndv: 1.0, count: 1.0], [bucket id: 7, min: "7.0", max: "7.0", ndv: 1.0, count: 1.0], [bucket id: 8, min: "8.0", max: "8.0", ndv: 1.0, count: 1.0], [bucket id: 9, min: "9.0", max: "9.0", ndv: 1.0, count: 1.0]
165165

166166
# Test string comparison with histogram
167167
query I

‎tests/sqllogictests/suites/base/09_fuse_engine/09_0044_issue_17314.test‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ analyze table t1;
3737
query IIT
3838
select * from fuse_statistic('issue_17314', 't1') order by column_name;
3939
----
40-
a 3 0 17 [bucket id: 0, min: "1", max: "1", ndv: 1.0, count: 1.0], [bucket id: 1, min: "2", max: "2", ndv: 1.0, count: 1.0], [bucket id: 2, min: "3", max: "3", ndv: 1.0, count: 1.0]
41-
b 3 0 17 [bucket id: 0, min: "11", max: "11", ndv: 1.0, count: 1.0], [bucket id: 1, min: "22", max: "22", ndv: 1.0, count: 1.0], [bucket id: 2, min: "33", max: "33", ndv: 1.0, count: 1.0]
40+
a 3 0 22 [bucket id: 0, min: "1", max: "1", ndv: 1.0, count: 1.0], [bucket id: 1, min: "2", max: "2", ndv: 1.0, count: 1.0], [bucket id: 2, min: "3", max: "3", ndv: 1.0, count: 1.0]
41+
b 3 0 23 [bucket id: 0, min: "11", max: "11", ndv: 1.0, count: 1.0], [bucket id: 1, min: "22", max: "22", ndv: 1.0, count: 1.0], [bucket id: 2, min: "33", max: "33", ndv: 1.0, count: 1.0]
4242

4343
statement ok
4444
drop table t1 all;

‎tests/sqllogictests/suites/base/20+_others/20_0013_query_result_cache.test‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ EXPLAIN SELECT * FROM t1, t2 ORDER BY a, b;
8989
ReadQueryResultCache
9090
├── SQL: SELECT * FROM t1, t2 ORDER BY a, b
9191
├── Number of rows: 9
92-
└── Result size: 180
92+
└── Result size: 225
9393

9494

9595

‎tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -323,26 +323,26 @@ query TIIIT
323323
select * from fuse_statistic('test_virtual_column', 'tweets');
324324
----
325325
id 10 0 4 (empty)
326-
data['create'] 10 0 16 (empty)
326+
data['create'] 10 0 24 (empty)
327327
data['id'] 10 0 8 (empty)
328328
data['replies'] 7 3 8 (empty)
329-
data['text'] 7 0 16 (empty)
329+
data['text'] 7 0 21 (empty)
330330
data['user']['id'] 6 0 8 (empty)
331331
data['likes'] 2 0 1 (empty)
332-
data['tags'][0] 2 0 3 (empty)
333-
data['tags'][1] 2 0 3 (empty)
332+
data['tags'][0] 2 0 4 (empty)
333+
data['tags'][1] 2 0 5 (empty)
334334

335335
query TTTIIIITTIT
336336
SHOW STATISTICS FROM TABLE test_virtual_column.tweets;
337337
----
338338
test_virtual_column tweets data 10 10 NULL NULL NULL NULL NULL (empty)
339-
test_virtual_column tweets data['create'] 10 10 10 0 1/08 6/07 16 (empty)
339+
test_virtual_column tweets data['create'] 10 10 10 0 1/08 6/07 24 (empty)
340340
test_virtual_column tweets data['id'] 10 10 10 0 1 10 8 (empty)
341341
test_virtual_column tweets data['likes'] 10 10 2 0 10 25 1 (empty)
342342
test_virtual_column tweets data['replies'] 10 10 7 3 0 9 8 (empty)
343-
test_virtual_column tweets data['tags'][0] 10 10 2 0 good new 3 (empty)
344-
test_virtual_column tweets data['tags'][1] 10 10 2 0 interesting popular 3 (empty)
345-
test_virtual_column tweets data['text'] 10 10 7 0 a z 16 (empty)
343+
test_virtual_column tweets data['tags'][0] 10 10 2 0 good new 4 (empty)
344+
test_virtual_column tweets data['tags'][1] 10 10 2 0 interesting popular 5 (empty)
345+
test_virtual_column tweets data['text'] 10 10 7 0 a z 21 (empty)
346346
test_virtual_column tweets data['user']['id'] 10 10 6 0 1 7 8 (empty)
347347
test_virtual_column tweets id 10 10 10 0 1 10 4 (empty)
348348

@@ -354,20 +354,20 @@ query TIIIT
354354
select * from fuse_statistic('test_virtual_column', 'tweets');
355355
----
356356
id 10 0 4 (empty)
357-
data['create'] 10 0 16 (empty)
357+
data['create'] 10 0 24 (empty)
358358
data['id'] 10 0 8 (empty)
359359
data['replies'] 7 3 8 (empty)
360-
data['text'] 8 0 16 (empty)
360+
data['text'] 8 0 21 (empty)
361361
data['user']['id'] 4 0 8 (empty)
362362

363363
query TTTIIIITTIT
364364
SHOW STATISTICS FROM TABLE test_virtual_column.tweets;
365365
----
366366
test_virtual_column tweets data 10 10 NULL NULL NULL NULL NULL (empty)
367-
test_virtual_column tweets data['create'] 10 10 10 0 1/08 6/07 16 (empty)
367+
test_virtual_column tweets data['create'] 10 10 10 0 1/08 6/07 24 (empty)
368368
test_virtual_column tweets data['id'] 10 10 10 0 1 10 8 (empty)
369369
test_virtual_column tweets data['replies'] 10 10 7 3 0 10 8 (empty)
370-
test_virtual_column tweets data['text'] 10 10 8 0 a z 16 (empty)
370+
test_virtual_column tweets data['text'] 10 10 8 0 a z 21 (empty)
371371
test_virtual_column tweets data['user']['id'] 10 10 4 0 1 7 8 (empty)
372372
test_virtual_column tweets id 10 10 10 0 1 10 4 (empty)
373373

‎tests/sqllogictests/suites/mode/cluster/explain_analyze.test‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ EvalScalar
3333
└── AggregatePartial
3434
├── cpu time: <slt:ignore>
3535
├── output rows: 12
36-
├── output bytes: 204.00 B
36+
├── output bytes: <slt:ignore>
3737
├── group by: []
3838
├── aggregate functions: [sum(number), count()]
3939
├── estimated rows: 1.00
@@ -182,7 +182,7 @@ Exchange
182182
├── wait time: <slt:ignore>
183183
├── exchange bytes: 1.09 KiB
184184
├── output rows: 1
185-
├── output bytes: 42.00 B
185+
├── output bytes: <slt:ignore>
186186
├── output columns: [article.article_id (#0), article.author_id (#1), article.viewer_id (#2), article.view_date (#3), author.name (#5), author.id (#4)]
187187
├── join type: INNER
188188
├── build keys: [author.id (#4)]
@@ -198,16 +198,16 @@ Exchange
198198
│ └── Filter
199199
│ ├── cpu time: <slt:ignore>
200200
│ ├── output rows: 1
201-
│ ├── output bytes: 31.00 B
201+
│ ├── output bytes: <slt:ignore>
202202
│ ├── output columns: [author.id (#4), author.name (#5)]
203203
│ ├── filters: [is_true(author.name (#5) = 'mark4')]
204204
│ ├── estimated rows: 1.25
205205
│ └── TableScan
206206
│ ├── cpu time: <slt:ignore>
207207
│ ├── wait time: <slt:ignore>
208208
│ ├── output rows: 1
209-
│ ├── output bytes: 31.00 B
210-
│ ├── bytes scanned: 31.00 B
209+
│ ├── output bytes: <slt:ignore>
210+
│ ├── bytes scanned: <slt:ignore>
211211
│ ├── runtime filter inlist/min-max time: <slt:ignore>
212212
│ ├── table: default.default.author
213213
│ ├── scan id: 1

0 commit comments

Comments
 (0)