Skip to content

GH-3235: Row count limit for each row group #3236

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gszadovszky merged 3 commits into from
Jun 18, 2025
Merged

GH-3235: Row count limit for each row group #3236

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5a5a8dc
[GH-3235] Row count limit for each row group
pan3793 Jun 4, 2025
5cff61d
nit
pan3793 Jun 4, 2025
a8c19b3
int
pan3793 Jun 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public class ParquetProperties {
public static final int DEFAULT_PAGE_VALUE_COUNT_THRESHOLD = Integer.MAX_VALUE / 2;
public static final int DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH = 64;
public static final int DEFAULT_STATISTICS_TRUNCATE_LENGTH = Integer.MAX_VALUE;
public static final int DEFAULT_ROW_GROUP_ROW_COUNT_LIMIT = Integer.MAX_VALUE;
public static final int DEFAULT_PAGE_ROW_COUNT_LIMIT = 20_000;
public static final int DEFAULT_MAX_BLOOM_FILTER_BYTES = 1024 * 1024;
public static final boolean DEFAULT_BLOOM_FILTER_ENABLED = false;
Expand Down Expand Up @@ -122,6 +123,7 @@ public static WriterVersion fromString(String name) {
private final ColumnProperty<Boolean> bloomFilterEnabled;
private final ColumnProperty<Boolean> adaptiveBloomFilterEnabled;
private final ColumnProperty<Integer> numBloomFilterCandidates;
private final int rowGroupRowCountLimit;
private final int pageRowCountLimit;
private final boolean pageWriteChecksumEnabled;
private final ColumnProperty<ByteStreamSplitMode> byteStreamSplitEnabled;
Expand Down Expand Up @@ -153,6 +155,7 @@ private ParquetProperties(Builder builder) {
this.maxBloomFilterBytes = builder.maxBloomFilterBytes;
this.adaptiveBloomFilterEnabled = builder.adaptiveBloomFilterEnabled.build();
this.numBloomFilterCandidates = builder.numBloomFilterCandidates.build();
this.rowGroupRowCountLimit = builder.rowGroupRowCountLimit;
this.pageRowCountLimit = builder.pageRowCountLimit;
this.pageWriteChecksumEnabled = builder.pageWriteChecksumEnabled;
this.byteStreamSplitEnabled = builder.byteStreamSplitEnabled.build();
Expand Down Expand Up @@ -302,6 +305,10 @@ public boolean estimateNextSizeCheck() {
return estimateNextSizeCheck;
}

public int getRowGroupRowCountLimit() {
return rowGroupRowCountLimit;
}

public int getPageRowCountLimit() {
return pageRowCountLimit;
}
Expand Down Expand Up @@ -400,6 +407,7 @@ public static class Builder {
private final ColumnProperty.Builder<Boolean> adaptiveBloomFilterEnabled;
private final ColumnProperty.Builder<Integer> numBloomFilterCandidates;
private final ColumnProperty.Builder<Boolean> bloomFilterEnabled;
private int rowGroupRowCountLimit = DEFAULT_ROW_GROUP_ROW_COUNT_LIMIT;
private int pageRowCountLimit = DEFAULT_PAGE_ROW_COUNT_LIMIT;
private boolean pageWriteChecksumEnabled = DEFAULT_PAGE_WRITE_CHECKSUM_ENABLED;
private final ColumnProperty.Builder<ByteStreamSplitMode> byteStreamSplitEnabled;
Expand Down Expand Up @@ -679,6 +687,12 @@ public Builder withBloomFilterEnabled(String columnPath, boolean enabled) {
return this;
}

public Builder withRowGroupRowCountLimit(int rowCount) {
Preconditions.checkArgument(rowCount > 0, "Invalid row count limit for row groups: %s", rowCount);
rowGroupRowCountLimit = rowCount;
return this;
}

public Builder withPageRowCountLimit(int rowCount) {
Preconditions.checkArgument(rowCount > 0, "Invalid row count limit for pages: %s", rowCount);
pageRowCountLimit = rowCount;
Expand Down
Loading