Skip to content

[core]Support enums without spaces in TagPeriodFormatter #5165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 3 commits into from
Mar 3, 2025
Merged

[core]Support enums without spaces in TagPeriodFormatter #5165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2d2f287
[core]Support enums without spaces in TagPeriodFormatter
chenxinwei Feb 26, 2025
52a0f63
fix comments
chenxinwei Feb 27, 2025
4868ab6
Fix bug in ci
chenxinwei Feb 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/layouts/shortcodes/generated/core_configuration.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1012,7 +1012,7 @@
<td><h5>tag.period-formatter</h5></td>
<td style="word-wrap: break-word;">with_dashes</td>
<td><p>Enum</p></td>
<td>The date format for tag periods.<br /><br />Possible values:<ul><li>"with_dashes": Dates and hours with dashes, e.g., 'yyyy-MM-dd HH'</li><li>"without_dashes": Dates and hours without dashes, e.g., 'yyyyMMdd HH'</li></ul></td>
<td>The date format for tag periods.<br /><br />Possible values:<ul><li>"with_dashes": Dates and hours with dashes, e.g., 'yyyy-MM-dd HH'</li><li>"without_dashes": Dates and hours without dashes, e.g., 'yyyyMMdd HH'</li><li>"without_dashes_and_spaces": Dates and hours without dashes and spaces, e.g., 'yyyyMMddHH'</li></ul></td>
</tr>
<tr>
<td><h5>target-file-size</h5></td>
Expand Down
5 changes: 4 additions & 1 deletion paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2999,7 +2999,10 @@ public InlineElement getDescription() {
/** The period format options for tag creation. */
public enum TagPeriodFormatter implements DescribedEnum {
WITH_DASHES("with_dashes", "Dates and hours with dashes, e.g., 'yyyy-MM-dd HH'"),
WITHOUT_DASHES("without_dashes", "Dates and hours without dashes, e.g., 'yyyyMMdd HH'");
WITHOUT_DASHES("without_dashes", "Dates and hours without dashes, e.g., 'yyyyMMdd HH'"),
WITHOUT_DASHES_AND_SPACES(
"without_dashes_and_spaces",
"Dates and hours without dashes and spaces, e.g., 'yyyyMMddHH'");

private final String value;
private final String description;
Expand Down
12 changes: 12 additions & 0 deletions paimon-core/src/main/java/org/apache/paimon/tag/TagPeriodHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,15 @@ public interface TagPeriodHandler {
.toFormatter()
.withResolverStyle(ResolverStyle.LENIENT);

DateTimeFormatter HOUR_FORMATTER_WITHOUT_DASHES_AND_SPACES =
new DateTimeFormatterBuilder()
.appendValue(YEAR, 1, 10, SignStyle.NORMAL)
.appendValue(MONTH_OF_YEAR, 2, 2, SignStyle.NORMAL)
.appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NORMAL)
.appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NORMAL)
.toFormatter()
.withResolverStyle(ResolverStyle.LENIENT);

DateTimeFormatter MINUTE_FORMATTER =
new DateTimeFormatterBuilder()
.appendValue(YEAR, 1, 10, SignStyle.NORMAL)
Expand Down Expand Up @@ -179,6 +188,8 @@ protected DateTimeFormatter formatter() {
return HOUR_FORMATTER;
case WITHOUT_DASHES:
return HOUR_FORMATTER_WITHOUT_DASHES;
case WITHOUT_DASHES_AND_SPACES:
return HOUR_FORMATTER_WITHOUT_DASHES_AND_SPACES;
default:
throw new IllegalArgumentException("Unsupported date format type");
}
Expand Down Expand Up @@ -207,6 +218,7 @@ protected DateTimeFormatter formatter() {
case WITH_DASHES:
return DAY_FORMATTER;
case WITHOUT_DASHES:
case WITHOUT_DASHES_AND_SPACES:
return DAY_FORMATTER_WITHOUT_DASHES;
default:
throw new IllegalArgumentException("Unsupported date format type");
Expand Down
17 changes: 17 additions & 0 deletions paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,23 @@ public void testTagHourlyPeriodFormatter() {
assertThat(tagManager.allTagNames()).contains("20230718 11", "20230718 12");
}

@Test
public void testTagHourlyPeriodFormatterWithoutDashesAndSpaces() {
Options options = new Options();
options.set(TAG_AUTOMATIC_CREATION, TagCreationMode.WATERMARK);
options.set(TAG_CREATION_PERIOD, TagCreationPeriod.HOURLY);
options.set(TAG_PERIOD_FORMATTER, TagPeriodFormatter.WITHOUT_DASHES_AND_SPACES);
FileStoreTable table = this.table.copy(options.toMap());
TableCommitImpl commit = table.newCommit(commitUser).ignoreEmptyCommit(false);
TagManager tagManager = table.store().newTagManager();

commit.commit(new ManifestCommittable(0, utcMills("2023-07-18T12:12:00")));
assertThat(tagManager.allTagNames()).containsOnly("2023071811");

commit.commit(new ManifestCommittable(1, utcMills("2023-07-18T13:13:00")));
assertThat(tagManager.allTagNames()).contains("2023071811", "2023071812");
}

@Test
public void testOnlyExpireAutoCreatedTag() {
Options options = new Options();
Expand Down
Loading