diff --git a/documentation/src/docs/asciidoc/release-notes/release-notes-6.0.1.adoc b/documentation/src/docs/asciidoc/release-notes/release-notes-6.0.1.adoc index 721fa2bcf8b9..9b14d8144d79 100644 --- a/documentation/src/docs/asciidoc/release-notes/release-notes-6.0.1.adoc +++ b/documentation/src/docs/asciidoc/release-notes/release-notes-6.0.1.adoc @@ -35,7 +35,12 @@ repository on GitHub. [[release-notes-6.0.1-junit-jupiter-bug-fixes]] ==== Bug Fixes -* ❓ +* A regression introduced in version 6.0.0 caused an exception when using `@CsvSource` or + `@CsvFileSource` if the `delimiter` or `delimiterString` attribute was set to `+++#+++`. + This occurred because `+++#+++` was used as the default comment character without an + option to change it. To resolve this, a new `commentCharacter` attribute has been added + to both annotations. Its default value remains `+++#+++`, but it can now be customized + to avoid conflicts with other control characters. [[release-notes-6.0.1-junit-jupiter-deprecations-and-breaking-changes]] ==== Deprecations and Breaking Changes @@ -45,7 +50,8 @@ repository on GitHub. [[release-notes-6.0.1-junit-jupiter-new-features-and-improvements]] ==== New Features and Improvements -* ❓ +* The `@CsvSource` and `@CsvFileSource` annotations now allow specifying + a custom comment character using the new `commentCharacter` attribute. [[release-notes-6.0.1-junit-vintage]] diff --git a/documentation/src/docs/asciidoc/user-guide/writing-tests.adoc b/documentation/src/docs/asciidoc/user-guide/writing-tests.adoc index b181e75ecf95..11a6e6554c26 100644 --- a/documentation/src/docs/asciidoc/user-guide/writing-tests.adoc +++ b/documentation/src/docs/asciidoc/user-guide/writing-tests.adoc @@ -2270,12 +2270,16 @@ The generated display names for the previous example include the CSV header name ---- In contrast to CSV records supplied via the `value` attribute, a text block can contain -comments. Any line beginning with a `+++#+++` symbol will be treated as a comment and -ignored. Note, however, that the `+++#+++` symbol must be the first character on the line -without any leading whitespace. It is therefore recommended that the closing text block -delimiter (`"""`) be placed either at the end of the last line of input or on the -following line, left aligned with the rest of the input (as can be seen in the example -below which demonstrates formatting similar to a table). +comments. Any line beginning with the value of the `commentCharacter` attribute (`+++#+++` +by default) will be treated as a comment and ignored. Note that there is one exception +to this rule: if the comment character appears within a quoted field, it loses +its special meaning. + +The comment character must be the first character on the line without any leading +whitespace. It is therefore recommended that the closing text block delimiter (`"""`) +be placed either at the end of the last line of input or on the following line, +left aligned with the rest of the input (as can be seen in the example below which +demonstrates formatting similar to a table). [source,java,indent=0] ---- @@ -2325,8 +2329,8 @@ The default delimiter is a comma (`,`), but you can use another character by set cannot be set simultaneously. .Comments in CSV files -NOTE: Any line beginning with a `+++#+++` symbol will be interpreted as a comment and will -be ignored. +NOTE: Any line beginning with the value of the `commentCharacter` attribute (`+++#+++` +by default) will be interpreted as a comment and will be ignored. [source,java,indent=0] ---- diff --git a/gradle/config/japicmp/accepted-breaking-changes.txt b/gradle/config/japicmp/accepted-breaking-changes.txt index e69de29bb2d1..19c25d71f94d 100644 --- a/gradle/config/japicmp/accepted-breaking-changes.txt +++ b/gradle/config/japicmp/accepted-breaking-changes.txt @@ -0,0 +1,2 @@ +org.junit.jupiter.params.provider.CsvFileSource#commentCharacter +org.junit.jupiter.params.provider.CsvSource#commentCharacter diff --git a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvFileSource.java b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvFileSource.java index 28a1bf469c77..2e4287b314ec 100644 --- a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvFileSource.java +++ b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvFileSource.java @@ -10,6 +10,7 @@ package org.junit.jupiter.params.provider; +import static org.apiguardian.api.API.Status.EXPERIMENTAL; import static org.apiguardian.api.API.Status.STABLE; import java.lang.annotation.Documented; @@ -35,8 +36,8 @@ * that the first record may optionally be used to supply CSV headers (see * {@link #useHeadersInDisplayName}). * - *

Any line beginning with a {@code #} symbol will be interpreted as a comment - * and will be ignored. + *

Any line beginning with a {@link #commentCharacter} + * will be interpreted as a comment and will be ignored. * *

The column delimiter (which defaults to a comma ({@code ,})) can be customized * via either {@link #delimiter} or {@link #delimiterString}. @@ -63,6 +64,10 @@ * column is trimmed by default. This behavior can be changed by setting the * {@link #ignoreLeadingAndTrailingWhitespace} attribute to {@code true}. * + *

Note that {@link #delimiter} (or {@link #delimiterString}), + * {@link #quoteCharacter}, and {@link #commentCharacter} are treated as + * control characters and must all be distinct. + * *

Inheritance

* *

This annotation is inherited to subclasses. @@ -235,4 +240,22 @@ @API(status = STABLE, since = "5.10") boolean ignoreLeadingAndTrailingWhitespace() default true; + /** + * The character used to denote comments when reading the CSV files. + * + *

Any line that begins with this character will be treated as a comment + * and ignored during parsing. Note that there is one exception to this rule: + * if the comment character appears within a quoted field, it loses its + * special meaning. + * + *

The comment character must be the first character on the line without + * any leading whitespace. + * + *

Defaults to {@code '#'}. + * + * @since 6.0.1 + */ + @API(status = EXPERIMENTAL, since = "6.0.1") + char commentCharacter() default '#'; + } diff --git a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvReaderFactory.java b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvReaderFactory.java index 3a1bc58ddf04..122d426d5ea3 100644 --- a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvReaderFactory.java +++ b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvReaderFactory.java @@ -18,7 +18,9 @@ import java.nio.charset.Charset; import java.util.Set; import java.util.UUID; +import java.util.stream.Stream; +import de.siegmar.fastcsv.reader.CommentStrategy; import de.siegmar.fastcsv.reader.CsvCallbackHandler; import de.siegmar.fastcsv.reader.CsvReader; import de.siegmar.fastcsv.reader.CsvRecord; @@ -65,7 +67,11 @@ private static void validateDelimiter(char delimiter, String delimiterString, An static CsvReader createReaderFor(CsvSource csvSource, String data) { String delimiter = selectDelimiter(csvSource.delimiter(), csvSource.delimiterString()); + var commentStrategy = csvSource.textBlock().isEmpty() ? NONE : SKIP; // @formatter:off + validateControlCharactersDiffer( + delimiter, csvSource.quoteCharacter(), csvSource.commentCharacter(), commentStrategy); + var builder = CsvReader.builder() .skipEmptyLines(SKIP_EMPTY_LINES) .trimWhitespacesAroundQuotes(TRIM_WHITESPACES_AROUND_QUOTES) @@ -73,7 +79,8 @@ static CsvReader createReaderFor(CsvSource csvSource, Strin .allowMissingFields(ALLOW_MISSING_FIELDS) .fieldSeparator(delimiter) .quoteCharacter(csvSource.quoteCharacter()) - .commentStrategy(csvSource.textBlock().isEmpty() ? NONE : SKIP); + .commentStrategy(commentStrategy) + .commentCharacter(csvSource.commentCharacter()); var callbackHandler = createCallbackHandler( csvSource.emptyValue(), @@ -90,7 +97,11 @@ static CsvReader createReaderFor(CsvFileSource csvFileSourc Charset charset) { String delimiter = selectDelimiter(csvFileSource.delimiter(), csvFileSource.delimiterString()); + var commentStrategy = SKIP; // @formatter:off + validateControlCharactersDiffer( + delimiter, csvFileSource.quoteCharacter(), csvFileSource.commentCharacter(), commentStrategy); + var builder = CsvReader.builder() .skipEmptyLines(SKIP_EMPTY_LINES) .trimWhitespacesAroundQuotes(TRIM_WHITESPACES_AROUND_QUOTES) @@ -98,7 +109,8 @@ static CsvReader createReaderFor(CsvFileSource csvFileSourc .allowMissingFields(ALLOW_MISSING_FIELDS) .fieldSeparator(delimiter) .quoteCharacter(csvFileSource.quoteCharacter()) - .commentStrategy(SKIP); + .commentStrategy(commentStrategy) + .commentCharacter(csvFileSource.commentCharacter()); var callbackHandler = createCallbackHandler( csvFileSource.emptyValue(), @@ -121,6 +133,26 @@ private static String selectDelimiter(char delimiter, String delimiterString) { return DEFAULT_DELIMITER; } + private static void validateControlCharactersDiffer(String delimiter, char quoteCharacter, char commentCharacter, + CommentStrategy commentStrategy) { + + if (commentStrategy == NONE) { + Preconditions.condition(stringValuesUnique(delimiter, quoteCharacter), + () -> ("delimiter or delimiterString: '%s' and quoteCharacter: '%s' " + // + "must differ").formatted(delimiter, quoteCharacter)); + } + else { + Preconditions.condition(stringValuesUnique(delimiter, quoteCharacter, commentCharacter), + () -> ("delimiter or delimiterString: '%s', quoteCharacter: '%s', and commentCharacter: '%s' " + // + "must all differ").formatted(delimiter, quoteCharacter, commentCharacter)); + } + } + + private static boolean stringValuesUnique(Object... values) { + long uniqueCount = Stream.of(values).map(String::valueOf).distinct().count(); + return uniqueCount == values.length; + } + private static CsvCallbackHandler createCallbackHandler(String emptyValue, Set nullValues, boolean ignoreLeadingAndTrailingWhitespaces, int maxCharsPerColumn, boolean useHeadersInDisplayName) { diff --git a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvSource.java b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvSource.java index b8dd46cd5b61..ebd505e9b634 100644 --- a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvSource.java +++ b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/provider/CsvSource.java @@ -10,6 +10,7 @@ package org.junit.jupiter.params.provider; +import static org.apiguardian.api.API.Status.EXPERIMENTAL; import static org.apiguardian.api.API.Status.STABLE; import java.lang.annotation.Documented; @@ -62,6 +63,16 @@ * physical line within the text block. Thus, if a CSV column wraps across a * new line in a text block, the column must be a quoted string. * + *

Note that {@link #delimiter} (or {@link #delimiterString}), + * {@link #quoteCharacter}, and {@link #commentCharacter} (when + * {@link #textBlock} is used) are treated as control characters. + * + *

+ * *

Inheritance

* *

This annotation is inherited to subclasses. @@ -132,17 +143,20 @@ * {@link #useHeadersInDisplayName}). * *

In contrast to CSV records supplied via {@link #value}, a text block - * can contain comments. Any line beginning with a hash tag ({@code #}) will - * be treated as a comment and ignored. Note, however, that the {@code #} - * symbol must be the first character on the line without any leading - * whitespace. It is therefore recommended that the closing text block + * can contain comments. Any line beginning with a {@link #commentCharacter} + * will be treated as a comment and ignored. Note that there is one exception + * to this rule: if the comment character appears within a quoted field, + * it loses its special meaning. + * + *

The comment character must be the first character on the line without + * any leading whitespace. It is therefore recommended that the closing text block * delimiter {@code """} be placed either at the end of the last line of * input or on the following line, vertically aligned with the rest of the * input (as can be seen in the example below). * *

Java's text block * feature automatically removes incidental whitespace when the code - * is compiled. However other JVM languages such as Groovy and Kotlin do not. + * is compiled. However, other JVM languages such as Groovy and Kotlin do not. * Thus, if you are using a programming language other than Java and your text * block contains comments or new lines within quoted strings, you will need * to ensure that there is no leading whitespace within your text block. @@ -296,4 +310,22 @@ @API(status = STABLE, since = "5.10") boolean ignoreLeadingAndTrailingWhitespace() default true; + /** + * The character used to denote comments in a {@linkplain #textBlock text block}. + * + *

Any line that begins with this character will be treated as a comment + * and ignored during parsing. Note that there is one exception to this rule: + * if the comment character appears within a quoted field, it loses its + * special meaning. + * + *

The comment character must be the first character on the line without + * any leading whitespace. + * + *

Defaults to {@code '#'}. + * + * @since 6.0.1 + */ + @API(status = EXPERIMENTAL, since = "6.0.1") + char commentCharacter() default '#'; + } diff --git a/jupiter-tests/src/test/java/org/junit/jupiter/params/provider/CsvArgumentsProviderTests.java b/jupiter-tests/src/test/java/org/junit/jupiter/params/provider/CsvArgumentsProviderTests.java index 49e98ea4be36..b11102889352 100644 --- a/jupiter-tests/src/test/java/org/junit/jupiter/params/provider/CsvArgumentsProviderTests.java +++ b/jupiter-tests/src/test/java/org/junit/jupiter/params/provider/CsvArgumentsProviderTests.java @@ -400,6 +400,83 @@ void honorsCommentCharacterWhenUsingTextBlockAttribute() { assertThat(arguments).containsExactly(array("bar", "#baz"), array("#bar", "baz")); } + @Test + void honorsCustomCommentCharacter() { + var annotation = csvSource().textBlock(""" + *foo + bar, *baz + '*bar', baz + """).commentCharacter('*').build(); + + var arguments = provideArguments(annotation); + + assertThat(arguments).containsExactly(array("bar", "*baz"), array("*bar", "baz")); + } + + @Test + void doesNotThrowExceptionWhenDelimiterAndCommentCharacterTheSameWhenUsingValueAttribute() { + var annotation = csvSource().lines("foo#bar").delimiter('#').commentCharacter('#').build(); + + var arguments = provideArguments(annotation); + + assertThat(arguments).containsExactly(array("foo", "bar")); + } + + @ParameterizedTest + @MethodSource("invalidDelimiterAndQuoteCharacterCombinations") + void doesNotThrowExceptionWhenDelimiterAndCommentCharacterAreTheSameWhenUsingValueAttribute(Object delimiter, + char quoteCharacter) { + + var builder = csvSource().lines("foo").quoteCharacter(quoteCharacter); + + var annotation = delimiter instanceof Character c // + ? builder.delimiter(c).build() // + : builder.delimiterString(delimiter.toString()).build(); + + var message = "delimiter or delimiterString: '%s' and quoteCharacter: '%s' must differ"; + assertPreconditionViolationFor(() -> provideArguments(annotation).findAny()) // + .withMessage(message.formatted(delimiter, quoteCharacter)); + } + + static Stream invalidDelimiterAndQuoteCharacterCombinations() { + return Stream.of( + // delimiter + Arguments.of('*', '*'), // + // delimiterString + Arguments.of("*", '*')); + } + + @ParameterizedTest + @MethodSource("invalidDelimiterQuoteCharacterAndCommentCharacterCombinations") + void throwsExceptionWhenControlCharactersAreTheSameWhenUsingTextBlockAttribute(Object delimiter, + char quoteCharacter, char commentCharacter) { + + var builder = csvSource().textBlock(""" + foo""").quoteCharacter(quoteCharacter).commentCharacter(commentCharacter); + + var annotation = delimiter instanceof Character c // + ? builder.delimiter(c).build() // + : builder.delimiterString(delimiter.toString()).build(); + + var message = "delimiter or delimiterString: '%s', quoteCharacter: '%s', and commentCharacter: '%s' " + // + "must all differ"; + assertPreconditionViolationFor(() -> provideArguments(annotation).findAny()) // + .withMessage(message.formatted(delimiter, quoteCharacter, commentCharacter)); + } + + static Stream invalidDelimiterQuoteCharacterAndCommentCharacterCombinations() { + return Stream.of( + // delimiter + Arguments.of('#', '#', '#'), // + Arguments.of('#', '#', '*'), // + Arguments.of('*', '#', '#'), // + Arguments.of('#', '*', '#'), // + // delimiterString + Arguments.of("#", '#', '*'), // + Arguments.of("#", '*', '#') // + ); + } + @Test void supportsCsvHeadersWhenUsingTextBlockAttribute() { var annotation = csvSource().useHeadersInDisplayName(true).textBlock(""" diff --git a/jupiter-tests/src/test/java/org/junit/jupiter/params/provider/CsvFileArgumentsProviderTests.java b/jupiter-tests/src/test/java/org/junit/jupiter/params/provider/CsvFileArgumentsProviderTests.java index 3197611f18ec..aadfa6d0f3d6 100644 --- a/jupiter-tests/src/test/java/org/junit/jupiter/params/provider/CsvFileArgumentsProviderTests.java +++ b/jupiter-tests/src/test/java/org/junit/jupiter/params/provider/CsvFileArgumentsProviderTests.java @@ -134,6 +134,36 @@ void ignoresCommentedOutEntries() { assertThat(arguments).containsExactly(array("foo", "bar")); } + @Test + void honorsCustomCommentCharacter() { + var annotation = csvFileSource()// + .resources("test.csv")// + .commentCharacter(';')// + .delimiter(',')// + .build(); + + var arguments = provideArguments(annotation, "foo, bar \n;baz, qux"); + + assertThat(arguments).containsExactly(array("foo", "bar")); + } + + @ParameterizedTest + @MethodSource("org.junit.jupiter.params.provider.CsvArgumentsProviderTests#" + + "invalidDelimiterQuoteCharacterAndCommentCharacterCombinations") + void throwsExceptionWhenControlCharactersNotDiffer(Object delimiter, char quoteCharacter, char commentCharacter) { + var builder = csvFileSource().resources("test.csv") // + .quoteCharacter(quoteCharacter).commentCharacter(commentCharacter); + + var annotation = delimiter instanceof Character c // + ? builder.delimiter(c).build() // + : builder.delimiterString(delimiter.toString()).build(); + + var message = "delimiter or delimiterString: '%s', quoteCharacter: '%s', and commentCharacter: '%s' " + + "must all differ"; + assertPreconditionViolationFor(() -> provideArguments(annotation, "foo").findAny()) // + .withMessage(message.formatted(delimiter, quoteCharacter, commentCharacter)); + } + @Test void closesInputStreamForClasspathResource() { var closed = new AtomicBoolean(false); diff --git a/jupiter-tests/src/test/java/org/junit/jupiter/params/provider/MockCsvAnnotationBuilder.java b/jupiter-tests/src/test/java/org/junit/jupiter/params/provider/MockCsvAnnotationBuilder.java index 7386fed94278..e05863261273 100644 --- a/jupiter-tests/src/test/java/org/junit/jupiter/params/provider/MockCsvAnnotationBuilder.java +++ b/jupiter-tests/src/test/java/org/junit/jupiter/params/provider/MockCsvAnnotationBuilder.java @@ -42,6 +42,7 @@ static MockCsvFileSourceBuilder csvFileSource() { protected String[] nullValues = new String[0]; protected int maxCharsPerColumn = 4096; protected boolean ignoreLeadingAndTrailingWhitespace = true; + private char commentCharacter = '#'; private MockCsvAnnotationBuilder() { } @@ -88,6 +89,11 @@ B ignoreLeadingAndTrailingWhitespace(boolean ignoreLeadingAndTrailingWhitespace) return getSelf(); } + B commentCharacter(char commentCharacter) { + this.commentCharacter = commentCharacter; + return getSelf(); + } + abstract A build(); // ------------------------------------------------------------------------- @@ -129,6 +135,7 @@ CsvSource build() { when(annotation.nullValues()).thenReturn(super.nullValues); when(annotation.maxCharsPerColumn()).thenReturn(super.maxCharsPerColumn); when(annotation.ignoreLeadingAndTrailingWhitespace()).thenReturn(super.ignoreLeadingAndTrailingWhitespace); + when(annotation.commentCharacter()).thenReturn(super.commentCharacter); // @CsvSource when(annotation.value()).thenReturn(this.lines); @@ -188,6 +195,7 @@ CsvFileSource build() { when(annotation.nullValues()).thenReturn(super.nullValues); when(annotation.maxCharsPerColumn()).thenReturn(super.maxCharsPerColumn); when(annotation.ignoreLeadingAndTrailingWhitespace()).thenReturn(super.ignoreLeadingAndTrailingWhitespace); + when(annotation.commentCharacter()).thenReturn(super.commentCharacter); // @CsvFileSource when(annotation.resources()).thenReturn(this.resources);