diff --git a/pom.xml b/pom.xml index 1f22800..686a27f 100644 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,11 @@ junit-jupiter-api test + + org.junit.jupiter + junit-jupiter-params + test + diff --git a/src/main/java/io/github/treesitter/jtreesitter/InputEncoding.java b/src/main/java/io/github/treesitter/jtreesitter/InputEncoding.java index 9ce00b0..d4a9d8c 100644 --- a/src/main/java/io/github/treesitter/jtreesitter/InputEncoding.java +++ b/src/main/java/io/github/treesitter/jtreesitter/InputEncoding.java @@ -1,5 +1,9 @@ package io.github.treesitter.jtreesitter; +import static io.github.treesitter.jtreesitter.internal.TreeSitter.TSInputEncodingUTF16BE; +import static io.github.treesitter.jtreesitter.internal.TreeSitter.TSInputEncodingUTF16LE; +import static io.github.treesitter.jtreesitter.internal.TreeSitter.TSInputEncodingUTF8; + import java.nio.ByteOrder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -8,30 +12,36 @@ /** The encoding of source code. */ public enum InputEncoding { /** UTF-8 encoding. */ - UTF_8(StandardCharsets.UTF_8), + UTF_8(StandardCharsets.UTF_8, TSInputEncodingUTF8()), /** * UTF-16 little endian encoding. * * @since 0.25.0 */ - UTF_16LE(StandardCharsets.UTF_16LE), + UTF_16LE(StandardCharsets.UTF_16LE, TSInputEncodingUTF16LE()), /** * UTF-16 big endian encoding. * * @since 0.25.0 */ - UTF_16BE(StandardCharsets.UTF_16BE); + UTF_16BE(StandardCharsets.UTF_16BE, TSInputEncodingUTF16BE()); private final @NonNull Charset charset; + private final int tsInputEncoding; - InputEncoding(@NonNull Charset charset) { + InputEncoding(@NonNull Charset charset, int tsInputEncoding) { this.charset = charset; + this.tsInputEncoding = tsInputEncoding; } Charset charset() { return charset; } + int tsInputEncoding() { + return tsInputEncoding; + } + private static final boolean IS_BIG_ENDIAN = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); /** diff --git a/src/main/java/io/github/treesitter/jtreesitter/Parser.java b/src/main/java/io/github/treesitter/jtreesitter/Parser.java index feb9eb2..78a1fda 100644 --- a/src/main/java/io/github/treesitter/jtreesitter/Parser.java +++ b/src/main/java/io/github/treesitter/jtreesitter/Parser.java @@ -254,7 +254,7 @@ public Optional parse(String source, InputEncoding encoding, @Nullable Tre var bytes = source.getBytes(encoding.charset()); var string = alloc.allocateFrom(C_CHAR, bytes); var old = oldTree == null ? MemorySegment.NULL : oldTree.segment(); - var tree = ts_parser_parse_string_encoding(self, old, string, bytes.length, encoding.ordinal()); + var tree = ts_parser_parse_string_encoding(self, old, string, bytes.length, encoding.tsInputEncoding()); if (tree.equals(MemorySegment.NULL)) return Optional.empty(); return Optional.of(new Tree(tree, language, source, encoding.charset())); } @@ -303,7 +303,7 @@ public Optional parse( var input = TSInput.allocate(arena); TSInput.payload(input, MemorySegment.NULL); - TSInput.encoding(input, encoding.ordinal()); + TSInput.encoding(input, encoding.tsInputEncoding()); // NOTE: can't use _ because of palantir/palantir-java-format#934 var read = TSInput.read.allocate( (payload, index, point, bytes) -> { diff --git a/src/test/java/io/github/treesitter/jtreesitter/ParserTest.java b/src/test/java/io/github/treesitter/jtreesitter/ParserTest.java index 231bf78..d96a45c 100644 --- a/src/test/java/io/github/treesitter/jtreesitter/ParserTest.java +++ b/src/test/java/io/github/treesitter/jtreesitter/ParserTest.java @@ -8,6 +8,8 @@ import java.util.List; import java.util.concurrent.*; import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; class ParserTest { private static Language language; @@ -86,6 +88,21 @@ void parseUtf16() { } } + @ParameterizedTest + @EnumSource(InputEncoding.class) + @DisplayName("parse(encoding)") + void parseEncoding(InputEncoding encoding) { + parser.setLanguage(language); + var source = "var text = \"☕fi𝄞\";"; + try (var tree = parser.parse(source, encoding).orElseThrow()) { + var rootNode = tree.getRootNode(); + + assertFalse(rootNode.isError()); + assertEquals(source, tree.getText()); + assertEquals(source, rootNode.getText()); + } + } + @Test @DisplayName("parse(logger)") void parseLogger() {