fix issue with string encoding length

Author: TimWhiting

test/corpus/dart.txt

@@ -346,3 +346,38 @@ void main() {
             (argument 
               (identifier) 
               (selector (unconditional_assignable_selector (identifier)))))))))))
+
+======================================
+Basic program
+======================================
+void main() {
+  test('', () {
+    'åÅ';
+    'åÅ';
+  });
+}
+
+---
+(program
+      (function_signature
+        (void_type)
+        (identifier)
+        (formal_parameter_list))
+      (function_body
+        (block
+          (expression_statement
+            (identifier)
+            (selector
+              (argument_part
+                (arguments
+                  (argument
+                    (string_literal))
+                  (argument
+                    (function_expression
+                      (formal_parameter_list)
+                      (function_expression_body
+                        (block
+                          (expression_statement
+                            (string_literal))
+                          (expression_statement
+                            (string_literal)))))))))))))

tree_sitter/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.1.2
+- Fix issue with string encoding length
+
 ## 0.1.1
 - Added some idiomatic dart apis
 

tree_sitter/example/tree_sitter.dart

@@ -7,16 +7,16 @@ void main() {
   final parser = Parser(
       sharedLibrary: resolveDylibPath('dart', path: Directory.current.path),
       entryPoint: 'tree_sitter_dart');
-  const program = "class A {}";
-//   const program = r"""
-// void main() {
-//   test('', () {
-//     'åÅ';
-//     'åÅ';
-//   });
-// }
-// """;
+  // const program = "class A {}";
+  const program = r"""
+void main() {
+  test('', () {
+    'åÅ';
+    'åÅ';
+  });
+}
+""";
   final tree = parser.parse(program);
-  print(tree.root.child(0).namedChild(0).string);
+  print(tree.root.string);
   print(parser.getText(tree.root.child(0).namedChild(0)));
 }

tree_sitter/lib/tree_sitter.dart

@@ -7,6 +7,7 @@ import 'dart:convert';
 import 'dart:ffi';
 import 'dart:ffi' as ffi;
 import 'dart:io';
+import 'dart:typed_data';
 
 import 'package:dylib/dylib.dart';
 import 'package:ffi/ffi.dart';
@@ -76,13 +77,13 @@ base class Parser implements Finalizable {
   /// Parses out a tree from the given string
   Tree parse(String program, {int? encoding}) {
     _contents = program;
-    final pProgram = program.toNativeUtf8().cast<Char>();
+    final (pProgram, len) = program.toNativeUtf8Len();
     if (encoding == null) {
       return Tree(treeSitterApi.ts_parser_parse_string(
-          parser, nullptr, pProgram, program.length));
+          parser, nullptr, pProgram.cast(), len));
     } else {
       return Tree(treeSitterApi.ts_parser_parse_string_encoding(
-          parser, nullptr, pProgram, program.length, encoding));
+          parser, nullptr, pProgram.cast(), len, encoding));
     }
   }
 
@@ -160,13 +161,12 @@ base class Query implements Finalizable {
 
   Query.fromSource(
       {required Pointer<TSLanguage> language, required String source}) {
-    final pSource = source.toNativeUtf8().cast<Char>();
-    final length = utf8.encode(source).length;
+    final (pSource, len) = source.toNativeUtf8Len();
     using((alloc) {
       final errorOffset = alloc<Uint32>(1);
       final errorType = alloc<Int32>(1);
       query = treeSitterApi.ts_query_new(
-          language, pSource, length, errorOffset, errorType);
+          language, pSource.cast(), len, errorOffset, errorType);
       if (query == nullptr) {
         final errOff = errorOffset.value;
         final errType = errorType.value;
@@ -286,10 +286,10 @@ extension TSNodeX on TSNode {
 
   int get namedChildCount => treeSitterApi.ts_node_named_child_count(this);
 
-  TSNode childByFieldName(String fieldName, int fieldNameLength) {
-    final pFieldName = fieldName.toNativeUtf8().cast<Char>();
+  TSNode childByFieldName(String fieldName) {
+    final (pFieldName, nameLength) = fieldName.toNativeUtf8Len();
     final result = treeSitterApi.ts_node_child_by_field_name(
-        this, pFieldName, fieldNameLength);
+        this, pFieldName.cast(), nameLength);
     malloc.free(pFieldName);
     return result;
   }
@@ -303,3 +303,14 @@ extension TSNodeX on TSNode {
   TSNode get nextNamedSibling => treeSitterApi.ts_node_next_named_sibling(this);
   TSNode get prevNamedSibling => treeSitterApi.ts_node_prev_named_sibling(this);
 }
+
+extension on String {
+  (Pointer<Utf8>, int) toNativeUtf8Len({Allocator allocator = malloc}) {
+    final units = utf8.encode(this);
+    final Pointer<Uint8> result = allocator<Uint8>(units.length + 1);
+    final Uint8List nativeString = result.asTypedList(units.length + 1);
+    nativeString.setAll(0, units);
+    nativeString[units.length] = 0;
+    return (result.cast(), units.length);
+  }
+}

tree_sitter/pubspec.yaml

@@ -1,6 +1,6 @@
 name: tree_sitter
 description: A tree sitter parsing library
-version: 0.1.1
+version: 0.1.2
 repository: https://github.com/UserNobody14/tree-sitter-dart
 
 environment: