fix unicode and eof positions; enable token verification; fixes #122

Signed-off-by: Denys Smirnov <[email protected]>

Author: Denys Smirnov

driver/fixtures/fixtures_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/bblfsh/sdk/v3/driver"
 	"github.com/bblfsh/sdk/v3/driver/fixtures"
 	"github.com/bblfsh/sdk/v3/driver/native"
+	"github.com/bblfsh/sdk/v3/uast/transformer/positioner"
 )
 
 const projectRoot = "../../"
@@ -33,6 +34,12 @@ var Suite = &fixtures.Suite{
 			"MethodDeclaration",
 		},
 	},
+	VerifyTokens: []positioner.VerifyToken{
+		{Types: []string{
+			"SimpleName",
+			"StringLiteral",
+		}},
+	},
 }
 
 func TestJavaDriver(t *testing.T) {

driver/normalizer/normalizer.go

@@ -3,14 +3,17 @@ package normalizer
 import (
 	"github.com/bblfsh/sdk/v3/uast"
 	. "github.com/bblfsh/sdk/v3/uast/transformer"
+	"github.com/bblfsh/sdk/v3/uast/transformer/positioner"
 )
 
 var Preprocess []Transformer
 
 // PreprocessCode is a preprocessor stage that can use the source code to
 // fix tokens and positional information.
 // Java already provides all the information we need.
-var PreprocessCode []CodeTransformer
+var PreprocessCode = []CodeTransformer{
+	positioner.FromUTF16Offset(),
+}
 
 var Normalize = Transformers([][]Transformer{
 	// The main block of normalization rules.

fixtures/_integration.java.legacy

@@ -7,8 +7,8 @@ CompilationUnit {
 .  }
 .  EndPosition: {
 .  .  Offset: 18
-.  .  Line: 0
-.  .  Col: 0
+.  .  Line: 2
+.  .  Col: 1
 .  }
 .  Children: {
 .  .  0: TypeDeclaration {

fixtures/_integration.java.sem.uast

@@ -8,7 +8,8 @@
       },
       end: { '@type': "uast:Position",
          offset: 18,
-         col: 0,
+         line: 2,
+         col: 1,
       },
    },
    imports: ~,

fixtures/_integration.java.uast

@@ -8,7 +8,8 @@
       },
       end: { '@type': "uast:Position",
          offset: 18,
-         col: 0,
+         line: 2,
+         col: 1,
       },
    },
    imports: ~,

fixtures/annotation_type_member_declaration.java.sem.uast

@@ -8,7 +8,8 @@
       },
       end: { '@type': "uast:Position",
          offset: 37,
-         col: 0,
+         line: 3,
+         col: 2,
       },
    },
    imports: ~,
@@ -24,7 +25,8 @@
             },
             end: { '@type': "uast:Position",
                offset: 37,
-               col: 0,
+               line: 3,
+               col: 2,
             },
          },
          bodyDeclarations: [

fixtures/annotation_type_member_declaration.java.uast

@@ -8,7 +8,8 @@
       },
       end: { '@type': "uast:Position",
          offset: 37,
-         col: 0,
+         line: 3,
+         col: 2,
       },
    },
    imports: ~,
@@ -24,7 +25,8 @@
             },
             end: { '@type': "uast:Position",
                offset: 37,
-               col: 0,
+               line: 3,
+               col: 2,
             },
          },
          bodyDeclarations: [

fixtures/anonymous_class_declaration.java.sem.uast

@@ -8,7 +8,8 @@
       },
       end: { '@type': "uast:Position",
          offset: 86,
-         col: 0,
+         line: 5,
+         col: 1,
       },
    },
    imports: ~,

fixtures/anonymous_class_declaration.java.uast

@@ -8,7 +8,8 @@
       },
       end: { '@type': "uast:Position",
          offset: 86,
-         col: 0,
+         line: 5,
+         col: 1,
       },
    },
    imports: ~,

fixtures/array.java.sem.uast

@@ -8,7 +8,8 @@
       },
       end: { '@type': "uast:Position",
          offset: 77,
-         col: 0,
+         line: 7,
+         col: 1,
       },
    },
    imports: ~,