Update UnigramTokenizer.java

maxhniebergall · web-flow · commit aa3556b93b0c · 2024-11-19T15:25:48.000-05:00
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/UnigramTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/UnigramTokenizer.java
@@ -367,8 +367,10 @@ List<DelimitedToken.Encoded> tokenize(CharSequence inputSequence, IntToIntFuncti
                         new DelimitedToken.Encoded(
                             Strings.format("<0x%02X>", bytes[i]),
                             pieces[i],
+                            // even though we are changing the number of characters in the output, we don't
+                            // need to change the offsets. The offsets refer to the input characters
                             offsetCorrection.apply(node.startsAtCharPos),
-                            offsetCorrection.apply(node.startsAtCharPos + i)
+                            offsetCorrection.apply(endsAtChars)
                         )
                     );
                 }

Original file line number	Diff line number	Diff line change
`@@ -367,8 +367,10 @@ List<DelimitedToken.Encoded> tokenize(CharSequence inputSequence, IntToIntFuncti`
`367`	`367`	`new DelimitedToken.Encoded(`
`368`	`368`	`Strings.format("<0x%02X>", bytes[i]),`
`369`	`369`	`pieces[i],`
	`370`	`+ // even though we are changing the number of characters in the output, we don't`
	`371`	`+ // need to change the offsets. The offsets refer to the input characters`
`370`	`372`	`offsetCorrection.apply(node.startsAtCharPos),`
`371`		`- offsetCorrection.apply(node.startsAtCharPos + i)`
	`373`	`+ offsetCorrection.apply(endsAtChars)`
`372`	`374`	`)`
`373`	`375`	`);`
`374`	`376`	`}`