First phase handling trimmed characters

Author: eyalkoren

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/PatternedTextParserBenchmark.java

@@ -9,12 +9,12 @@
 
 package org.elasticsearch.benchmark.index.mapper;
 
-import org.elasticsearch.xpack.logsdb.patternedtext.charparser.api.Parser;
-import org.elasticsearch.xpack.logsdb.patternedtext.charparser.api.ParseException;
-import org.elasticsearch.xpack.logsdb.patternedtext.charparser.api.ParserFactory;
 import org.elasticsearch.xpack.logsdb.patternedtext.charparser.api.Argument;
 import org.elasticsearch.xpack.logsdb.patternedtext.charparser.api.IPv4Argument;
 import org.elasticsearch.xpack.logsdb.patternedtext.charparser.api.IntegerArgument;
+import org.elasticsearch.xpack.logsdb.patternedtext.charparser.api.ParseException;
+import org.elasticsearch.xpack.logsdb.patternedtext.charparser.api.Parser;
+import org.elasticsearch.xpack.logsdb.patternedtext.charparser.api.ParserFactory;
 import org.elasticsearch.xpack.logsdb.patternedtext.charparser.api.PatternedMessage;
 import org.elasticsearch.xpack.logsdb.patternedtext.charparser.api.Timestamp;
 import org.openjdk.jmh.annotations.Benchmark;

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/charparser/compiler/SchemaCompiler.java

@@ -31,9 +31,9 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.StringJoiner;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.xpack.logsdb.patternedtext.charparser.common.CharCodes.ALPHABETIC_CHAR_CODE;
@@ -84,31 +84,31 @@ public static CompiledSchema compile(Schema schema) {
         BitmaskRegistry<MultiTokenType> multiTokenBitmaskRegistry = new BitmaskRegistry<>();
         for (org.elasticsearch.xpack.logsdb.patternedtext.charparser.schema.MultiTokenType multiTokenType : schema.getMultiTokenTypes()) {
             MultiTokenFormat format = multiTokenType.getFormat();
-            org.elasticsearch.xpack.logsdb.patternedtext.charparser.schema.TokenType[] tokens = format.getTokens();
+            List<org.elasticsearch.xpack.logsdb.patternedtext.charparser.schema.TokenType> tokens = format.getTokens();
 
             TimestampFormat timestampFormat = null;
             if (multiTokenType.encodingType() == EncodingType.TIMESTAMP) {
-                timestampFormat = createTimestampFormat(tokens);
+                timestampFormat = createTimestampFormat(format);
             }
 
             int subTokenCount = multiTokenType.getNumberOfSubTokens();
             int multiTokenBitmask = multiTokenBitmaskRegistry.register(
                 new MultiTokenType(multiTokenType.name(), multiTokenType.encodingType(), subTokenCount, timestampFormat)
             );
 
-            maxTokensPerMultiToken = Math.max(maxTokensPerMultiToken, tokens.length);
+            maxTokensPerMultiToken = Math.max(maxTokensPerMultiToken, tokens.size());
             maxSubTokensPerMultiToken = Math.max(maxSubTokensPerMultiToken, subTokenCount);
 
-            int bitmaskForTokenCount = tokenCountToMultiTokenBitmaskMap.computeIfAbsent(tokens.length, input -> 0);
+            int bitmaskForTokenCount = tokenCountToMultiTokenBitmaskMap.computeIfAbsent(tokens.size(), input -> 0);
             bitmaskForTokenCount |= multiTokenBitmask;
-            tokenCountToMultiTokenBitmaskMap.put(tokens.length, bitmaskForTokenCount);
+            tokenCountToMultiTokenBitmaskMap.put(tokens.size(), bitmaskForTokenCount);
 
             int bitmaskForSubTokenCount = subTokenCountToMultiTokenBitmaskMap.computeIfAbsent(subTokenCount, input -> 0);
             bitmaskForSubTokenCount |= multiTokenBitmask;
             subTokenCountToMultiTokenBitmaskMap.put(subTokenCount, bitmaskForSubTokenCount);
 
-            for (int i = 0; i < tokens.length; i++) {
-                String tokenName = tokens[i].name();
+            for (int i = 0; i < tokens.size(); i++) {
+                String tokenName = tokens.get(i).name();
                 ArrayList<Integer> bitmaskList = tokenTypeToMultiTokenBitmaskByPosition.computeIfAbsent(
                     tokenName,
                     input -> new ArrayList<>()
@@ -635,48 +635,56 @@ private record RangeBoundary(int boundary, boolean isLowerBound, int bitmask) {}
     // =================================================== Timestamp formatting ========================================================
 
     /**
-     * Creates a {@link TimestampFormat} from an array of {@link org.elasticsearch.xpack.logsdb.patternedtext.charparser.schema.TokenType}
-     * objects that represent timestamp components.
-     * @param formatTokens An array of TokenType objects representing the timestamp format.
+     * Creates a {@link TimestampFormat} from a {@link MultiTokenFormat} object that represents timestamp components.
+     * This method processes both token parts and literal string parts to construct the final Java time format.
+     * @param format The MultiTokenFormat object representing the timestamp format.
      * @return A TimestampFormat object containing the format string and an array indicating the order of timestamp components.
      */
-    static TimestampFormat createTimestampFormat(org.elasticsearch.xpack.logsdb.patternedtext.charparser.schema.TokenType[] formatTokens) {
-        StringJoiner javaTimeFormat = new StringJoiner(" ");
+    static TimestampFormat createTimestampFormat(MultiTokenFormat format) {
+        StringBuilder javaTimeFormat = new StringBuilder();
         int[] timestampComponentsOrder = new int[TimestampComponentType.values().length];
         Arrays.fill(timestampComponentsOrder, -1);
         int nextComponentIndex = 0;
-        for (org.elasticsearch.xpack.logsdb.patternedtext.charparser.schema.TokenType token : formatTokens) {
-            nextComponentIndex += appendTimestampComponents(token, javaTimeFormat, timestampComponentsOrder, nextComponentIndex);
+        for (Object part : format.getFormatParts()) {
+            if (part instanceof String) {
+                for (char c : ((String) part).toCharArray()) {
+                    appendDelimiter(javaTimeFormat, c);
+                }
+            } else if (part instanceof org.elasticsearch.xpack.logsdb.patternedtext.charparser.schema.TokenType token) {
+                StringBuilder tokenJavaTimeFormat = new StringBuilder();
+                nextComponentIndex += appendTimestampComponents(token, tokenJavaTimeFormat, timestampComponentsOrder, nextComponentIndex);
+                javaTimeFormat.append(tokenJavaTimeFormat);
+            }
         }
         return new TimestampFormat(javaTimeFormat.toString(), timestampComponentsOrder);
     }
 
     /**
      * Creates a {@link TimestampFormat} from a single {@link org.elasticsearch.xpack.logsdb.patternedtext.charparser.schema.TokenType}
      * object that represents a timestamp format.
-     * @param formatToken A TokenType object representing the timestamp format.
+     * @param timestampToken A TokenType object representing the timestamp format.
      * @return A TimestampFormat object containing the format string and an array indicating the order of timestamp components.
      */
-    static TimestampFormat createTimestampFormat(org.elasticsearch.xpack.logsdb.patternedtext.charparser.schema.TokenType formatToken) {
-        StringJoiner javaTimeFormat = new StringJoiner("");
+    static TimestampFormat createTimestampFormat(org.elasticsearch.xpack.logsdb.patternedtext.charparser.schema.TokenType timestampToken) {
+        StringBuilder javaTimeFormat = new StringBuilder();
         int[] timestampComponentsOrder = new int[TimestampComponentType.values().length];
         Arrays.fill(timestampComponentsOrder, -1);
-        appendTimestampComponents(formatToken, javaTimeFormat, timestampComponentsOrder, 0);
+        appendTimestampComponents(timestampToken, javaTimeFormat, timestampComponentsOrder, 0);
         return new TimestampFormat(javaTimeFormat.toString(), timestampComponentsOrder);
     }
 
     /**
      * Appends the details of a given token to the provided javaTimeFormat and updates the timestampComponentsOrder array to reflect the
      * order of timestamp components.
      * @param token the TokenType object representing the timestamp format
-     * @param javaTimeFormat the StringJoiner to append the Java time format string
+     * @param javaTimeFormat the StringBuilder to append the Java time format string
      * @param timestampComponentsOrder an array to store the order of timestamp components
      * @param nextComponentIndex the next index to use in the timestampComponentsOrder array
      * @return the number of timestamp components appended to the javaTimeFormat
      */
     private static int appendTimestampComponents(
         org.elasticsearch.xpack.logsdb.patternedtext.charparser.schema.TokenType token,
-        StringJoiner javaTimeFormat,
+        StringBuilder javaTimeFormat,
         int[] timestampComponentsOrder,
         int nextComponentIndex
     ) {
@@ -698,7 +706,7 @@ private static int appendTimestampComponents(
                 appendedComponents++;
             }
         }
-        javaTimeFormat.add(tokenJavaTimeFormat.toString());
+        javaTimeFormat.append(tokenJavaTimeFormat);
         return appendedComponents;
     }