A specialized Trie for Block Tree Index (#14563)

* A specialized Trie for Block Tree Index (#14333)

* Fix OOM of TestTrie (#14488)

* Compute the doc range more efficiently when flushing doc block (#14447)

* Fix TestForTooMuchCloning (#14547)

* Fix tests: too many calls to IndexInput.clone during merging (#14595)

* Simplify rootCodeFp (#14596)

---------

Co-authored-by: panguixin <[email protected]>

Author: gf2121

gradle/generation/forUtil.gradle

@@ -23,7 +23,7 @@ configure(project(":lucene:core")) {
     description "Regenerate gen_ForUtil.py"
     group "generation"
 
-    def genDir = file("src/java/org/apache/lucene/codecs/lucene101")
+    def genDir = file("src/java/org/apache/lucene/codecs/lucene103")
     def genScript = file("${genDir}/gen_ForUtil.py")
     def genOutput = file("${genDir}/ForUtil.java")
 
@@ -48,7 +48,7 @@ configure(project(":lucene:core")) {
     description "Regenerate gen_ForDeltaUtil.py"
     group "generation"
 
-    def genDir = file("src/java/org/apache/lucene/codecs/lucene101")
+    def genDir = file("src/java/org/apache/lucene/codecs/lucene103")
     def genScript = file("${genDir}/gen_ForDeltaUtil.py")
     def genOutput = file("${genDir}/ForDeltaUtil.java")
 
@@ -197,5 +197,55 @@ configure(project(":lucene:backward-codecs")) {
           andThenTasks: ["spotlessJava", "spotlessJavaApply"],
           mustRunBefore: [ "compileJava" ]
   ])
+
+  task generateForUtil101Internal() {
+    description "Regenerate gen_ForUtil.py"
+    group "generation"
+
+    def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene101")
+    def genScript = file("${genDir}/gen_ForUtil.py")
+    def genOutput = file("${genDir}/ForUtil.java")
+
+    inputs.file genScript
+    outputs.file genOutput
+
+    doLast {
+      quietExec {
+        workingDir genDir
+        executable project.externalTool("python3")
+        args = [ '-B', genScript ]
+      }
+    }
+  }
+
+  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil101Internal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
+
+  task generateForDeltaUtil101Internal() {
+    description "Regenerate gen_ForDeltaUtil.py"
+    group "generation"
+
+    def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene101")
+    def genScript = file("${genDir}/gen_ForDeltaUtil.py")
+    def genOutput = file("${genDir}/ForDeltaUtil.java")
+
+    inputs.file genScript
+    outputs.file genOutput
+
+    doLast {
+      quietExec {
+        workingDir genDir
+        executable project.externalTool("python3")
+        args = [ '-B', genScript ]
+      }
+    }
+  }
+
+  regenerate.dependsOn wrapWithPersistentChecksums(generateForDeltaUtil101Internal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }
 

lucene/CHANGES.txt

@@ -41,6 +41,10 @@ Optimizations
 
 * GITHUB#14425: KeywordField.newSetQuery() reuses prefixed terms (Mikhail Khludnev)
 
+* GITHUB#14333: Introduce a specialized trie for block tree index, instead of FST. (Guo Feng)
+
+* GITHUB#14447: Compute the doc range more efficiently when flushing doc block. (Pan Guixin)
+
 Bug Fixes
 ---------------------
 (No changes)

lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil101.json

@@ -0,0 +1,4 @@
+{
+    "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/ForDeltaUtil.java": "f4cff08d9a5dd99f5332c2f9f6d386f0d7f58677",
+    "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/gen_ForDeltaUtil.py": "ea46cd6b2384fc1cddb8c1dc5e30bf5f76054d91"
+}

lucene/backward-codecs/src/generated/checksums/generateForUtil101.json

@@ -0,0 +1,4 @@
+{
+    "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/ForUtil.java": "5a7c2e1e09780a2ccd31c22a1e1fa47443cf2a32",
+    "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/gen_ForUtil.py": "c98cce3be0698048ebda6beaa9d404f25089930d"
+}

lucene/backward-codecs/src/java/module-info.java

@@ -31,13 +31,15 @@
   exports org.apache.lucene.backward_codecs.lucene86;
   exports org.apache.lucene.backward_codecs.lucene87;
   exports org.apache.lucene.backward_codecs.lucene90;
+  exports org.apache.lucene.backward_codecs.lucene90.blocktree;
   exports org.apache.lucene.backward_codecs.lucene91;
   exports org.apache.lucene.backward_codecs.lucene92;
   exports org.apache.lucene.backward_codecs.lucene94;
   exports org.apache.lucene.backward_codecs.lucene95;
   exports org.apache.lucene.backward_codecs.lucene99;
   exports org.apache.lucene.backward_codecs.lucene912;
   exports org.apache.lucene.backward_codecs.lucene100;
+  exports org.apache.lucene.backward_codecs.lucene101;
   exports org.apache.lucene.backward_codecs.packed;
   exports org.apache.lucene.backward_codecs.store;
 
@@ -48,7 +50,8 @@
       org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat,
       org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat,
       org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat,
-      org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat;
+      org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat,
+      org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat;
   provides org.apache.lucene.codecs.KnnVectorsFormat with
       org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat,
       org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,
@@ -67,5 +70,6 @@
       org.apache.lucene.backward_codecs.lucene95.Lucene95Codec,
       org.apache.lucene.backward_codecs.lucene99.Lucene99Codec,
       org.apache.lucene.backward_codecs.lucene912.Lucene912Codec,
-      org.apache.lucene.backward_codecs.lucene100.Lucene100Codec;
+      org.apache.lucene.backward_codecs.lucene100.Lucene100Codec,
+      org.apache.lucene.backward_codecs.lucene101.Lucene101Codec;
 }