Skip to content

[Backport] A specialized Trie for Block Tree Index #14563

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 9, 2025
Merged

[Backport] A specialized Trie for Block Tree Index #14563

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
df06c78
A specialized Trie for Block Tree Index (#14333)
gf2121 Apr 14, 2025
1329d6d
Fix OOM of TestTrie (#14488)
gf2121 Apr 15, 2025
3c242c6
Compute the doc range more efficiently when flushing doc block (#14447)
bugmakerrrrrr Apr 23, 2025
463efa1
Fix TestForTooMuchCloning (#14547)
gf2121 Apr 23, 2025
d1f5b2d
Fix tests: too many calls to IndexInput.clone during merging (#14595)
gf2121 May 7, 2025
1abeb3d
Simplify rootCodeFp (#14596)
gf2121 May 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 52 additions & 2 deletions gradle/generation/forUtil.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ configure(project(":lucene:core")) {
description "Regenerate gen_ForUtil.py"
group "generation"

def genDir = file("src/java/org/apache/lucene/codecs/lucene101")
def genDir = file("src/java/org/apache/lucene/codecs/lucene103")
def genScript = file("${genDir}/gen_ForUtil.py")
def genOutput = file("${genDir}/ForUtil.java")

Expand All @@ -48,7 +48,7 @@ configure(project(":lucene:core")) {
description "Regenerate gen_ForDeltaUtil.py"
group "generation"

def genDir = file("src/java/org/apache/lucene/codecs/lucene101")
def genDir = file("src/java/org/apache/lucene/codecs/lucene103")
def genScript = file("${genDir}/gen_ForDeltaUtil.py")
def genOutput = file("${genDir}/ForDeltaUtil.java")

Expand Down Expand Up @@ -197,5 +197,55 @@ configure(project(":lucene:backward-codecs")) {
andThenTasks: ["spotlessJava", "spotlessJavaApply"],
mustRunBefore: [ "compileJava" ]
])

task generateForUtil101Internal() {
description "Regenerate gen_ForUtil.py"
group "generation"

def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene101")
def genScript = file("${genDir}/gen_ForUtil.py")
def genOutput = file("${genDir}/ForUtil.java")

inputs.file genScript
outputs.file genOutput

doLast {
quietExec {
workingDir genDir
executable project.externalTool("python3")
args = [ '-B', genScript ]
}
}
}

regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil101Internal, [
andThenTasks: ["spotlessJava", "spotlessJavaApply"],
mustRunBefore: [ "compileJava" ]
])

task generateForDeltaUtil101Internal() {
description "Regenerate gen_ForDeltaUtil.py"
group "generation"

def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene101")
def genScript = file("${genDir}/gen_ForDeltaUtil.py")
def genOutput = file("${genDir}/ForDeltaUtil.java")

inputs.file genScript
outputs.file genOutput

doLast {
quietExec {
workingDir genDir
executable project.externalTool("python3")
args = [ '-B', genScript ]
}
}
}

regenerate.dependsOn wrapWithPersistentChecksums(generateForDeltaUtil101Internal, [
andThenTasks: ["spotlessJava", "spotlessJavaApply"],
mustRunBefore: [ "compileJava" ]
])
}

4 changes: 4 additions & 0 deletions lucene/CHANGES.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ Optimizations

* GITHUB#14425: KeywordField.newSetQuery() reuses prefixed terms (Mikhail Khludnev)

* GITHUB#14333: Introduce a specialized trie for block tree index, instead of FST. (Guo Feng)

* GITHUB#14447: Compute the doc range more efficiently when flushing doc block. (Pan Guixin)

Bug Fixes
---------------------
(No changes)
Expand Down
4 changes: 4 additions & 0 deletions lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil101.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/ForDeltaUtil.java": "f4cff08d9a5dd99f5332c2f9f6d386f0d7f58677",
"lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/gen_ForDeltaUtil.py": "ea46cd6b2384fc1cddb8c1dc5e30bf5f76054d91"
}
4 changes: 4 additions & 0 deletions lucene/backward-codecs/src/generated/checksums/generateForUtil101.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/ForUtil.java": "5a7c2e1e09780a2ccd31c22a1e1fa47443cf2a32",
"lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/gen_ForUtil.py": "c98cce3be0698048ebda6beaa9d404f25089930d"
}
8 changes: 6 additions & 2 deletions lucene/backward-codecs/src/java/module-info.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,15 @@
exports org.apache.lucene.backward_codecs.lucene86;
exports org.apache.lucene.backward_codecs.lucene87;
exports org.apache.lucene.backward_codecs.lucene90;
exports org.apache.lucene.backward_codecs.lucene90.blocktree;
exports org.apache.lucene.backward_codecs.lucene91;
exports org.apache.lucene.backward_codecs.lucene92;
exports org.apache.lucene.backward_codecs.lucene94;
exports org.apache.lucene.backward_codecs.lucene95;
exports org.apache.lucene.backward_codecs.lucene99;
exports org.apache.lucene.backward_codecs.lucene912;
exports org.apache.lucene.backward_codecs.lucene100;
exports org.apache.lucene.backward_codecs.lucene101;
exports org.apache.lucene.backward_codecs.packed;
exports org.apache.lucene.backward_codecs.store;

Expand All @@ -48,7 +50,8 @@
org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat,
org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat,
org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat,
org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat;
org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat,
org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat;
provides org.apache.lucene.codecs.KnnVectorsFormat with
org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat,
org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,
Expand All @@ -67,5 +70,6 @@
org.apache.lucene.backward_codecs.lucene95.Lucene95Codec,
org.apache.lucene.backward_codecs.lucene99.Lucene99Codec,
org.apache.lucene.backward_codecs.lucene912.Lucene912Codec,
org.apache.lucene.backward_codecs.lucene100.Lucene100Codec;
org.apache.lucene.backward_codecs.lucene100.Lucene100Codec,
org.apache.lucene.backward_codecs.lucene101.Lucene101Codec;
}
Loading