Merge remote-tracking branch 'origin/main' into int_score

Author: gf2121

dev-tools/scripts/addBackcompatIndexes.py

@@ -22,6 +22,7 @@
 
 import os
 import sys
+from pathlib import Path
 
 sys.path.append(os.path.dirname(__file__))
 import argparse
@@ -188,7 +189,7 @@ def read_config():
 def main():
   c = read_config()
   if not os.path.exists(c.temp_dir):
-    os.makedirs(c.temp_dir)
+    Path(c.temp_dir).mkdir(parents=True)
 
   print("\nCreating backwards compatibility indexes")
   source = download_release(c.version, c.temp_dir, c.force)

dev-tools/scripts/buildAndPushRelease.py

@@ -24,6 +24,7 @@
 import time
 import urllib.request
 import xml.etree.ElementTree as ET
+from pathlib import Path
 from subprocess import TimeoutExpired
 
 import scriptutil
@@ -213,13 +214,13 @@ def normalizeVersion(tup: tuple[str, ...]):
 
 def pushLocal(version: str, root: str, rcNum: int, localDir: str):
   print("Push local [%s]..." % localDir)
-  os.makedirs(localDir)
+  Path(localDir).mkdir(parents=True)
 
   lucene_dist_dir = "%s/lucene/distribution/build/release" % root
   rev = open("%s/lucene/distribution/build/release/.gitrev" % root, encoding="UTF-8").read()
 
   dir = "lucene-%s-RC%d-rev-%s" % (version, rcNum, rev)
-  os.makedirs("%s/%s/lucene" % (localDir, dir))
+  Path("%s/%s/lucene" % (localDir, dir)).mkdir(parents=True)
   print("  Lucene")
   os.chdir(lucene_dist_dir)
   print("    archive...")
@@ -237,7 +238,7 @@ def pushLocal(version: str, root: str, rcNum: int, localDir: str):
   run("chmod -R a+rX-w .")
 
   print("  done!")
-  return "file://%s/%s" % (os.path.abspath(localDir), dir)
+  return "file://%s/%s" % (Path(localDir).resolve(), dir)
 
 
 def read_version(_path: str):

dev-tools/scripts/pyproject.toml

@@ -95,8 +95,6 @@ ignore = [
   "PLW0602", # using global for variable but no assignment is done
   "PLW0603", # using global statement to update variable is discouraged
   "PLW2901", # loop variable overwritten by assignment target
-  "PTH100",  # os.path.abspath should be replaced by Path.resolve
-  "PTH103",  # os.mkdirs should be replaced by Path.mkdir(parents=True)
   "PTH104",  # os.rename should be replaced by Path.rename
   "PTH107",  # os.remove shoudl be replaced by Path.unlink
   "PTH109",  # os.getcwd should be replaced by Path.cwd

dev-tools/scripts/releaseWizard.py

@@ -48,6 +48,7 @@
 from collections import OrderedDict
 from collections.abc import Callable
 from datetime import UTC, datetime, timedelta
+from pathlib import Path
 from typing import Any, Self, TextIO, cast, override
 
 import yaml
@@ -488,7 +489,7 @@ def save(self):
     print("Saving")
     if not os.path.exists(os.path.join(self.config_path, self.release_version)):
       print("Creating folder %s" % os.path.join(self.config_path, self.release_version))
-      os.makedirs(os.path.join(self.config_path, self.release_version))
+      Path(os.path.join(self.config_path, self.release_version)).mkdir(parents=True)
 
     with open(os.path.join(self.config_path, self.release_version, "state.yaml"), "w") as fp:
       yaml.dump(self.to_dict(), fp, sort_keys=False, default_flow_style=False)
@@ -532,14 +533,14 @@ def get_release_folder(self):
     folder = os.path.join(self.config_path, self.release_version)
     if not os.path.exists(folder):
       print("Creating folder %s" % folder)
-      os.makedirs(folder)
+      Path(folder).mkdir(parents=True)
     return folder
 
   def get_rc_folder(self):
     folder = os.path.join(self.get_release_folder(), "RC%d" % self.rc_number)
     if not os.path.exists(folder):
       print("Creating folder %s" % folder)
-      os.makedirs(folder)
+      Path(folder).mkdir(parents=True)
     return folder
 
   def get_dist_folder(self):
@@ -1282,11 +1283,11 @@ def main():
       if root.startswith("~/"):
         release_root = os.path.expanduser(root)
       else:
-        release_root = os.path.abspath(root)
+        release_root = str(Path(root).resolve())
     if not os.path.exists(release_root):
       try:
         print("Creating release root %s" % release_root)
-        os.makedirs(release_root)
+        Path(release_root).mkdir(parents=True)
       except Exception as e:
         sys.exit("Error while creating %s: %s" % (release_root, e))
     release_version = get_release_version()
@@ -1305,7 +1306,7 @@ def main():
     y = yaml.load(open(os.path.join(script_path, "releaseWizard.yaml")), Loader=yaml.Loader)
     templates = y.get("templates")
     todo_list = y.get("groups")
-    state = ReleaseState(release_root, release_version, getScriptVersion())
+    state = ReleaseState(str(release_root), release_version, getScriptVersion())
     state.init_todos(bootstrap_todos(todo_list))
     state.load()
   except Exception as e:
@@ -1351,7 +1352,7 @@ def main():
 
 
 sys.path.append(os.path.dirname(__file__))
-current_git_root = os.path.abspath(os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir, os.path.pardir))
+current_git_root: str = str(Path(os.path.join(Path(os.path.dirname(__file__)).resolve(), os.path.pardir, os.path.pardir)).resolve())
 
 dry_run = False
 
@@ -1389,7 +1390,7 @@ def run_with_log_tail(command: str | list[str], cwd: str | None, logfile: str |
   if logfile:
     logdir = os.path.dirname(logfile)
     if not os.path.exists(logdir):
-      os.makedirs(logdir)
+      Path(logdir).mkdir(parents=True)
     fh = open(logfile, "w")
   rc = run_follow(command, cwd, fh=fh, tee=tee, live=live, shell=shell)
   if logfile:

dev-tools/scripts/reproduceJenkinsFailures.py

@@ -25,6 +25,7 @@
 import traceback
 import urllib.error
 import urllib.request
+from pathlib import Path
 from textwrap import dedent
 
 # Example: Checking out Revision e441a99009a557f82ea17ee9f9c3e9b89c75cee6 (refs/remotes/origin/master)
@@ -236,7 +237,7 @@ def printAndMoveReports(testIters: int, newSubDir: str, location: str):
                 break
           # have to play nice with 'ant clean'...
           newDirPath = os.path.join("repro-reports", newSubDir, dir)
-          os.makedirs(newDirPath, exist_ok=True)
+          Path(newDirPath).mkdir(exist_ok=True, parents=True)
           os.rename(filePath, os.path.join(newDirPath, file))
   print("[repro] Failures%s:" % location)
   for testcase in sorted(failures, key=lambda t: (failures[t], t)):  # sort by failure count, then by testcase

dev-tools/scripts/scriptutil.py

@@ -22,6 +22,7 @@
 import urllib.request
 from collections.abc import Callable
 from enum import Enum
+from pathlib import Path
 from re import Match, Pattern
 from typing import Self, override
 
@@ -188,7 +189,7 @@ def attemptDownload(urlString: str, fileName: str):
 
 def find_current_version():
   script_path = os.path.dirname(os.path.realpath(__file__))
-  top_level_dir = os.path.join(os.path.abspath("%s/" % script_path), os.path.pardir, os.path.pardir)
+  top_level_dir = os.path.join(Path("%s/" % script_path).resolve(), os.path.pardir, os.path.pardir)
   match = version_prop_re.search(open("%s/build.gradle" % top_level_dir).read())
   assert match
   return match.group(2).strip()

dev-tools/scripts/smokeTestRelease.py

@@ -35,6 +35,7 @@
 import zipfile
 from collections import namedtuple
 from collections.abc import Callable
+from pathlib import Path
 from typing import Any
 
 import scriptutil
@@ -262,7 +263,7 @@ def checkSigs(urlString: str, version: str, tmpDir: str, isSigned: bool, keysFil
   gpgHomeDir = "%s/lucene.gpg" % tmpDir
   if os.path.exists(gpgHomeDir):
     shutil.rmtree(gpgHomeDir)
-  os.makedirs(gpgHomeDir, 0o700)
+  Path(gpgHomeDir).mkdir(mode=0o700, parents=True)
   run("gpg --homedir %s --import %s" % (gpgHomeDir, keysFile), "%s/lucene.gpg.import.log" % tmpDir)
 
   if mavenURL is None:
@@ -446,7 +447,7 @@ def run(command: str, logFile: str):
   if cygwin:
     command = cygwinifyPaths(command)
   if os.system("%s > %s 2>&1" % (command, logFile)):
-    logPath = os.path.abspath(logFile)
+    logPath: str = str(Path(logFile).resolve())
     print('\ncommand "%s" failed:' % command)
     printFileContents(logFile)
     raise RuntimeError('command "%s" failed; see log file %s' % (command, logPath))
@@ -501,7 +502,7 @@ def unpackAndVerify(java: Any, tmpDir: str, artifact: str, gitRevision: str, ver
   destDir = "%s/unpack" % tmpDir
   if os.path.exists(destDir):
     shutil.rmtree(destDir)
-  os.makedirs(destDir)
+  Path(destDir).mkdir(parents=True)
   os.chdir(destDir)
   print("  unpack %s..." % artifact)
   unpackLogFile = "%s/lucene-unpack-%s.log" % (tmpDir, artifact)
@@ -722,7 +723,7 @@ def checkMaven(baseURL: str, tmpDir: str, gitRevision: str, version: str, isSign
   artifactsURL = "%s/lucene/maven/org/apache/lucene/" % baseURL
   targetDir = "%s/maven/org/apache/lucene" % tmpDir
   if not os.path.exists(targetDir):
-    os.makedirs(targetDir)
+    Path(targetDir).mkdir(parents=True)
   crawl(artifacts, artifactsURL, targetDir)
   print()
   verifyPOMperBinaryArtifact(artifacts, version)
@@ -747,7 +748,7 @@ def getBinaryDistFiles(tmpDir: str, version: str, baseURL: str):
   destDir = "%s/unpack-lucene-getBinaryDistFiles" % tmpDir
   if os.path.exists(destDir):
     shutil.rmtree(destDir)
-  os.makedirs(destDir)
+  Path(destDir).mkdir(parents=True)
   os.chdir(destDir)
   print("    unpack %s..." % distribution)
   unpackLogFile = "%s/unpack-%s-getBinaryDistFiles.log" % (tmpDir, distribution)
@@ -864,7 +865,7 @@ def verifyMavenSigs(tmpDir: str, artifacts: list[str], keysFile: str):
   gpgHomeDir = "%s/lucene.gpg" % tmpDir
   if os.path.exists(gpgHomeDir):
     shutil.rmtree(gpgHomeDir)
-  os.makedirs(gpgHomeDir, 0o700)
+  Path(gpgHomeDir).mkdir(mode=0o700, parents=True)
   run("gpg --homedir %s --import %s" % (gpgHomeDir, keysFile), "%s/lucene.gpg.import.log" % tmpDir)
 
   reArtifacts = re.compile(r"\.(?:pom|[jw]ar)$")
@@ -929,7 +930,7 @@ def crawl(downloadedFiles: list[str], urlString: str, targetDir: str, exclusions
       path = os.path.join(targetDir, text)
       if text.endswith("/"):
         if not os.path.exists(path):
-          os.makedirs(path)
+          Path(path).mkdir(parents=True)
         crawl(downloadedFiles, subURL, path, exclusions)
       else:
         if not os.path.exists(path) or FORCE_CLEAN:
@@ -1022,7 +1023,7 @@ def parse_config():
   c.java = make_java_config(parser, c.test_alternative_java)
 
   if c.tmp_dir:
-    c.tmp_dir = os.path.abspath(c.tmp_dir)
+    c.tmp_dir = str(Path(c.tmp_dir).resolve())
   else:
     tmp = "/tmp/smoke_lucene_%s_%s" % (c.version, c.revision)
     c.tmp_dir = tmp
@@ -1162,7 +1163,7 @@ def smokeTest(java: Any, baseURL: str, gitRevision: str, version: str, tmpDir: s
       raise RuntimeError("temp dir %s exists; please remove first" % tmpDir)
 
   if not os.path.exists(tmpDir):
-    os.makedirs(tmpDir)
+    Path(tmpDir).mkdir(parents=True)
 
   lucenePath = None
   print()

lucene/CHANGES.txt

@@ -106,6 +106,8 @@ Optimizations
 
 * GITHUB#14447: Compute the doc range more efficiently when flushing doc block. (Pan Guixin)
 
+* GITHUB#14527: Reduce NeighborArray heap memory. (weizijun)
+
 * GITHUB#14529, GITHUB#14555, GITHUB#14618: Impl intoBitset for IndexedDISI and Docvalues. (Guo Feng)
 
 * GITHUB#14552: Speed up flush of softdelete by intoBitset. (Guo Feng)
@@ -119,6 +121,13 @@ Optimizations
 * GITHUB#14700: Return MatchNoDocsQuery when IndexOrDocValuesQuery::rewrite does not match
   (Chris Hegarty)
 
+* GITHUB#14701: Optimize top-n bulk scorers by evaluating scoring windows in a
+  term-at-a-time fashion instead of doc-at-a-time. (Adrien Grand)
+
+* GITHUB#14709: Speed up TermQuery by Scorer#nextDocsAndScores. (Guo Feng)
+
+* GITHUB#14674: Optimize AbstractKnnVectorQuery#createBitSet with intoBitset. (Guo Feng)
+
 Bug Fixes
 ---------------------
 * GITHUB#14654: ValueSource.fromDoubleValuesSource(dvs).getSortField() would throw errors when
@@ -142,9 +151,9 @@ Other
 
 * GITHUB#14516: Move sloppySin into SloppyMath from GeoUtils (Ankit Jain)
 
-* GITHUB#14627, GITHUB#11920: Fix mock filesystem WindowsFS to also work on Windows.
-  This is required to make tests pass on Windows 11 which no longer has
-  all limitations of previous Windows versions.  (Uwe Schindler)
+* GITHUB#14627, GITHUB#11920, GITHUB#14706: Fix mock filesystem WindowsFS to also
+  work on Windows. This is required to make tests pass on Windows 11 which no
+  longer has all limitations of previous Windows versions.  (Uwe Schindler)
 
 
 ======================= Lucene 10.2.1 =======================

lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java

@@ -1097,6 +1097,32 @@ private void bufferIntoBitSet(int start, int end, FixedBitSet bitSet, int offset
       }
     }
 
+    @Override
+    public int docIDRunEnd() throws IOException {
+      // Note: this assumes that BLOCK_SIZE == 128, this bit of the code would need to be changed if
+      // the block size was changed.
+      // Hack to avoid compiler warning that both sides of the equal sign are identical.
+      long blockSize = BLOCK_SIZE;
+      assert blockSize == 2 * Long.SIZE;
+      boolean level0IsDense =
+          encoding == DeltaEncoding.UNARY
+              && docBitSet.getBits()[0] == -1L
+              && docBitSet.getBits()[1] == -1L;
+      if (level0IsDense) {
+
+        int level0DocCountUpto = docFreq - docCountLeft;
+        boolean level1IsDense =
+            level1LastDocID - level0LastDocID == level1DocCountUpto - level0DocCountUpto;
+        if (level1IsDense) {
+          return level1LastDocID + 1;
+        }
+
+        return level0LastDocID + 1;
+      }
+
+      return super.docIDRunEnd();
+    }
+
     private void skipPositions(int freq) throws IOException {
       // Skip positions now:
       int toSkip = posPendingCount - freq;