fix(whoosh): correct skip list implementation errors

Co-authored-by: Matheus Virgolino <matheus.virgolino.abilio.da.silva@ccc.ufcg.edu.br>
Co-authored-by: Manoel Netto <manoel.da.nobrega.eustaqueo.netto@ccc.ufcg.edu.br>
Co-authored-by: Pedro <pedroalmeida1896@gmail.com>
Co-authored-by: Lucaslg7 <lucasmoizinholg7@gmail.com>
Co-authored-by: RailtonDantas <railtondantas.code@gmail.com>
Co-authored-by: João Pereira <joao.pereira.de.oliveira@ccc.ufcg.edu.br>

Author: 6 people

src/whoosh/codec/memory.py

@@ -30,7 +30,7 @@
 from threading import Lock
 
 from whoosh.codec import base
-from whoosh.matching import ListMatcher
+from whoosh.matching import ListMatcher, SkipListMatcher
 from whoosh.reading import SegmentReader, TermInfo, TermNotFound
 from whoosh.writing import SegmentWriter
 
@@ -182,7 +182,7 @@ def has_vector(self, docnum, fieldname):
     def vector(self, docnum, fieldname, format_):
         items = self._segment._vectors[docnum][fieldname]
         ids, weights, values = zip(*items)
-        return ListMatcher(ids, weights, values, format_)
+        return SkipListMatcher(ids, weights, values, format_)
 
     def stored_fields(self, docnum):
         return self._segment._stored[docnum]
@@ -284,7 +284,7 @@ def term_info(self, fieldname, text):
     def matcher(self, fieldname, btext, format_, scorer=None):
         items = self._invindex[fieldname][btext]
         ids, weights, values = zip(*items)
-        return ListMatcher(ids, weights, values, format_, scorer=scorer)
+        return SkipListMatcher(ids, weights, values, format_, scorer=scorer)
 
     def indexed_field_names(self):
         return self._invindex.keys()

src/whoosh/codec/plaintext.py

@@ -29,7 +29,7 @@
 from pickle import dumps, loads
 
 from whoosh.codec import base
-from whoosh.matching import ListMatcher
+from whoosh.matching import ListMatcher, SkipListMatcher
 from whoosh.reading import TermInfo, TermNotFound
 
 _reprable = (bytes, str, int, float)
@@ -273,7 +273,7 @@ def vector(self, docnum, fieldname, format_):
             values.append(c["v"])
             c = self._find_line(3, "VPOST")
 
-        return ListMatcher(
+        return SkipListMatcher(
             ids,
             weights,
             values,
@@ -433,7 +433,7 @@ def matcher(self, fieldname, btext, format_, scorer=None):
             values.append(c["v"])
             c = self._find_line(3, "POST")
 
-        return ListMatcher(ids, weights, values, format_, scorer=scorer)
+        return SkipListMatcher(ids, weights, values, format_, scorer=scorer)
 
     def close(self):
         self._dbfile.close()

src/whoosh/codec/whoosh3.py

@@ -37,7 +37,7 @@
 from whoosh import columns, formats
 from whoosh.codec import base
 from whoosh.filedb import compound, filetables
-from whoosh.matching import LeafMatcher, ListMatcher, ReadTooFar
+from whoosh.matching import LeafMatcher, ListMatcher, ReadTooFar, SkipListMatcher
 from whoosh.reading import TermInfo, TermNotFound
 from whoosh.system import (
     _FLOAT_SIZE,
@@ -111,9 +111,9 @@ def postings_writer(self, dbfile, byteids=False):
     def postings_reader(self, dbfile, terminfo, format_, term=None, scorer=None):
         if terminfo.is_inlined():
             # If the postings were inlined into the terminfo object, pull them
-            # out and use a ListMatcher to wrap them in a Matcher interface
+            # out and use a SkipListMatcher to wrap them in a Matcher interface
             ids, weights, values = terminfo.inlined_postings()
-            m = ListMatcher(
+            m = SkipListMatcher(
                 ids,
                 weights,
                 values,

src/whoosh/matching/mcore.py

@@ -573,11 +573,36 @@ def score(self):
         else:
             return self.weight()
 
+
 class SkipListMatcher(ListMatcher):
-    def __init__(self, ids, **kwargs):
-        super().__init__(ids, **kwargs)
+    def __init__(
+        self,
+        ids,
+        weights=None,
+        values=None,
+        format=None,
+        scorer=None,
+        position=0,
+        all_weights=None,
+        term=None,
+        terminfo=None,
+    ):
+        super().__init__(
+            ids, weights, values, format, scorer, position, all_weights, term, terminfo
+        )
         self._skiplist = SkipList(ids)
 
+    def copy(self):
+        return self.__class__(
+            self._ids,
+            self._weights,
+            self._values,
+            self._format,
+            self._scorer,
+            self._i,
+            self._all_weights,
+        )
+
     def skip_to(self, id):
         if not self.is_active():
             raise ReadTooFar
@@ -586,10 +611,11 @@ def skip_to(self, id):
 
         node = self._skiplist.skip_to(id)
         if node is not None:
-           self._i = bisect_left(self._ids, node.doc_id, self._i)
+            self._i = bisect_left(self._ids, node.doc_id, self._i)
         else:
             self._i = len(self._ids)
 
+
 # Term/vector leaf posting matcher middleware
 
 

src/whoosh/matching/skiplist.py

@@ -1,5 +1,8 @@
 import random
 
+_rng = random.Random(42)
+
+
 class SkipNode:
     __slots__ = ("doc_id", "forward")
 
@@ -18,7 +21,7 @@ def __init__(self, ids, max_level=16, p = 0.5):
 
     def _random_level(self):
         level = 0
-        while random.random() < self.p and level < self.max_level:
+        while _rng.random() < self.p and level < self.max_level:
             level += 1
         return level
 
@@ -60,4 +63,3 @@ def __iter__(self):
     def __contains__(self, doc_id):
         node = self.skip_to(doc_id)
         return node is not None and node.doc_id == doc_id
-    

src/whoosh/query/qcore.py

@@ -735,4 +735,4 @@ def matcher(self, searcher, context=None):
                 doclist.update(pr.all_ids())
             doclist = sorted(doclist)
 
-        return matching.ListMatcher(doclist, all_weights=self.boost)
+        return matching.SkipListMatcher(doclist, all_weights=self.boost)

src/whoosh/query/wrappers.py

@@ -178,7 +178,7 @@ def matcher(self, searcher, context=None):
             return m
         else:
             ids = array("I", m.all_ids())
-            return matching.ListMatcher(ids, all_weights=self.score, term=m.term())
+            return matching.SkipListMatcher(ids, all_weights=self.score, term=m.term())
 
 
 class WeightingQuery(WrappingQuery):