Fixed CompressedBin algo, updated heuristic based on benchmarks

Author: darkcoderrises

algo/uidlist.go

@@ -60,7 +60,7 @@ func IntersectCompressedWith(pack *pb.UidPack, afterUID uint64, v, o *pb.List) {
 
 	// Select appropriate function based on heuristics.
 	ratio := float64(m) / float64(n)
-	if ratio < 500 {
+	if ratio < 100 {
 		IntersectCompressedWithLinJump(&dec, v.Uids, &dst)
 	} else {
 		IntersectCompressedWithBin(&dec, v.Uids, &dst)
@@ -94,7 +94,7 @@ func IntersectCompressedWithLinJump(dec *codec.Decoder, v []uint64, o *[]uint64)
 // https://link.springer.com/chapter/10.1007/978-3-642-12476-1_3
 // Call seek on dec before calling this function
 func IntersectCompressedWithBin(dec *codec.Decoder, q []uint64, o *[]uint64) {
-	ld := dec.ApproxLen()
+	ld := codec.ExactLen(dec.Pack)
 	lq := len(q)
 
 	if lq == 0 {
@@ -105,13 +105,19 @@ func IntersectCompressedWithBin(dec *codec.Decoder, q []uint64, o *[]uint64) {
 	}
 
 	// Pick the shorter list and do binary search
-	if ld < lq {
+	if ld <= lq {
 		for {
 			blockUids := dec.Uids()
 			if len(blockUids) == 0 {
 				break
 			}
-			IntersectWithBin(blockUids, q, o)
+			if ld*10 < lq {
+				IntersectWithBin(blockUids, q, o)
+			} else {
+				// For small enough difference between two arrays, we should just
+				// do lin intersect
+				IntersectWithLin(blockUids, q, o)
+			}
 			lastUid := blockUids[len(blockUids)-1]
 			qidx := sort.Search(len(q), func(idx int) bool {
 				return q[idx] >= lastUid
@@ -125,26 +131,29 @@ func IntersectCompressedWithBin(dec *codec.Decoder, q []uint64, o *[]uint64) {
 		return
 	}
 
-	var uids []uint64
-	for _, u := range q {
+	uids := dec.Uids()
+	qidx := -1
+	for {
+		qidx += 1
+		if qidx >= len(q) {
+			return
+		}
+		u := q[qidx]
 		if len(uids) == 0 || u > uids[len(uids)-1] {
-			uids = dec.Seek(u, codec.SeekStart)
+			if lq*10 < ld {
+				uids = dec.LinearSeek(u)
+			} else {
+				uids = dec.SeekToBlock(u, codec.SeekStart)
+			}
 			if len(uids) == 0 {
 				return
 			}
 		}
-		uidIdx := sort.Search(len(uids), func(idx int) bool {
-			return uids[idx] >= u
-		})
-		if uidIdx >= len(uids) {
-			// We know that u < max(uids). If we didn't find it here, it's not here.
-			continue
-		}
-		if uids[uidIdx] == u {
-			*o = append(*o, u)
-			uidIdx++
+		_, off := IntersectWithJump(uids, q[qidx:], o)
+		if off == 0 {
+			off = 1 // if v[k] isn't in u, move forward
 		}
-		uids = uids[uidIdx:]
+		qidx += off
 	}
 }
 

algo/uidlist_test.go

@@ -373,7 +373,7 @@ func BenchmarkListIntersectCompressBin(b *testing.B) {
 		for _, r := range rs {
 			sz1 := sz
 			sz2 := int(float64(sz) * r)
-			if sz2 > 1000000 || sz2 == 0 {
+			if sz2 > 10000000 || sz2 == 0 {
 				break
 			}
 

codec/codec.go

@@ -223,6 +223,59 @@ func (d *Decoder) ApproxLen() int {
 
 type searchFunc func(int) bool
 
+// SeekToBlock will find the nearest block, and unpack it. Unlike Seek, it doesn't
+// apply search in the resulting uid list and then move the pointer forward. When we are going
+// to intersect the list later, this function is useful.
+func (d *Decoder) SeekToBlock(uid uint64, whence seekPos) []uint64 {
+	if d.Pack == nil {
+		return []uint64{}
+	}
+	prevBlockIdx := d.blockIdx
+	d.blockIdx = 0
+	if uid == 0 {
+		return d.UnpackBlock()
+	}
+
+	pack := d.Pack
+	blocksFunc := func() searchFunc {
+		var f searchFunc
+		switch whence {
+		case SeekStart:
+			f = func(i int) bool { return pack.Blocks[i+prevBlockIdx].Base >= uid }
+		case SeekCurrent:
+			f = func(i int) bool { return pack.Blocks[i+prevBlockIdx].Base > uid }
+		}
+		return f
+	}
+
+	idx := sort.Search(len(pack.Blocks[prevBlockIdx:]), blocksFunc()) + prevBlockIdx
+	// The first block.Base >= uid.
+	if idx == 0 {
+		return d.UnpackBlock()
+	}
+	// The uid is the first entry in the block.
+	if idx < len(pack.Blocks) && pack.Blocks[idx].Base == uid {
+		d.blockIdx = idx
+		return d.UnpackBlock()
+	}
+
+	// Either the idx = len(pack.Blocks) that means it wasn't found in any of the block's base. Or,
+	// we found the first block index whose base is greater than uid. In these cases, go to the
+	// previous block and search there.
+	d.blockIdx = idx - 1 // Move to the previous block. If blockIdx<0, unpack will deal with it.
+	if d.blockIdx != prevBlockIdx {
+		d.UnpackBlock() // And get all their uids.
+	}
+
+	if uid < d.uids[len(d.uids)-1] {
+		return d.uids
+	}
+
+	// Could not find any uid in the block, which is >= uid. The next block might still have valid
+	// entries > uid.
+	return d.Next()
+}
+
 // Seek will search for uid in a packed block using the specified whence position.
 // The value of whence must be one of the predefined values SeekStart or SeekCurrent.
 // SeekStart searches uid and includes it as part of the results.
@@ -233,6 +286,7 @@ func (d *Decoder) Seek(uid uint64, whence seekPos) []uint64 {
 	if d.Pack == nil {
 		return []uint64{}
 	}
+	prevBlockIdx := d.blockIdx
 	d.blockIdx = 0
 	if uid == 0 {
 		return d.UnpackBlock()
@@ -243,14 +297,14 @@ func (d *Decoder) Seek(uid uint64, whence seekPos) []uint64 {
 		var f searchFunc
 		switch whence {
 		case SeekStart:
-			f = func(i int) bool { return pack.Blocks[i].Base >= uid }
+			f = func(i int) bool { return pack.Blocks[i+prevBlockIdx].Base >= uid }
 		case SeekCurrent:
-			f = func(i int) bool { return pack.Blocks[i].Base > uid }
+			f = func(i int) bool { return pack.Blocks[i+prevBlockIdx].Base > uid }
 		}
 		return f
 	}
 
-	idx := sort.Search(len(pack.Blocks), blocksFunc())
+	idx := sort.Search(len(pack.Blocks[prevBlockIdx:]), blocksFunc()) + prevBlockIdx
 	// The first block.Base >= uid.
 	if idx == 0 {
 		return d.UnpackBlock()
@@ -265,7 +319,9 @@ func (d *Decoder) Seek(uid uint64, whence seekPos) []uint64 {
 	// we found the first block index whose base is greater than uid. In these cases, go to the
 	// previous block and search there.
 	d.blockIdx = idx - 1 // Move to the previous block. If blockIdx<0, unpack will deal with it.
-	d.UnpackBlock()      // And get all their uids.
+	if d.blockIdx != prevBlockIdx {
+		d.UnpackBlock() // And get all their uids.
+	}
 
 	uidsFunc := func() searchFunc {
 		var f searchFunc