Skip to content

Commit afb709e

Browse files
antas-marcinantas-marcin
authored
Merge pull request #21 from weaviate/optimizations
optimizations: inline And/AndNot array merges, removed empty container
2 parents 0574c03 + 85bc232 commit afb709e

File tree

6 files changed

+567
-358
lines changed

6 files changed

+567
-358
lines changed

‎bitmap.go‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ import (
2626
"github.com/pkg/errors"
2727
)
2828

29-
var empty = make([]uint16, 16<<20)
30-
3129
const mask = uint64(0xFFFFFFFFFFFF0000)
3230

3331
type Bitmap struct {
@@ -51,7 +49,7 @@ func FromBuffer(data []byte) *Bitmap {
5149
if len(data) < 8 {
5250
return NewBitmap()
5351
}
54-
du := toUint16Slice(data)
52+
du := byteTo16SliceUnsafe(data)
5553
x := toUint64Slice(du[:4])[indexNodeSize]
5654
return &Bitmap{
5755
data: du,
@@ -67,7 +65,7 @@ func FromBufferWithCopy(src []byte) *Bitmap {
6765
if len(src) < 8 {
6866
return NewBitmap()
6967
}
70-
src16 := toUint16Slice(src)
68+
src16 := byteTo16SliceUnsafe(src)
7169
dst16 := make([]uint16, len(src16))
7270
copy(dst16, src16)
7371
x := toUint64Slice(dst16[:4])[indexNodeSize]
@@ -180,7 +178,7 @@ func (ra *Bitmap) expandKeys(bySize uint64) uint64 {
180178
ra.scootRight(curSize, bySize)
181179
ra.keys = uint16To64SliceUnsafe(ra.data[:curSize+bySize])
182180
ra.keys.setNodeSize(int(curSize + bySize))
183-
181+
184182
// All containers have moved to the right by bySize bytes.
185183
// Update their offsets.
186184
n := ra.keys
@@ -202,7 +200,6 @@ func (ra *Bitmap) expandNoLengthChange(bySize uint64) (toSize int) {
202200
// This following statement also works. But, given how much fastExpand gets
203201
// called (a lot), probably better to control allocation.
204202
// ra.data = append(ra.data, empty[:bySize]...)
205-
206203
toSize = len(ra.data) + int(bySize)
207204
if toSize <= cap(ra.data) {
208205
return

‎bitmap_opt.go‎

Lines changed: 81 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ func andNotContainers(a, b, res *Bitmap, optBuf []uint16) {
131131

132132
akToAc := map[uint64][]uint16{}
133133
sizeContainers := uint64(0)
134-
sizeKeys := uint64(0)
134+
newKeys := 0
135135

136136
for ai < an && bi < bn {
137137
ak := a.keys.key(ai)
@@ -155,7 +155,7 @@ func andNotContainers(a, b, res *Bitmap, optBuf []uint16) {
155155
if getCardinality(ac) > 0 {
156156
akToAc[ak] = ac
157157
sizeContainers += uint64(len(ac))
158-
sizeKeys += 8 // 2x uint64 = 8x uint16; for key and offset
158+
newKeys++
159159
}
160160
ai++
161161
} else {
@@ -169,11 +169,13 @@ func andNotContainers(a, b, res *Bitmap, optBuf []uint16) {
169169
ak := a.keys.key(ai)
170170
akToAc[ak] = ac
171171
sizeContainers += uint64(len(ac))
172-
sizeKeys += 8 // 2x uint64 = 8x uint16; for key and offset
172+
newKeys++
173173
}
174174
}
175175

176176
if sizeContainers > 0 {
177+
// 2x uint64 = 8x uint16; for key and offset; twice as much as actually needed
178+
sizeKeys := 8 * uint64(newKeys) * 2
177179
// ensure enough space for new containers and keys,
178180
// allocate required memory just once to avoid copying underlying data slice multiple times
179181
res.expandNoLengthChange(sizeContainers + sizeKeys)
@@ -267,7 +269,7 @@ func orContainers(a, b, res *Bitmap, buf []uint16) {
267269
akToAc := map[uint64][]uint16{}
268270
bkToBc := map[uint64][]uint16{}
269271
sizeContainers := uint64(0)
270-
sizeKeys := uint64(0)
272+
newKeys := 0
271273

272274
for ai < an && bi < bn {
273275
ak := a.keys.key(ai)
@@ -295,7 +297,7 @@ func orContainers(a, b, res *Bitmap, buf []uint16) {
295297
if getCardinality(ac) > 0 {
296298
akToAc[ak] = ac
297299
sizeContainers += uint64(len(ac))
298-
sizeKeys += 8 // 2x uint64 = 8x uint16; for key and offset
300+
newKeys++
299301
}
300302
ai++
301303
} else {
@@ -304,7 +306,7 @@ func orContainers(a, b, res *Bitmap, buf []uint16) {
304306
if getCardinality(bc) > 0 {
305307
bkToBc[bk] = bc
306308
sizeContainers += uint64(len(bc))
307-
sizeKeys += 8 // 2x uint64 = 8x uint16; for key and offset
309+
newKeys++
308310
}
309311
bi++
310312
}
@@ -316,7 +318,7 @@ func orContainers(a, b, res *Bitmap, buf []uint16) {
316318
ak := a.keys.key(ai)
317319
akToAc[ak] = ac
318320
sizeContainers += uint64(len(ac))
319-
sizeKeys += 8 // 2x uint64 = 8x uint16; for key and offset
321+
newKeys++
320322
}
321323
}
322324
for ; bi < bn; bi++ {
@@ -326,11 +328,13 @@ func orContainers(a, b, res *Bitmap, buf []uint16) {
326328
bk := b.keys.key(bi)
327329
bkToBc[bk] = bc
328330
sizeContainers += uint64(len(bc))
329-
sizeKeys += 8 // 2x uint64 = 8x uint16; for key and offset
331+
newKeys++
330332
}
331333
}
332334

333335
if sizeContainers > 0 {
336+
// 2x uint64 = 8x uint16; for key and offset; twice as much as actually needed
337+
sizeKeys := 8 * uint64(newKeys) * 2
334338
// ensure enough space for new containers and keys,
335339
// allocate required memory just once avoid copying underlying data slice multiple times
336340
res.expandNoLengthChange(sizeContainers + sizeKeys)
@@ -370,7 +374,7 @@ func orContainersInRange(a, b *Bitmap, bi, bn int, buf []uint16) {
370374
// expanding underlying data slice and keys subslice once
371375
bkToBc := map[uint64][]uint16{}
372376
sizeContainers := uint64(0)
373-
sizeKeys := uint64(0)
377+
newKeys := 0
374378

375379
for ai < an && bi < bn {
376380
ak := a.keys.key(ai)
@@ -410,7 +414,7 @@ func orContainersInRange(a, b *Bitmap, bi, bn int, buf []uint16) {
410414
if getCardinality(bc) > 0 {
411415
bkToBc[bk] = bc
412416
sizeContainers += uint64(len(bc))
413-
sizeKeys += 8 // 2x uint64 = 8x uint16; for key and offset
417+
newKeys++
414418
}
415419
bi++
416420
}
@@ -422,15 +426,22 @@ func orContainersInRange(a, b *Bitmap, bi, bn int, buf []uint16) {
422426
bk := b.keys.key(bi)
423427
bkToBc[bk] = bc
424428
sizeContainers += uint64(len(bc))
425-
sizeKeys += 8 // 2x uint64 = 8x uint16; for key and offset
429+
newKeys++
426430
}
427431
}
428432

429433
if sizeContainers > 0 {
430-
// ensure enough space for new containers and keys,
431-
// allocate required memory just once to avoid copying underlying data slice multiple times
432-
a.expandNoLengthChange(sizeContainers + sizeKeys)
433-
a.expandKeys(sizeKeys)
434+
if a.keysExpansionRequired(newKeys) {
435+
// 2x uint64 = 8x uint16; for key and offset; twice as much as actually needed
436+
sizeKeys := 8 * uint64(newKeys) * 2
437+
438+
// ensure enough space for new containers and keys,
439+
// allocate required memory just once to avoid copying underlying data slice multiple times
440+
a.expandNoLengthChange(sizeContainers + sizeKeys)
441+
a.expandKeys(sizeKeys)
442+
} else {
443+
a.expandNoLengthChange(sizeContainers)
444+
}
434445

435446
for bk, bc := range bkToBc {
436447
// create a new container and update the key offset to this container.
@@ -465,29 +476,36 @@ func (ra *Bitmap) OrConc(bm *Bitmap, maxConcurrency int) *Bitmap {
465476
return ra
466477
}
467478

468-
var totalSizeContainers, totalSizeKeys uint64
479+
var totalSizeContainers uint64
480+
var totalNewKeys int
469481
var allKeys []uint64
470482
var allContainers [][]uint16
471483
lock := new(sync.Mutex)
472484
inlineVsMutateLock := new(sync.RWMutex)
473485
callback := func(bi, bj, _ int) {
474486
buf := make([]uint16, maxContainerSize)
475-
sizeContainers, sizeKeys, keys, containers := orContainersInRangeConc(ra, bm, bi, bj, buf, inlineVsMutateLock)
487+
sizeContainers, newKeys, keys, containers := orContainersInRangeConc(ra, bm, bi, bj, buf, inlineVsMutateLock)
476488

477489
lock.Lock()
478490
totalSizeContainers += sizeContainers
479-
totalSizeKeys += sizeKeys
491+
totalNewKeys += newKeys
480492
allKeys = append(allKeys, keys...)
481493
allContainers = append(allContainers, containers...)
482494
lock.Unlock()
483495
}
484496
concurrentlyInRanges(numContainers, concurrency, callback)
485-
486497
if totalSizeContainers > 0 {
487-
// ensure enough space for new containers and keys,
488-
// allocate required memory just once to avoid copying underlying data slice multiple times
489-
ra.expandNoLengthChange(totalSizeContainers + totalSizeKeys)
490-
ra.expandKeys(totalSizeKeys)
498+
if ra.keysExpansionRequired(totalNewKeys) {
499+
// 2x uint64 = 8x uint16; for key and offset; twice as much as actually needed
500+
totalSizeKeys := 8 * uint64(totalNewKeys) * 2
501+
502+
// ensure enough space for new containers and keys,
503+
// allocate required memory just once to avoid copying underlying data slice multiple times
504+
ra.expandNoLengthChange(totalSizeContainers + totalSizeKeys)
505+
ra.expandKeys(totalSizeKeys)
506+
} else {
507+
ra.expandNoLengthChange(totalSizeContainers)
508+
}
491509

492510
for i, container := range allContainers {
493511
// create a new container and update the key offset to this container.
@@ -501,15 +519,15 @@ func (ra *Bitmap) OrConc(bm *Bitmap, maxConcurrency int) *Bitmap {
501519
}
502520

503521
func orContainersInRangeConc(a, b *Bitmap, bi, bn int, buf []uint16, inlineVsMutateLock *sync.RWMutex,
504-
) (sizeContainers, sizeKeys uint64, bKeys []uint64, bContainers [][]uint16) {
522+
) (sizeContainers uint64, newKeys int, bKeys []uint64, bContainers [][]uint16) {
505523
bk := b.keys.key(bi)
506524
ai := a.keys.search(bk)
507525
an := a.keys.numKeys()
508526

509527
// copy containers from b to a all at once
510528
// expanding underlying data slice and keys subslice once
511529
sizeContainers = 0
512-
sizeKeys = 0
530+
newKeys = 0
513531
bKeys = []uint64{}
514532
bContainers = [][]uint16{}
515533

@@ -558,7 +576,7 @@ func orContainersInRangeConc(a, b *Bitmap, bi, bn int, buf []uint16, inlineVsMut
558576
bKeys = append(bKeys, bk)
559577
bContainers = append(bContainers, bc)
560578
sizeContainers += uint64(len(bc))
561-
sizeKeys += 8 // 2x uint64 = 8x uint16; for key and offset
579+
newKeys++
562580
}
563581
bi++
564582
}
@@ -571,7 +589,7 @@ func orContainersInRangeConc(a, b *Bitmap, bi, bn int, buf []uint16, inlineVsMut
571589
bKeys = append(bKeys, bk)
572590
bContainers = append(bContainers, bc)
573591
sizeContainers += uint64(len(bc))
574-
sizeKeys += 8 // 2x uint64 = 8x uint16; for key and offset
592+
newKeys++
575593
}
576594
}
577595

@@ -696,10 +714,33 @@ func (ra *Bitmap) CloneToBuf(buf []byte) *Bitmap {
696714
return bm
697715
}
698716

717+
// FromBufferUnlimited returns a pointer to bitmap corresponding to the given buffer.
718+
// Entire buffer capacity is utlized for future bitmap modifications and expansions.
719+
func FromBufferUnlimited(buf []byte) *Bitmap {
720+
ln := len(buf)
721+
assert(ln%2 == 0)
722+
if len(buf) < 8 {
723+
return NewBitmap()
724+
}
725+
726+
cp := cap(buf)
727+
data := buf[:cp]
728+
if cp%2 != 0 {
729+
data = buf[:cp-1]
730+
}
731+
732+
du := byteTo16SliceUnsafe(data)
733+
x := uint16To64SliceUnsafe(du[:4])[indexNodeSize]
734+
return &Bitmap{
735+
data: du[:ln/2],
736+
_ptr: buf, // Keep a hold of data, otherwise GC would do its thing.
737+
keys: uint16To64SliceUnsafe(du[:x]),
738+
}
739+
}
740+
699741
// Prefill creates bitmap prefilled with elements [0-maxX]
700742
func Prefill(maxX uint64) *Bitmap {
701743
containersCount, remainingCount := calcFullContainersAndRemainingCounts(maxX)
702-
703744
// create additional container for remaining values
704745
// (or reserve space for new one if there are not any remaining)
705746
// +1 additional key to avoid keys expanding (there should always be 1 spare)
@@ -848,9 +889,14 @@ func (ra *Bitmap) FillUp(maxX uint64) {
848889

849890
// calculate required memory to allocate and expand underlying slice
850891
containersLen := uint64(requiredContainersCount * maxContainerSize)
851-
keysLen := uint64(requiredContainersCount * 2 * 4)
852-
ra.expandNoLengthChange(containersLen + keysLen)
853-
ra.expandKeys(keysLen)
892+
if ra.keysExpansionRequired(requiredContainersCount) {
893+
// 2x uint64 = 8x uint16; for key and offset; twice as much as actually needed
894+
keysLen := uint64(requiredContainersCount*2*4) * 2
895+
ra.expandNoLengthChange(containersLen + keysLen)
896+
ra.expandKeys(keysLen)
897+
} else {
898+
ra.expandNoLengthChange(containersLen)
899+
}
854900

855901
var onesBitmap bitmap
856902
if containerIdx < maxContainersCount {
@@ -996,3 +1042,7 @@ func (b bitmap) fillWithOnes() {
9961042
b64[i] = math.MaxUint64
9971043
}
9981044
}
1045+
1046+
func (ra *Bitmap) keysExpansionRequired(newKeys int) bool {
1047+
return ra.keys.numKeys()+newKeys >= ra.keys.maxKeys()
1048+
}

0 commit comments

Comments
 (0)