Skip to content

Commit fd435b7

Browse files
authored
Merge pull request #489 from RoaringBitmap/better_fuzzing
Better fuzzing
2 parents 311bb0e + c43a213 commit fd435b7

File tree

12 files changed

+825
-108
lines changed

12 files changed

+825
-108
lines changed

.github/pull_request_template.md

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
## Description
2+
3+
Please provide a brief description of the changes made in this pull request.
4+
5+
## Type of Change
6+
7+
- [ ] Bug fix (non-breaking change which fixes an issue)
8+
- [ ] New feature (non-breaking change which adds functionality)
9+
- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
10+
- [ ] Performance improvement
11+
- [ ] Code refactoring
12+
- [ ] Documentation update
13+
- [ ] Test improvements
14+
- [ ] Build/CI changes
15+
16+
## Changes Made
17+
18+
### What was changed?
19+
-
20+
21+
### Why was it changed?
22+
-
23+
24+
### How was it changed?
25+
-
26+
27+
## Testing
28+
29+
Please add a unit test if you are fixing a bug.
30+
31+
You must run
32+
33+
```
34+
go test
35+
```
36+
37+
## Formatting
38+
39+
Please run
40+
41+
```
42+
go fmt
43+
```
44+
45+
## Fuzzing
46+
47+
Please run our fuzzer on your changes.
48+
49+
50+
1. Generate initial smat corpus:
51+
```
52+
go test -tags=gofuzz -run=TestGenerateSmatCorpus
53+
```
54+
You should see a directory `workdir` created with initial corpus files.
55+
56+
2. Run the fuzz test:
57+
```
58+
go test -run='^$' -fuzz=FuzzSmat -fuzztime=300s -timeout=60s
59+
```
60+
61+
Adjust `-fuzztime` as needed for longer or shorter runs. If crashes are found,
62+
check the test output and the reproducer files in the `workdir` directory.
63+
You may copy the reproducers to roaring_tests.go
64+
65+
## Performance Impact
66+
67+
If applicable, describe any performance implications of these changes and include benchmark results.
68+
69+
### Running Benchmarks
70+
71+
This project includes comprehensive benchmarks. Please run relevant benchmarks before and after your changes:
72+
73+
#### Basic Benchmarks
74+
```bash
75+
# Run all benchmarks
76+
go test -bench=. -run=^$
77+
78+
# Run with memory allocation statistics
79+
go test -bench=. -benchmem -run=^$
80+
81+
# Run specific benchmark
82+
go test -bench=BenchmarkIteratorAlloc -run=^$
83+
```
84+
85+
#### Parallel Benchmarks
86+
```bash
87+
# Run parallel processing benchmarks
88+
go test -bench=BenchmarkIntersectionLargeParallel -run=^$
89+
```
90+
91+
#### Real Data Benchmarks
92+
```bash
93+
# Requires real-roaring-datasets (run: go get github.com/RoaringBitmap/real-roaring-datasets)
94+
BENCH_REAL_DATA=1 go test -bench=BenchmarkRealData -run=^$
95+
```
96+
97+
#### Benchmark Results Format
98+
Please include before/after results in the following format:
99+
```
100+
BenchmarkName-8 1000000 1234 ns/op 567 B/op 12 allocs/op
101+
```
102+
103+
### Performance Analysis
104+
- Compare benchmark results before and after changes
105+
- Note any significant improvements or regressions
106+
- Include memory usage changes if relevant
107+
- Mention any trade-offs (e.g., memory vs speed)
108+
109+
## Breaking Changes
110+
111+
If this PR introduces breaking changes, please describe them and the migration path:
112+
-
113+
114+
115+
## Related Issues
116+
117+
Fixes # (issue number)
118+
Related to # (issue number)
119+
120+
## Additional Notes
121+
122+
Any additional information or context about this pull request.

arraycontainer.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ type arrayContainer struct {
1111

1212
var (
1313
ErrArrayIncorrectSort = errors.New("incorrectly sorted array")
14+
ErrEmptyArray = errors.New("empty array")
1415
ErrArrayInvalidSize = errors.New("invalid array size")
1516
)
1617

@@ -1291,7 +1292,7 @@ func (ac *arrayContainer) validate() error {
12911292
cardinality := ac.getCardinality()
12921293

12931294
if cardinality <= 0 {
1294-
return ErrArrayInvalidSize
1295+
return ErrEmptyArray
12951296
}
12961297

12971298
if cardinality > arrayDefaultMaxSize {

go.mod

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
module github.com/RoaringBitmap/roaring/v2
22

3-
go 1.22
3+
go 1.24.0
4+
5+
toolchain go1.24.4
46

57
require (
68
github.com/bits-and-blooms/bitset v1.24.1
@@ -11,6 +13,12 @@ require (
1113

1214
require (
1315
github.com/davecgh/go-spew v1.1.1 // indirect
16+
github.com/dvyukov/go-fuzz v0.0.0-20240924070022-e577bee5275c // indirect
17+
github.com/elazarl/go-bindata-assetfs v1.0.1 // indirect
1418
github.com/pmezard/go-difflib v1.0.0 // indirect
19+
github.com/stephens2424/writerset v1.0.2 // indirect
20+
golang.org/x/mod v0.29.0 // indirect
21+
golang.org/x/sync v0.17.0 // indirect
22+
golang.org/x/tools v0.38.0 // indirect
1523
gopkg.in/yaml.v3 v3.0.1 // indirect
1624
)

go.sum

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,32 @@
1+
github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
12
github.com/bits-and-blooms/bitset v1.24.1 h1:hqnfFbjjk3pxGa5E9Ho3hjoU7odtUuNmJ9Ao+Bo8s1c=
23
github.com/bits-and-blooms/bitset v1.24.1/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
4+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
35
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
46
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
7+
github.com/dvyukov/go-fuzz v0.0.0-20240924070022-e577bee5275c h1:oLpHpHwNuAPvw3bBviEZNrJbigNNi5dRadfZnagGgZI=
8+
github.com/dvyukov/go-fuzz v0.0.0-20240924070022-e577bee5275c/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
9+
github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw=
10+
github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
511
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
612
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
713
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
814
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
915
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
1016
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
17+
github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s=
18+
github.com/stephens2424/writerset v1.0.2 h1:znRLgU6g8RS5euYRcy004XeE4W+Tu44kALzy7ghPif8=
19+
github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc=
20+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
21+
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
1122
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
1223
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
24+
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
25+
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
26+
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
27+
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
28+
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
29+
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
1330
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
1431
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
1532
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

iter123_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,11 @@ func TestBackward123(t *testing.T) {
8383

8484
t.Run("#6", func(t *testing.T) {
8585
b := New()
86-
b.AddInt(MaxUint32)
86+
b.Add(MaxUint32)
8787

8888
// only one value MaxUint32
8989
for val := range Backward(b) {
90-
assert.EqualValues(t, MaxUint32, val)
90+
assert.EqualValues(t, uint32(MaxUint32), val)
9191
}
9292
})
9393
}

roaring.go

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1302,6 +1302,10 @@ main:
13021302

13031303
// Xor computes the symmetric difference between two bitmaps and stores the result in the current bitmap
13041304
func (rb *Bitmap) Xor(x2 *Bitmap) {
1305+
if rb == x2 {
1306+
rb.Clear()
1307+
return
1308+
}
13051309
pos1 := 0
13061310
pos2 := 0
13071311
length1 := rb.highlowcontainer.size()
@@ -1316,8 +1320,7 @@ func (rb *Bitmap) Xor(x2 *Bitmap) {
13161320
break
13171321
}
13181322
} else if s1 > s2 {
1319-
c := x2.highlowcontainer.getWritableContainerAtIndex(pos2)
1320-
rb.highlowcontainer.insertNewKeyValueAt(pos1, x2.highlowcontainer.getKeyAtIndex(pos2), c)
1323+
rb.highlowcontainer.insertNewKeyValueAt(pos1, x2.highlowcontainer.getKeyAtIndex(pos2), x2.highlowcontainer.getContainerAtIndex(pos2).clone())
13211324
length1++
13221325
pos1++
13231326
pos2++
@@ -1370,7 +1373,8 @@ main:
13701373
}
13711374
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
13721375
} else {
1373-
rb.highlowcontainer.replaceKeyAndContainerAtIndex(pos1, s1, rb.highlowcontainer.getUnionedWritableContainer(pos1, x2.highlowcontainer.getContainerAtIndex(pos2)), false)
1376+
newcont := rb.highlowcontainer.getUnionedWritableContainer(pos1, x2.highlowcontainer.getContainerAtIndex(pos2))
1377+
rb.highlowcontainer.replaceKeyAndContainerAtIndex(pos1, s1, newcont, false)
13741378
pos1++
13751379
pos2++
13761380
if (pos1 == length1) || (pos2 == length2) {
@@ -1388,6 +1392,10 @@ main:
13881392

13891393
// AndNot computes the difference between two bitmaps and stores the result in the current bitmap
13901394
func (rb *Bitmap) AndNot(x2 *Bitmap) {
1395+
if rb == x2 {
1396+
rb.Clear()
1397+
return
1398+
}
13911399
pos1 := 0
13921400
pos2 := 0
13931401
intersectionsize := 0
@@ -1543,6 +1551,9 @@ main:
15431551

15441552
// Xor computes the symmetric difference between two bitmaps and returns the result
15451553
func Xor(x1, x2 *Bitmap) *Bitmap {
1554+
if x1 == x2 {
1555+
return NewBitmap()
1556+
}
15461557
answer := NewBitmap()
15471558
pos1 := 0
15481559
pos2 := 0
@@ -1580,6 +1591,9 @@ func Xor(x1, x2 *Bitmap) *Bitmap {
15801591

15811592
// AndNot computes the difference between two bitmaps and returns the result
15821593
func AndNot(x1, x2 *Bitmap) *Bitmap {
1594+
if x1 == x2 {
1595+
return NewBitmap()
1596+
}
15831597
answer := NewBitmap()
15841598
pos1 := 0
15851599
pos2 := 0
@@ -2145,6 +2159,34 @@ func (rb *Bitmap) Stats() Statistics {
21452159
return stats
21462160
}
21472161

2162+
// Describe prints a description of the bitmap's containers to stdout
2163+
func (rb *Bitmap) Describe() {
2164+
fmt.Printf("Bitmap with %d containers:\n", len(rb.highlowcontainer.containers))
2165+
for i, c := range rb.highlowcontainer.containers {
2166+
key := rb.highlowcontainer.keys[i]
2167+
shared := ""
2168+
if rb.highlowcontainer.needCopyOnWrite[i] {
2169+
shared = " (shared)"
2170+
}
2171+
switch c.(type) {
2172+
case *arrayContainer:
2173+
fmt.Printf(" Container %d (key %d): array, cardinality %d%s\n", i, key, c.getCardinality(), shared)
2174+
case *bitmapContainer:
2175+
fmt.Printf(" Container %d (key %d): bitmap, cardinality %d%s\n", i, key, c.getCardinality(), shared)
2176+
case *runContainer16:
2177+
fmt.Printf(" Container %d (key %d): run, cardinality %d%s\n", i, key, c.getCardinality(), shared)
2178+
default:
2179+
fmt.Printf(" Container %d (key %d): unknown type, cardinality %d%s\n", i, key, c.getCardinality(), shared)
2180+
}
2181+
}
2182+
valid := rb.Validate()
2183+
if valid != nil {
2184+
fmt.Printf(" Bitmap is INVALID: %v\n", valid)
2185+
} else {
2186+
fmt.Printf(" Bitmap is valid\n")
2187+
}
2188+
}
2189+
21482190
// Validate checks if the bitmap is internally consistent.
21492191
// You may call it after deserialization to check that the bitmap is valid.
21502192
// This function returns an error if the bitmap is invalid, nil otherwise.

0 commit comments

Comments
 (0)