Skip to content

Commit ae14176

Browse files
authored
Merge pull request #65 from filecoin-project/fix/clean-child-malformed
Fix malformed hamt after clean child and test
2 parents 5b4fdce + b9ae094 commit ae14176

File tree

4 files changed

+93
-4
lines changed

4 files changed

+93
-4
lines changed

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ require (
44
github.com/ipfs/go-block-format v0.0.2
55
github.com/ipfs/go-cid v0.0.6
66
github.com/ipfs/go-ipld-cbor v0.0.4
7-
github.com/ipfs/go-ipld-format v0.0.2 // indirect
87
github.com/spaolacci/murmur3 v1.1.0
8+
github.com/stretchr/testify v1.6.1
99
github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488
1010
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
1111
)

go.sum

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
2+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
13
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
24
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
35
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -8,7 +10,6 @@ github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmv
810
github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE=
911
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
1012
github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
11-
github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
1213
github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms=
1314
github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
1415
github.com/ipfs/go-cid v0.0.6 h1:go0y+GcDOGeJIV01FeBsta4FHngoA4Wz7KMeLkXAhMs=
@@ -19,8 +20,6 @@ github.com/ipfs/go-ipld-cbor v0.0.4 h1:Aw3KPOKXjvrm6VjwJvFf1F1ekR/BH3jdof3Bk7OTi
1920
github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4=
2021
github.com/ipfs/go-ipld-format v0.0.1 h1:HCu4eB/Gh+KD/Q0M8u888RFkorTWNIL3da4oc5dwc80=
2122
github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms=
22-
github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2G9Zs=
23-
github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k=
2423
github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
2524
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
2625
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
@@ -51,6 +50,8 @@ github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/B
5150
github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
5251
github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
5352
github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
53+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
54+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
5455
github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 h1:bzMe+2coZJYHnhGgVlcQKuRy4FSny4ds8dLQjw5P1XE=
5556
github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
5657
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
@@ -59,6 +60,10 @@ github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifz
5960
github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
6061
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
6162
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
63+
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
64+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
65+
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
66+
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
6267
github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc=
6368
github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
6469
github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 h1:WXhVOwj2USAXB5oMDwRl3piOux2XMV9TANaYxXHdkoE=
@@ -80,3 +85,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
8085
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
8186
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
8287
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
88+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
89+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
90+
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
91+
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

hamt.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"context"
66
"fmt"
77
"math/big"
8+
"sort"
89

910
cid "github.com/ipfs/go-cid"
1011
cbor "github.com/ipfs/go-ipld-cbor"
@@ -564,6 +565,13 @@ func (n *Node) cleanChild(chnd *Node, cindex byte) error {
564565
for _, p := range chnd.Pointers {
565566
chvals = append(chvals, p.KVs...)
566567
}
568+
kvLess := func(i, j int) bool {
569+
ki := chvals[i].Key
570+
kj := chvals[j].Key
571+
return bytes.Compare(ki, kj) < 0
572+
}
573+
sort.Slice(chvals, kvLess)
574+
567575
return n.setPointer(cindex, &Pointer{KVs: chvals})
568576
}
569577

hamt_test.go

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bytes"
55
"context"
66
"crypto/sha256"
7+
"encoding/binary"
78
"encoding/hex"
89
"fmt"
910
"math/rand"
@@ -15,6 +16,8 @@ import (
1516
block "github.com/ipfs/go-block-format"
1617
cid "github.com/ipfs/go-cid"
1718
cbor "github.com/ipfs/go-ipld-cbor"
19+
"github.com/stretchr/testify/assert"
20+
"github.com/stretchr/testify/require"
1821
cbg "github.com/whyrusleeping/cbor-gen"
1922
)
2023

@@ -1236,3 +1239,72 @@ func TestMalformedHamt(t *testing.T) {
12361239
t.Fatal("Should have returned found entry")
12371240
}
12381241
}
1242+
1243+
func TestCleanChildOrdering(t *testing.T) {
1244+
// This test originates from a case hit while testing filecoin-project/specs-actors.
1245+
// TODO a HAMT exercising this case can very likely be constructed with many fewer
1246+
// operations. I'm duplicating the full original test for expedience
1247+
//
1248+
// The important part of this HAMT is that at some point child with index 20 looks like:
1249+
// P0 -- []KV{KV{Key: 0x01a0, Value: v0}}
1250+
// P1 -- []KV{KV{Key: 0x01a8, Value: v1}}
1251+
// P2 -- []KV{KV{Key: 0x0181, Value: v2}}
1252+
// P3 -- []KV{KV{Key: 0x006c, Value: v2}}
1253+
//
1254+
// We then delete 0x006c. This forces this child node into a bucket.
1255+
// before writing this test cleanChild did not explicitly sort KVs from
1256+
// all pointers, so the new bucket looked like:
1257+
// []KV{
1258+
// KV{Key: 0x01a0, Value: v1},
1259+
// KV{Key: 0x01a8, Value: v2},
1260+
// KV{Key: 0x0181, Value: v2},
1261+
// }
1262+
//
1263+
// This violated the buckets-are-sorted-by-key condition
1264+
1265+
// Construct HAMT
1266+
makeKey := func(i uint64) string {
1267+
buf := make([]byte, 10)
1268+
n := binary.PutUvarint(buf, i)
1269+
return string(buf[:n])
1270+
}
1271+
dummyValue := []byte{0xaa, 0xbb, 0xcc, 0xdd}
1272+
1273+
ctx := context.Background()
1274+
cs := cbor.NewCborStore(newMockBlocks())
1275+
hamtOptions := []Option{
1276+
UseTreeBitWidth(5),
1277+
UseHashFunction(func(input []byte) []byte {
1278+
res := sha256.Sum256(input)
1279+
return res[:]
1280+
}),
1281+
}
1282+
1283+
h := NewNode(cs, hamtOptions...)
1284+
1285+
for i := uint64(100); i < uint64(195); i++ {
1286+
err := h.Set(ctx, makeKey(i), dummyValue)
1287+
require.NoError(t, err)
1288+
}
1289+
1290+
// Shouldn't matter but repeating original case exactly
1291+
require.NoError(t, h.Flush(ctx))
1292+
root, err := cs.Put(ctx, h)
1293+
require.NoError(t, err)
1294+
h, err = LoadNode(ctx, cs, root, hamtOptions...)
1295+
require.NoError(t, err)
1296+
1297+
// Delete key 104 so child indexed at 20 has four pointers
1298+
err = h.Delete(ctx, makeKey(104))
1299+
assert.NoError(t, err)
1300+
err = h.Delete(ctx, makeKey(108))
1301+
assert.NoError(t, err)
1302+
err = h.Flush(ctx)
1303+
assert.NoError(t, err)
1304+
root, err = cs.Put(ctx, h)
1305+
1306+
// Reload without error
1307+
require.NoError(t, err)
1308+
h, err = LoadNode(ctx, cs, root, hamtOptions...)
1309+
assert.NoError(t, err)
1310+
}

0 commit comments

Comments
 (0)