diff --git a/go.mod b/go.mod index 6390882..e99d54f 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,10 @@ module github.com/ipfs/go-ipld-cbor require ( + github.com/ipfs/boxo v0.13.1 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 + github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ipfs-util v0.0.2 github.com/ipfs/go-ipld-format v0.5.0 github.com/multiformats/go-multihash v0.2.3 @@ -11,18 +13,32 @@ require ( ) require ( - github.com/klauspost/cpuid/v2 v2.0.9 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect + github.com/ipfs/bbloom v0.0.4 // indirect + github.com/ipfs/go-detect-race v0.0.1 // indirect + github.com/ipfs/go-log/v2 v2.5.1 // indirect + github.com/ipfs/go-metrics-interface v0.0.1 // indirect + github.com/jbenet/goprocess v0.1.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect - github.com/multiformats/go-base32 v0.0.3 // indirect - github.com/multiformats/go-base36 v0.1.0 // indirect - github.com/multiformats/go-multibase v0.0.3 // indirect - github.com/multiformats/go-varint v0.0.6 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - golang.org/x/crypto v0.1.0 // indirect - golang.org/x/sys v0.1.0 // indirect - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect - lukechampine.com/blake3 v1.1.6 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.25.0 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + lukechampine.com/blake3 v1.2.1 // indirect ) go 1.20 + +replace github.com/ipfs/go-datastore => github.com/vulcanize/go-datastore v0.6.1-internal-0.0.1 + +replace github.com/ipfs/boxo => github.com/vulcanize/boxo v0.13.2-internal-0.0.2 diff --git a/go.sum b/go.sum index a642a40..f84def4 100644 --- a/go.sum +++ b/go.sum @@ -1,43 +1,75 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= +github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= +github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= +github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= +github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds= github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= +github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= +github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= +github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= -github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= -github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= -github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= @@ -49,26 +81,73 @@ github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hg github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vulcanize/boxo v0.13.2-internal-0.0.2 h1:C0sJqmC9VdAt7k1b8Kb0tudg3ByoAEavnzavJYSXi68= +github.com/vulcanize/boxo v0.13.2-internal-0.0.2/go.mod h1:fUinsAKODKpOfq1QNNRSEH7YNogCKD4spihzs36spFQ= +github.com/vulcanize/go-datastore v0.6.1-internal-0.0.1 h1:bQ7PHVqE2FMS7UqsGL0SGbT6Vs4tkaiKF8QAhG5SsRM= +github.com/vulcanize/go-datastore v0.6.1-internal-0.0.1/go.mod h1:Fn29+QAQnUE7g8BMOAevKP504ZaCjVTYQAey5Es+5xg= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 h1:yVYDLoN2gmB3OdBXFW8e1UwgVbmCvNlnAKhvHPaNARI= github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c= -lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= diff --git a/store.go b/store.go index 8210cf3..a224a1a 100644 --- a/store.go +++ b/store.go @@ -3,6 +3,7 @@ package cbornode import ( "bytes" "context" + "errors" "fmt" block "github.com/ipfs/go-block-format" @@ -207,3 +208,141 @@ func (mb *mockBlocks) Put(ctx context.Context, b block.Block) error { mb.data[b.Cid()] = b return nil } + +type Cursor struct { + CID cid.Cid + Index int + Err error +} + +// IpldBatchOpStore wraps a GetManyBlockstore and provides an interface for retrieving CBOR encoded data in batches +type IpldBatchOpStore interface { + GetMany(ctx context.Context, cs []cid.Cid, outs []interface{}) (<-chan Cursor, []cid.Cid, error) + PutMany(ctx context.Context, vs []interface{}) ([]cid.Cid, error) +} + +// IpldBatchOpBlockStore defines a subset of the go-ipfs-blockstore Blockstore interface providing a method +// for retrieving block-centered data in batches +type IpldBatchOpBlockStore interface { + IpldBlockstore + PutMany(ctx context.Context, blocks []block.Block) error + GetMany(ctx context.Context, cs []cid.Cid) ([]block.Block, []cid.Cid, error) +} + +// BatchOpIpldStore wraps and IpldBlockstore and implements the IpldBatchOpStore interface. +type BatchOpIpldStore struct { + *BasicIpldStore + BatchOps IpldBatchOpBlockStore +} + +var _ IpldStore = &BatchOpIpldStore{} +var _ IpldBatchOpStore = &BatchOpIpldStore{} + +// NewGetManyCborStore returns an IpldStore implementation backed by the provided IpldGetManyBlockStore. +func NewGetManyCborStore(bs IpldBatchOpBlockStore) *BatchOpIpldStore { + viewer, _ := bs.(IpldBlockstoreViewer) + bis := &BasicIpldStore{Blocks: bs, Viewer: viewer} + return &BatchOpIpldStore{BatchOps: bs, BasicIpldStore: bis} +} + +// GetMany reads and unmarshals the content at `cs` into `outs` +// it returns a channel for tracking the position, identify, and any decode errors in output list +// as well as a list of all the CIDs that could not be retrieved from the underlying blockstore +func (b *BatchOpIpldStore) GetMany(ctx context.Context, cs []cid.Cid, outs []interface{}) (<-chan Cursor, []cid.Cid, error) { + if len(cs) != len(outs) { + return nil, nil, errors.New("expected list of cids to be the same length as the destination decode list") + } + blks, missingCIDs, err := b.BatchOps.GetMany(ctx, cs) + if err != nil { + return nil, nil, err + } + cursors := make(chan Cursor) + go func() { + for i, blk := range blks { + err := b.decode(blk.RawData(), outs[i]) + cursors <- Cursor{ + CID: blk.Cid(), + Index: i, + Err: err, + } + } + close(cursors) // expected behavior is that the Cursor channel will be drained by the receiver even + // if errors are encountered. If not, and this close is never reached, the goroutine will be eligible for GC + // once the dropped channel reference is GC'ed + }() + return cursors, missingCIDs, nil +} + +// PutMany marshals and writes the content in `vs` into the underlying blockstore +func (b *BatchOpIpldStore) PutMany(ctx context.Context, vs []interface{}) ([]cid.Cid, error) { + mhType := DefaultMultihash + if b.DefaultMultihash != 0 { + mhType = b.DefaultMultihash + } + + mhLen := -1 + codec := uint64(cid.DagCBOR) + // convert vs to IPLD blocks + blocks := make([]block.Block, len(vs)) + cids := make([]cid.Cid, len(vs)) + for i, v := range vs { + var expCid cid.Cid + if c, ok := v.(cidProvider); ok { + expCid = c.Cid() + pref := expCid.Prefix() + mhType = pref.MhType + mhLen = pref.MhLength + codec = pref.Codec + } + + cm, ok := v.(cbg.CBORMarshaler) + if ok { + buf := new(bytes.Buffer) + if err := cm.MarshalCBOR(buf); err != nil { + return nil, err + } + + pref := cid.Prefix{ + Codec: codec, + MhType: mhType, + MhLength: mhLen, + Version: 1, + } + c, err := pref.Sum(buf.Bytes()) + if err != nil { + return nil, err + } + + blk, err := block.NewBlockWithCid(buf.Bytes(), c) + if err != nil { + return nil, err + } + + blkCid := blk.Cid() + if expCid != cid.Undef && blkCid != expCid { + return nil, fmt.Errorf("your object is not being serialized the way it expects to\r\n"+ + "expected cid: %s actual cid: %s", expCid.String(), blkCid.String()) + } + + cids[i] = blkCid + blocks[i] = blk + continue + } + + nd, err := WrapObject(v, mhType, mhLen) + if err != nil { + return nil, err + } + + ndCid := nd.Cid() + if expCid != cid.Undef && ndCid != expCid { + return nil, fmt.Errorf("your object is not being serialized the way it expects to\r\n"+ + "expected cid: %s actual cid: %s", expCid.String(), ndCid.String()) + } + + cids[i] = ndCid + blocks[i] = nd + } + + return cids, b.BatchOps.PutMany(ctx, blocks) +} diff --git a/store_test.go b/store_test.go new file mode 100644 index 0000000..6c255a5 --- /dev/null +++ b/store_test.go @@ -0,0 +1,137 @@ +package cbornode_test + +import ( + "bytes" + "context" + "testing" + + "github.com/ipfs/boxo/blockstore" + u "github.com/ipfs/boxo/util" + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + ds_sync "github.com/ipfs/go-datastore/sync" + dstest "github.com/ipfs/go-datastore/test" + cbornode "github.com/ipfs/go-ipld-cbor" + ipld "github.com/ipfs/go-ipld-format" +) + +var bg = context.Background() + +func TestGetWhenKeyNotPresent(t *testing.T) { + bs := blockstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + cborStore := cbornode.NewCborStore(bs) + c := cid.NewCidV1(uint64(cid.DagCBOR), u.Hash([]byte("stuff"))) + out := new(cbornode.CborByteArray) + err := cborStore.Get(bg, c, out) + + if !bytes.Equal([]byte{}, []byte(*out)) { + t.Error("nil block expected") + } + if err == nil { + t.Error("error expected, got nil") + } +} + +func TestGetWhenKeyIsNil(t *testing.T) { + bs := blockstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + cborStore := cbornode.NewCborStore(bs) + out := new(cbornode.CborByteArray) + err := cborStore.Get(bg, cid.Cid{}, out) + + if !bytes.Equal([]byte{}, []byte(*out)) { + t.Error("nil block expected") + } + if err == nil { + t.Error("error expected, got nil") + } +} + +func TestPutThenGetBlock(t *testing.T) { + bs := blockstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + cborStore := cbornode.NewCborStore(bs) + cborObj := cbornode.Cborstr("some data") + + c, err := cborStore.Put(bg, cborObj) + if err != nil { + t.Fatal(err) + } + if !cborObj.Cid().Equals(c) { + t.Fatalf("expected cid %s does not match actual cid %s", cborObj.Cid().String(), c.String()) + } + + out := new(cbornode.CborByteArray) + err = cborStore.Get(bg, cborObj.Cid(), out) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(cborObj.RawData(), out.RawData()) { + t.Fatalf("expected result %x does not match actual result %x", cborObj.RawData(), out.RawData()) + } +} + +func TestGetManyWhenKeyNotPresent(t *testing.T) { + bs := blockstore.NewGetManyBlockstore(dstest.NewTestTxnDatastore(ds.NewMapDatastore(), false)) + cborStore := cbornode.NewGetManyCborStore(bs) + c1 := cid.NewCidV1(uint64(cid.DagCBOR), u.Hash([]byte("stuff"))) + c2 := cid.NewCidV1(uint64(cid.DagCBOR), u.Hash([]byte("stuff2"))) + outs := []interface{}{new(cbornode.CborByteArray), new(cbornode.CborByteArray)} + cursors, missingCIDs, err := cborStore.GetMany(bg, []cid.Cid{c1, c2}, outs) + if len(cursors) != 0 { + t.Error("no cursors expected") + } + if len(missingCIDs) != 2 { + t.Error("2 missing cids expected") + } + if err != nil { + t.Error("no error expected") + } +} + +func TestGetManyWhenKeyIsNil(t *testing.T) { + bs := blockstore.NewGetManyBlockstore(dstest.NewTestTxnDatastore(ds.NewMapDatastore(), false)) + cborStore := cbornode.NewGetManyCborStore(bs) + outs := []interface{}{new(cbornode.CborByteArray), new(cbornode.CborByteArray)} + _, _, err := cborStore.GetMany(bg, []cid.Cid{{}, {}}, outs) + if !ipld.IsNotFound(err) { + t.Fail() + } +} + +func TestPutManyThenGetManyBlock(t *testing.T) { + bs := blockstore.NewGetManyBlockstore(dstest.NewTestTxnDatastore(ds.NewMapDatastore(), false)) + cborStore := cbornode.NewGetManyCborStore(bs) + cbornode.Cborstr("some data") + obj1 := cbornode.Cborstr("some data1") + obj2 := cbornode.Cborstr("some data2") + obj3 := cbornode.Cborstr("some data3") + obj4 := cbornode.Cborstr("some data4") + expectedResults := []interface{}{obj1, obj2, obj4} + _, err := cborStore.PutMany(bg, expectedResults) + if err != nil { + t.Fatal(err) + } + + outs := []interface{}{new(cbornode.CborByteArray), + new(cbornode.CborByteArray), + new(cbornode.CborByteArray), + new(cbornode.CborByteArray)} + cursors, missingCIDs, err := cborStore.GetMany(bg, []cid.Cid{obj1.Cid(), obj2.Cid(), obj3.Cid(), obj4.Cid()}, outs) + if err != nil { + t.Fatal(err) + } + if len(missingCIDs) != 1 { + println(len(missingCIDs)) + t.Fatal("unexpected number of missing CIDs") + } + for cursor := range cursors { + if !expectedResults[cursor.Index].(*cbornode.CborByteArray).Cid().Equals(cursor.CID) { + t.Fatalf("expected cid %s does not match actual result %s", expectedResults[cursor.Index].(*cbornode.CborByteArray).Cid().String(), cursor.CID.String()) + } + if !bytes.Equal(expectedResults[cursor.Index].(*cbornode.CborByteArray).RawData(), outs[cursor.Index].(*cbornode.CborByteArray).RawData()) { + t.Fatalf("expected result %x does not match actual result %x", expectedResults[cursor.Index].(*cbornode.CborByteArray).RawData(), outs[cursor.Index].(*cbornode.CborByteArray).RawData()) + } + } + if !bytes.Equal(missingCIDs[0].Bytes(), obj3.Cid().Bytes()) { + t.Fail() + } +} diff --git a/util.go b/util.go new file mode 100644 index 0000000..1dd982c --- /dev/null +++ b/util.go @@ -0,0 +1,64 @@ +package cbornode + +import ( + "bytes" + "fmt" + "io" + + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + + cbg "github.com/whyrusleeping/cbor-gen" +) + +type CborByteArray []byte + +func (c *CborByteArray) MarshalCBOR(w io.Writer) error { + if err := cbg.WriteMajorTypeHeader(w, cbg.MajByteString, uint64(len(*c))); err != nil { + return err + } + _, err := w.Write(*c) + return err +} + +func (c *CborByteArray) UnmarshalCBOR(r io.Reader) error { + maj, extra, err := cbg.CborReadHeader(r) + if err != nil { + return err + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + if uint64(cap(*c)) < extra { + *c = make([]byte, extra) + } + if _, err := io.ReadFull(r, *c); err != nil { + return err + } + return nil +} + +func Cborstr(s string) *CborByteArray { + v := CborByteArray(s) + return &v +} + +func (c *CborByteArray) Cid() cid.Cid { + pref := cid.Prefix{ + Codec: uint64(cid.DagCBOR), + MhType: mh.SHA2_256, + MhLength: 32, + Version: 1, + } + cc, err := pref.Sum(c.RawData()) + if err != nil { + return cid.Undef + } + return cc +} + +func (c *CborByteArray) RawData() []byte { + buf := new(bytes.Buffer) + c.MarshalCBOR(buf) + return buf.Bytes() +}