Skip to content

Commit f2af51a

Browse files
committed
feat(unixfsnode): initial implementation
1 parent 05d5109 commit f2af51a

File tree

7 files changed

+364
-1
lines changed

7 files changed

+364
-1
lines changed

README.md

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,5 @@
1-
# go-unixfs-node
1+
# go-unixfsnode
2+
3+
This is an IPLD ADL that provides string based pathing for protobuf nodes. The top level node behaves like a map where LookupByString returns the Hash property on the Link in the protobufs list of Links whos Name property matches the key. This should enable selector traversals that work based of paths.
4+
5+
Note that while it works internally with go-codec-dagpb, the Reify method (used to get a UnixFSNode from a DagPB node should actually work successfully with go-ipld-prime-proto nodes)

go.mod

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
module github.com/ipfs/go-unixfsnode
2+
3+
go 1.15
4+
5+
require (
6+
github.com/ipld/go-codec-dagpb v1.0.1
7+
github.com/ipld/go-ipld-prime v0.7.0
8+
)

go.sum

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
2+
github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
3+
github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY=
4+
github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
5+
github.com/ipld/go-codec-dagpb v1.0.1 h1:1bpbWKNyRGpCV2w9QIJPmmmFODjjNVhHiM9t/kT/HP8=
6+
github.com/ipld/go-codec-dagpb v1.0.1/go.mod h1:3RKe7tCm89O2jWw5nQDQx0MMMxJUQbZzBHkleyttP3M=
7+
github.com/ipld/go-ipld-prime v0.7.0 h1:eigF1ZpaL1prbsKYVMqPLoPJqD/pzkQOe2j1uzvVg7w=
8+
github.com/ipld/go-ipld-prime v0.7.0/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM=
9+
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
10+
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
11+
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
12+
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
13+
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
14+
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
15+
github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
16+
github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
17+
github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
18+
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
19+
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
20+
github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
21+
github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
22+
github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
23+
github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
24+
github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
25+
github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
26+
github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
27+
github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
28+
github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
29+
github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I=
30+
github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
31+
github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
32+
github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
33+
github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
34+
github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
35+
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls=
36+
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
37+
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
38+
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
39+
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
40+
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
41+
github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
42+
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
43+
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
44+
golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
45+
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
46+
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
47+
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
48+
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
49+
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
50+
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
51+
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
52+
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
53+
golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo=
54+
golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
55+
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
56+
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
57+
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
58+
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
59+
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

link.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package unixfsnode
2+
3+
import (
4+
dagpb "github.com/ipld/go-codec-dagpb"
5+
"github.com/ipld/go-ipld-prime"
6+
"github.com/ipld/go-ipld-prime/schema"
7+
)
8+
9+
var _ ipld.Node = UnixFSLink(nil)
10+
var _ schema.TypedNode = UnixFSLink(nil)
11+
var _ schema.TypedLinkNode = UnixFSLink(nil)
12+
13+
// UnixFSLink just adds a LinkTargetNodePrototype method to dagpb.Link so that you can cross
14+
// link boundaries correctly in path traversals
15+
type UnixFSLink = *_UnixFSLink
16+
17+
type _UnixFSLink struct {
18+
dagpb.Link
19+
}
20+
21+
func (n UnixFSLink) LinkTargetNodePrototype() ipld.NodePrototype {
22+
return _UnixFSNode__Prototype{}
23+
}

node.go

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
package unixfsnode
2+
3+
import (
4+
dagpb "github.com/ipld/go-codec-dagpb"
5+
"github.com/ipld/go-ipld-prime"
6+
"github.com/ipld/go-ipld-prime/schema"
7+
)
8+
9+
var _ ipld.Node = UnixFSNode(nil)
10+
var _ schema.TypedNode = UnixFSNode(nil)
11+
12+
type UnixFSNode = *_UnixFSNode
13+
14+
type _UnixFSNode struct {
15+
_substrate dagpb.PBNode
16+
}
17+
18+
func (n UnixFSNode) Kind() ipld.Kind {
19+
return n._substrate.Kind()
20+
}
21+
22+
// LookupByString looks for the key in the list of links with a matching name
23+
func (n UnixFSNode) LookupByString(key string) (ipld.Node, error) {
24+
links := n._substrate.FieldLinks()
25+
link := lookup(links, key)
26+
if link == nil {
27+
return nil, schema.ErrNoSuchField{Type: nil /*TODO*/, Field: ipld.PathSegmentOfString(key)}
28+
}
29+
return link, nil
30+
}
31+
32+
func (n UnixFSNode) LookupByNode(key ipld.Node) (ipld.Node, error) {
33+
ks, err := key.AsString()
34+
if err != nil {
35+
return nil, err
36+
}
37+
return n.LookupByString(ks)
38+
}
39+
40+
func (n UnixFSNode) LookupByIndex(idx int64) (ipld.Node, error) {
41+
return n._substrate.LookupByIndex(idx)
42+
}
43+
44+
func (n UnixFSNode) LookupBySegment(seg ipld.PathSegment) (ipld.Node, error) {
45+
return n.LookupByString(seg.String())
46+
}
47+
48+
func (n UnixFSNode) MapIterator() ipld.MapIterator {
49+
return &UnixFSNode__MapItr{n._substrate.Links.Iterator()}
50+
}
51+
52+
// UnixFSNode map iterator iterates throught the links as if they were a map
53+
// Note: for now it does return links with no name, where the key is just String("")
54+
type UnixFSNode__MapItr struct {
55+
_substrate *dagpb.PBLinks__Itr
56+
}
57+
58+
func (itr *UnixFSNode__MapItr) Next() (k ipld.Node, v ipld.Node, err error) {
59+
_, next := itr._substrate.Next()
60+
if next == nil {
61+
return nil, nil, ipld.ErrIteratorOverread{}
62+
}
63+
if next.FieldName().Exists() {
64+
return next.FieldName().Must(), &_UnixFSLink{next.FieldHash()}, nil
65+
}
66+
nb := dagpb.Type.String.NewBuilder()
67+
err = nb.AssignString("")
68+
if err != nil {
69+
return nil, nil, err
70+
}
71+
s := nb.Build()
72+
return s, next.FieldHash(), nil
73+
}
74+
75+
func (itr *UnixFSNode__MapItr) Done() bool {
76+
return itr._substrate.Done()
77+
}
78+
79+
// ListIterator returns an iterator which yields key-value pairs
80+
// traversing the node.
81+
// If the node kind is anything other than a list, nil will be returned.
82+
//
83+
// The iterator will yield every entry in the list; that is, it
84+
// can be expected that itr.Next will be called node.Length times
85+
// before itr.Done becomes true.
86+
func (n UnixFSNode) ListIterator() ipld.ListIterator {
87+
return nil
88+
}
89+
90+
// Length returns the length of a list, or the number of entries in a map,
91+
// or -1 if the node is not of list nor map kind.
92+
func (n UnixFSNode) Length() int64 {
93+
return n._substrate.FieldLinks().Length()
94+
}
95+
96+
func (n UnixFSNode) IsAbsent() bool {
97+
return false
98+
}
99+
100+
func (n UnixFSNode) IsNull() bool {
101+
return false
102+
}
103+
104+
func (n UnixFSNode) AsBool() (bool, error) {
105+
return n._substrate.AsBool()
106+
}
107+
108+
func (n UnixFSNode) AsInt() (int64, error) {
109+
return n._substrate.AsInt()
110+
}
111+
112+
func (n UnixFSNode) AsFloat() (float64, error) {
113+
return n._substrate.AsFloat()
114+
}
115+
116+
func (n UnixFSNode) AsString() (string, error) {
117+
return n._substrate.AsString()
118+
}
119+
120+
func (n UnixFSNode) AsBytes() ([]byte, error) {
121+
return n._substrate.AsBytes()
122+
}
123+
124+
func (n UnixFSNode) AsLink() (ipld.Link, error) {
125+
return n._substrate.AsLink()
126+
}
127+
128+
func (n UnixFSNode) Prototype() ipld.NodePrototype {
129+
return _UnixFSNode__Prototype{}
130+
}
131+
132+
// satisfy schema.TypedNode
133+
func (UnixFSNode) Type() schema.Type {
134+
return nil /*TODO:typelit*/
135+
}
136+
137+
func (n UnixFSNode) Representation() ipld.Node {
138+
return n._substrate.Representation()
139+
}
140+
141+
// Native map accessors
142+
143+
func (n UnixFSNode) Iterator() *UnixFSNode__Itr {
144+
145+
return &UnixFSNode__Itr{n._substrate.Links.Iterator()}
146+
}
147+
148+
type UnixFSNode__Itr struct {
149+
_substrate *dagpb.PBLinks__Itr
150+
}
151+
152+
func (itr *UnixFSNode__Itr) Next() (k dagpb.String, v UnixFSLink) {
153+
_, next := itr._substrate.Next()
154+
if next == nil {
155+
return nil, nil
156+
}
157+
if next.FieldName().Exists() {
158+
return next.FieldName().Must(), &_UnixFSLink{next.FieldHash()}
159+
}
160+
nb := dagpb.Type.String.NewBuilder()
161+
err := nb.AssignString("")
162+
if err != nil {
163+
return nil, nil
164+
}
165+
s := nb.Build()
166+
return s.(dagpb.String), &_UnixFSLink{next.FieldHash()}
167+
}
168+
func (itr *UnixFSNode__Itr) Done() bool {
169+
return itr._substrate.Done()
170+
}
171+
172+
func (n UnixFSNode) Lookup(key dagpb.String) UnixFSLink {
173+
return lookup(n._substrate.FieldLinks(), key.String())
174+
}
175+
176+
// direct access to the links and data
177+
178+
func (n UnixFSNode) FieldLinks() dagpb.PBLinks {
179+
return n._substrate.FieldLinks()
180+
}
181+
182+
func (n UnixFSNode) FieldData() dagpb.MaybeBytes {
183+
return n._substrate.FieldData()
184+
}
185+
186+
// we need to lookup by key in a list of dag pb links a fair amount, so just have
187+
// a shortcut method
188+
func lookup(links dagpb.PBLinks, key string) UnixFSLink {
189+
li := links.Iterator()
190+
for !li.Done() {
191+
_, next := li.Next()
192+
name := ""
193+
if next.FieldName().Exists() {
194+
name = next.FieldName().Must().String()
195+
}
196+
if key == name {
197+
return &_UnixFSLink{next.FieldHash()}
198+
}
199+
}
200+
return nil
201+
}

nodeprototype.go

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package unixfsnode
2+
3+
import (
4+
dagpb "github.com/ipld/go-codec-dagpb"
5+
"github.com/ipld/go-ipld-prime"
6+
)
7+
8+
// NodeBuilder for UnixFS Nodes -- note: this expects underlying data that
9+
// has the same format as a normal dagpb node -- in fact, it uses the
10+
// exact same builder but then wraps at the end
11+
12+
var Type typeSlab
13+
14+
type typeSlab struct {
15+
UnixFSNode _UnixFSNode__Prototype
16+
}
17+
18+
type _UnixFSNode__Prototype struct{}
19+
20+
func (_UnixFSNode__Prototype) NewBuilder() ipld.NodeBuilder {
21+
var nb _UnixFSNode__Builder
22+
nb.Reset()
23+
return &nb
24+
}
25+
26+
type _UnixFSNode__Builder struct {
27+
ipld.NodeBuilder
28+
}
29+
30+
func (nb *_UnixFSNode__Builder) Build() ipld.Node {
31+
n := nb.NodeBuilder.Build().(dagpb.PBNode)
32+
return &_UnixFSNode{_substrate: n}
33+
}
34+
35+
func (nb *_UnixFSNode__Builder) Reset() {
36+
snb := dagpb.Type.PBNode.NewBuilder()
37+
*nb = _UnixFSNode__Builder{snb}
38+
}

reification.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package unixfsnode
2+
3+
import (
4+
"fmt"
5+
6+
dagpb "github.com/ipld/go-codec-dagpb"
7+
"github.com/ipld/go-ipld-prime"
8+
)
9+
10+
// Reify looks at an ipld Node and tries to interpret it as a UnixFSNode
11+
// if successful, it returns the UnixFSNode
12+
func Reify(maybePBNodeRoot ipld.Node) (ipld.Node, error) {
13+
if pbNode, ok := maybePBNodeRoot.(dagpb.PBNode); ok {
14+
return &_UnixFSNode{_substrate: pbNode}, nil
15+
}
16+
17+
// Shortcut didn't work. Process via the data model.
18+
// The AssignNode method on the pb node already contains all the logic necessary for this, so we use that.
19+
nb := dagpb.Type.PBNode.NewBuilder()
20+
if err := nb.AssignNode(maybePBNodeRoot); err != nil {
21+
return nil, fmt.Errorf("unixfsnode.Reify failed: data does not match expected shape for Protobuf Node: %w", err)
22+
}
23+
return &_UnixFSNode{nb.Build().(dagpb.PBNode)}, nil
24+
25+
}
26+
27+
// Substrate returns the underlying PBNode -- note: only the substrate will encode successfully to protobuf if writing
28+
func (n UnixFSNode) Substrate() ipld.Node {
29+
return n._substrate
30+
}

0 commit comments

Comments
 (0)