Skip to content

Commit f859a00

Browse files
rvaggrvagg
authored
Merge pull request #45 from ipfs/rvagg/selector
feat: add UnixFSPathSelectorBuilder
2 parents ca00f89 + 51109f5 commit f859a00

File tree

2 files changed

+295
-13
lines changed

2 files changed

+295
-13
lines changed

‎signaling.go‎

Lines changed: 92 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,45 @@
11
package unixfsnode
22

33
import (
4-
"strings"
5-
64
"github.com/ipld/go-ipld-prime"
75
"github.com/ipld/go-ipld-prime/datamodel"
86
"github.com/ipld/go-ipld-prime/linking"
97
"github.com/ipld/go-ipld-prime/node/basicnode"
8+
"github.com/ipld/go-ipld-prime/traversal/selector"
109
"github.com/ipld/go-ipld-prime/traversal/selector/builder"
1110
)
1211

12+
// ExploreAllRecursivelySelector is a selector that will explore all nodes. It
13+
// is the same selector as selectorparse.CommonSelector_ExploreAllRecursively
14+
// but it is precompiled for use with UnixFSPathSelectorBuilder().
15+
var ExploreAllRecursivelySelector = specBuilder(func(ssb builder.SelectorSpecBuilder) builder.SelectorSpec {
16+
return ssb.ExploreRecursive(
17+
selector.RecursionLimitNone(),
18+
ssb.ExploreAll(ssb.ExploreRecursiveEdge()),
19+
)
20+
})
21+
22+
// MatchUnixFSPreloadSelector is a selector that will match a single node,
23+
// similar to selectorparse.CommonSelector_MatchPoint, but uses the
24+
// "unixfs-preload" ADL to load sharded files and directories as a single node.
25+
// Can be used to shallow load an entire UnixFS directory listing, sharded or
26+
// not, but not its contents.
27+
// MatchUnixfsPreloadSelector is precompiled for use with
28+
// UnixFSPathSelectorBuilder().
29+
var MatchUnixFSPreloadSelector = specBuilder(func(ssb builder.SelectorSpecBuilder) builder.SelectorSpec {
30+
return ssb.ExploreInterpretAs("unixfs-preload", ssb.Matcher())
31+
})
32+
33+
// MatchUnixFSSelector is a selector that will match a single node, similar to
34+
// selectorparse.CommonSelector_MatchPoint, but uses the "unixfs" ADL to load
35+
// as UnixFS data. Unlike MatchUnixFSPreloadSelector, this selector will not
36+
// preload all blocks in sharded directories or files. Use
37+
// MatchUnixFSPreloadSelector where the blocks that constitute the full UnixFS
38+
// resource being selected are important to load.
39+
var MatchUnixFSSelector = specBuilder(func(ssb builder.SelectorSpecBuilder) builder.SelectorSpec {
40+
return ssb.ExploreInterpretAs("unixfs", ssb.Matcher())
41+
})
42+
1343
func AddUnixFSReificationToLinkSystem(lsys *ipld.LinkSystem) {
1444
if lsys.KnownReifiers == nil {
1545
lsys.KnownReifiers = make(map[string]linking.NodeReifier)
@@ -18,18 +48,67 @@ func AddUnixFSReificationToLinkSystem(lsys *ipld.LinkSystem) {
1848
lsys.KnownReifiers["unixfs-preload"] = nonLazyReify
1949
}
2050

21-
// UnixFSPathSelector creates a selector for a file/path inside of a UnixFS directory
22-
// if reification is setup on a link system
51+
// UnixFSPathSelector creates a selector for IPLD path to a UnixFS resource if
52+
// UnixFS reification is setup on a LinkSystem being used for traversal.
53+
//
54+
// Use UnixFSPathSelectorBuilder for more control over the selector, this
55+
// function is the same as calling
56+
//
57+
// UnixFSPathSelectorBuilder(path, MatchUnixFSSelector, false)
2358
func UnixFSPathSelector(path string) datamodel.Node {
24-
segments := strings.Split(path, "/")
59+
return UnixFSPathSelectorBuilder(path, MatchUnixFSSelector, false)
60+
}
61+
62+
// UnixFSPathSelectorBuilder creates a selector for IPLD path to a UnixFS
63+
// resource if UnixFS reification is setup on a LinkSystem being used for
64+
// traversal.
65+
//
66+
// The path is interpreted according to
67+
// github.com/ipld/go-ipld-prime/datamodel/Path rules,
68+
// i.e.
69+
// - leading and trailing slashes are ignored
70+
// - redundant slashes are ignored
71+
// - the segment `..` is a field named `..`, same with `.`
72+
//
73+
// targetSelector is the selector to apply to the final node in the path.
74+
// Use ExploreAllRecursivelySelector to explore (i.e. load the blocks) all of
75+
// the content from the terminus of the path. Use MatchUnixFSPreloadSelector to
76+
// match the terminus of the path, but preload all blocks in sharded files and
77+
// directories. Use MatchUnixFSSelector to match the terminus of the path, but
78+
// not preload any blocks if the terminus is sharded. Or any other custom
79+
// SelectorSpec can be supplied.
80+
//
81+
// If matchPath is false, the selector will explore, not match, so it's useful
82+
// for traversals where block loads are important, not where the matcher visitor
83+
// callback is important. if matchPath is true, the selector will match the
84+
// nodes along the path while exploring them.
85+
func UnixFSPathSelectorBuilder(path string, targetSelector builder.SelectorSpec, matchPath bool) ipld.Node {
86+
segments := ipld.ParsePath(path)
87+
88+
ss := targetSelector
2589
ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
26-
selectorSoFar := ssb.ExploreInterpretAs("unixfs", ssb.Matcher())
27-
for i := len(segments) - 1; i >= 0; i-- {
28-
selectorSoFar = ssb.ExploreInterpretAs("unixfs",
29-
ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) {
30-
efsb.Insert(segments[i], selectorSoFar)
31-
}),
32-
)
90+
91+
for segments.Len() > 0 {
92+
// Wrap selector in ExploreFields as we walk back up through the path.
93+
// We can assume each segment to be a unixfs path section, so we
94+
// InterpretAs to make sure the node is reified through go-unixfsnode
95+
// (if possible) and we can traverse through according to unixfs pathing
96+
// rather than bare IPLD pathing - which also gives us the ability to
97+
// traverse through HAMT shards.
98+
ss = ssb.ExploreInterpretAs("unixfs", ssb.ExploreFields(
99+
func(efsb builder.ExploreFieldsSpecBuilder) {
100+
efsb.Insert(segments.Last().String(), ss)
101+
},
102+
))
103+
if matchPath {
104+
ss = ssb.ExploreUnion(ssb.Matcher(), ss)
105+
}
106+
segments = segments.Pop()
33107
}
34-
return selectorSoFar.Node()
108+
109+
return ss.Node()
110+
}
111+
112+
func specBuilder(b func(ssb builder.SelectorSpecBuilder) builder.SelectorSpec) builder.SelectorSpec {
113+
return b(builder.NewSelectorSpecBuilder(basicnode.Prototype.Any))
35114
}

‎signalling_test.go‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
package unixfsnode_test
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
"testing"
7+
8+
"github.com/ipfs/go-unixfsnode"
9+
"github.com/ipld/go-ipld-prime"
10+
"github.com/ipld/go-ipld-prime/codec/dagjson"
11+
"github.com/ipld/go-ipld-prime/traversal/selector/builder"
12+
selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
13+
"github.com/stretchr/testify/require"
14+
)
15+
16+
// Selectors are tested against JSON expected forms; this doesn't necessarily
17+
// validate that they work as advertised. It's just a sanity check that the
18+
// selectors are being built as expected.
19+
20+
var exploreAllJson = mustDagJson(selectorparse.CommonSelector_ExploreAllRecursively)
21+
22+
// explore interpret-as (~), next (>), match (.), interpreted as unixfs-preload
23+
var matchUnixfsPreloadJson = `{"~":{">":{".":{}},"as":"unixfs-preload"}}`
24+
25+
// match interpret-as (~), next (>), match (.), interpreted as unixfs
26+
var matchUnixfsJson = `{"~":{">":{".":{}},"as":"unixfs"}}`
27+
28+
func TestUnixFSPathSelector(t *testing.T) {
29+
testCases := []struct {
30+
name string
31+
path string
32+
expextedSelector string
33+
}{
34+
{
35+
name: "empty path",
36+
path: "",
37+
expextedSelector: matchUnixfsJson,
38+
},
39+
{
40+
name: "single field",
41+
path: "/foo",
42+
expextedSelector: jsonFields(matchUnixfsJson, "foo"),
43+
},
44+
{
45+
name: "multiple fields",
46+
path: "/foo/bar",
47+
expextedSelector: jsonFields(matchUnixfsJson, "foo", "bar"),
48+
},
49+
{
50+
name: "leading slash optional",
51+
path: "foo/bar",
52+
expextedSelector: jsonFields(matchUnixfsJson, "foo", "bar"),
53+
},
54+
{
55+
name: "trailing slash optional",
56+
path: "/foo/bar/",
57+
expextedSelector: jsonFields(matchUnixfsJson, "foo", "bar"),
58+
},
59+
{
60+
// a go-ipld-prime specific thing, not clearly specified by path spec (?)
61+
name: ".. is a field named ..",
62+
path: "/foo/../bar/",
63+
expextedSelector: jsonFields(matchUnixfsJson, "foo", "..", "bar"),
64+
},
65+
{
66+
// a go-ipld-prime specific thing, not clearly specified by path spec
67+
name: "redundant slashes ignored",
68+
path: "foo///bar",
69+
expextedSelector: jsonFields(matchUnixfsJson, "foo", "bar"),
70+
},
71+
}
72+
73+
for _, tc := range testCases {
74+
t.Run(tc.name, func(t *testing.T) {
75+
sel := unixfsnode.UnixFSPathSelector(tc.path)
76+
require.Equal(t, tc.expextedSelector, mustDagJson(sel))
77+
})
78+
}
79+
}
80+
81+
func TestUnixFSPathSelectorBuilder(t *testing.T) {
82+
testCases := []struct {
83+
name string
84+
path string
85+
target builder.SelectorSpec
86+
matchPath bool
87+
expextedSelector string
88+
}{
89+
{
90+
name: "empty path",
91+
path: "",
92+
target: unixfsnode.ExploreAllRecursivelySelector,
93+
expextedSelector: exploreAllJson,
94+
},
95+
{
96+
name: "empty path shallow",
97+
path: "",
98+
target: unixfsnode.MatchUnixFSPreloadSelector,
99+
expextedSelector: matchUnixfsPreloadJson,
100+
},
101+
{
102+
name: "single field",
103+
path: "/foo",
104+
expextedSelector: jsonFields(exploreAllJson, "foo"),
105+
target: unixfsnode.ExploreAllRecursivelySelector,
106+
},
107+
{
108+
name: "single field, match path",
109+
path: "/foo",
110+
expextedSelector: jsonFieldsMatchPoint(exploreAllJson, "foo"),
111+
target: unixfsnode.ExploreAllRecursivelySelector,
112+
matchPath: true,
113+
},
114+
{
115+
name: "single field shallow",
116+
path: "/foo",
117+
expextedSelector: jsonFields(matchUnixfsPreloadJson, "foo"),
118+
target: unixfsnode.MatchUnixFSPreloadSelector,
119+
},
120+
{
121+
name: "multiple fields",
122+
path: "/foo/bar",
123+
expextedSelector: jsonFields(exploreAllJson, "foo", "bar"),
124+
target: unixfsnode.ExploreAllRecursivelySelector,
125+
},
126+
{
127+
name: "multiple fields, match path",
128+
path: "/foo/bar",
129+
expextedSelector: jsonFieldsMatchPoint(exploreAllJson, "foo", "bar"),
130+
target: unixfsnode.ExploreAllRecursivelySelector,
131+
matchPath: true,
132+
},
133+
{
134+
name: "multiple fields shallow",
135+
path: "/foo/bar",
136+
expextedSelector: jsonFields(matchUnixfsPreloadJson, "foo", "bar"),
137+
target: unixfsnode.MatchUnixFSPreloadSelector,
138+
},
139+
{
140+
name: "leading slash optional",
141+
path: "foo/bar",
142+
expextedSelector: jsonFields(exploreAllJson, "foo", "bar"),
143+
target: unixfsnode.ExploreAllRecursivelySelector,
144+
},
145+
{
146+
name: "trailing slash optional",
147+
path: "/foo/bar/",
148+
expextedSelector: jsonFields(exploreAllJson, "foo", "bar"),
149+
target: unixfsnode.ExploreAllRecursivelySelector,
150+
},
151+
// a go-ipld-prime specific thing, not clearly specified by path spec (?)
152+
{
153+
name: ".. is a field named ..",
154+
path: "/foo/../bar/",
155+
expextedSelector: jsonFields(exploreAllJson, "foo", "..", "bar"),
156+
target: unixfsnode.ExploreAllRecursivelySelector,
157+
},
158+
{
159+
// a go-ipld-prime specific thing, not clearly specified by path spec
160+
name: "redundant slashes ignored",
161+
path: "foo///bar",
162+
expextedSelector: jsonFields(exploreAllJson, "foo", "bar"),
163+
target: unixfsnode.ExploreAllRecursivelySelector,
164+
},
165+
}
166+
167+
for _, tc := range testCases {
168+
t.Run(tc.name, func(t *testing.T) {
169+
sel := unixfsnode.UnixFSPathSelectorBuilder(tc.path, tc.target, tc.matchPath)
170+
require.Equal(t, tc.expextedSelector, mustDagJson(sel))
171+
})
172+
}
173+
}
174+
175+
func jsonFields(target string, fields ...string) string {
176+
var sb strings.Builder
177+
for _, n := range fields {
178+
// explore interpret-as (~) next (>), explore field (f) + specific field (f>), with field name
179+
sb.WriteString(fmt.Sprintf(`{"~":{">":{"f":{"f>":{"%s":`, n))
180+
}
181+
sb.WriteString(target)
182+
sb.WriteString(strings.Repeat(`}}},"as":"unixfs"}}`, len(fields)))
183+
return sb.String()
184+
}
185+
186+
func jsonFieldsMatchPoint(target string, fields ...string) string {
187+
var sb strings.Builder
188+
for _, n := range fields {
189+
// union (|) of match (.) and explore interpret-as (~) next (>), explore field (f) + specific field (f>), with field name
190+
sb.WriteString(fmt.Sprintf(`{"|":[{".":{}},{"~":{">":{"f":{"f>":{"%s":`, n))
191+
}
192+
sb.WriteString(target)
193+
sb.WriteString(strings.Repeat(`}}},"as":"unixfs"}}]}`, len(fields)))
194+
return sb.String()
195+
}
196+
197+
func mustDagJson(n ipld.Node) string {
198+
byts, err := ipld.Encode(n, dagjson.Encode)
199+
if err != nil {
200+
panic(err)
201+
}
202+
return string(byts)
203+
}

0 commit comments

Comments
 (0)