Skip to content

Commit fd6fd12

Browse files
XuanwoXuanwo
authored
storage: Implement proposal Unify List Operation (#20)
* storage: Implement proposal Unify List Operation Signed-off-by: Xuanwo <github@xuanwo.io> * Fix integration tests Signed-off-by: Xuanwo <github@xuanwo.io> * Fix integration tests Signed-off-by: Xuanwo <github@xuanwo.io> * Update deps Signed-off-by: Xuanwo <github@xuanwo.io>
1 parent 0bbe065 commit fd6fd12

File tree

7 files changed

+76
-137
lines changed

7 files changed

+76
-137
lines changed

‎generated.go‎

Lines changed: 35 additions & 84 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎go.mod‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@ go 1.14
44

55
require (
66
bou.ke/monkey v1.0.2
7-
github.com/aos-dev/go-integration-test/v2 v2.0.0-20201030072349-1418884fad78
8-
github.com/aos-dev/go-storage/v2 v2.0.1-0.20201203030158-b94397533b85
7+
github.com/aos-dev/go-integration-test/v2 v2.0.0-20210112071015-339e9258b7b2
8+
github.com/aos-dev/go-storage/v2 v2.0.1-0.20210112061652-0b7e2ab57b88
99
github.com/golang/mock v1.4.4
10-
github.com/google/uuid v1.1.2
10+
github.com/google/uuid v1.1.4
1111
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14
1212
github.com/qingstor/qingstor-sdk-go/v4 v4.2.0
1313
github.com/stretchr/testify v1.6.1

‎go.sum‎

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,12 @@ github.com/Xuanwo/templateutils v0.0.0-20201019101431-c73b871118f2 h1:ydpBZm38sX
66
github.com/Xuanwo/templateutils v0.0.0-20201019101431-c73b871118f2/go.mod h1:x0qS7gfgEm24b8V9U+0zBEBAu/VpxZhO/+EWtpgEgDw=
77
github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
88
github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
9-
github.com/aos-dev/go-integration-test/v2 v2.0.0-20201030072349-1418884fad78 h1:s/rzb/Zsilm/DOUjlgHNwBo9fByRsTGNCEh/T/uhI0c=
10-
github.com/aos-dev/go-integration-test/v2 v2.0.0-20201030072349-1418884fad78/go.mod h1:NfmZvOzPY/WaTXz7t7+/LaBecD/rvZWZKaQWR5oTQBg=
11-
github.com/aos-dev/go-storage/v2 v2.0.0-20201030030412-57f6d503c7c7/go.mod h1:eHpnrdrHP8mMoT1XdPcd7WRYtJKAYIcHmd2MhMO6l9E=
12-
github.com/aos-dev/go-storage/v2 v2.0.1-0.20201203030158-b94397533b85 h1:TmhNmt5NbanmhRXO47mg/QAqE7JHfiEx4n2ar0TCMKA=
13-
github.com/aos-dev/go-storage/v2 v2.0.1-0.20201203030158-b94397533b85/go.mod h1:BkdSR42JZFFjKJ+vq1UhjFODd0FPng9nMVR24IxKONI=
14-
github.com/aos-dev/specs/go v0.0.0-20201202111357-b50ee4ea7df4 h1:QHZ2ZCNLMlse6MoGyY/M+jVVJhoAq7Y0wjHQ5tH38OQ=
15-
github.com/aos-dev/specs/go v0.0.0-20201202111357-b50ee4ea7df4/go.mod h1:bsR49kgar1eScqMrZh4GvmZbeY2o4UtcDbC0jFtuJ0s=
9+
github.com/aos-dev/go-integration-test/v2 v2.0.0-20210112071015-339e9258b7b2 h1:gBe610/EhKOJKDay1jGzJXY/+OdxJ0at0ukAO7lDF5I=
10+
github.com/aos-dev/go-integration-test/v2 v2.0.0-20210112071015-339e9258b7b2/go.mod h1:/QyyTGYulgbdZr0smf9PPWsSplLp7ULbZe2zBTk5ycA=
11+
github.com/aos-dev/go-storage/v2 v2.0.1-0.20210112061652-0b7e2ab57b88 h1:fvQJJnDBtxCQjGBxRjGxVMbnunQHKeKOMXLUxlpQptE=
12+
github.com/aos-dev/go-storage/v2 v2.0.1-0.20210112061652-0b7e2ab57b88/go.mod h1:6mNwhRUOEiBFebskENkrfQPljurlnBY87sRl4VynWZY=
13+
github.com/aos-dev/specs/go v0.0.0-20210112040534-0e8e3c366e28 h1:iJJQfdxoBsWrUEVnhAiCpzhn713tKGMh0Bv7Yf1TyD0=
14+
github.com/aos-dev/specs/go v0.0.0-20210112040534-0e8e3c366e28/go.mod h1:lJ9DsZG2t1cMcn4DiUqfbGoKoyUgF+TOC4R5YVyqhns=
1615
github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
1716
github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0=
1817
github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
@@ -23,19 +22,17 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
2322
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
2423
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2524
github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
26-
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
2725
github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
2826
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
2927
github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
3028
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
3129
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
32-
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
33-
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
34-
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
30+
github.com/google/uuid v1.1.4 h1:0ecGp3skIrHWPNGPJDaBIghfA6Sp7Ruo2Io8eLKzWm0=
31+
github.com/google/uuid v1.1.4/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
3532
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
3633
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
37-
github.com/hashicorp/hcl/v2 v2.7.1 h1:eZWfKEO93WyE4OnZSD5LVy70YHDV/mmQ49gpKDLpDVM=
38-
github.com/hashicorp/hcl/v2 v2.7.1/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yIfAEN3jqY=
34+
github.com/hashicorp/hcl/v2 v2.8.2 h1:wmFle3D1vu0okesm8BTLVDyJ6/OL9DCLUwn0b2OptiY=
35+
github.com/hashicorp/hcl/v2 v2.8.2/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yIfAEN3jqY=
3936
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
4037
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
4138
github.com/kevinburke/go-bindata v3.22.0+incompatible h1:/JmqEhIWQ7GRScV0WjX/0tqBrC5D21ALg0H0U/KZ/ts=
@@ -81,7 +78,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
8178
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
8279
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
8380
golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
84-
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
8581
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
8682
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
8783
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -98,5 +94,3 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
9894
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
9995
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
10096
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
101-
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
102-
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

‎service.hcl‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,16 @@ namespace "service" {
2121
}
2222
}
2323
namespace "storage" {
24-
implement = ["copier", "dir_lister", "fetcher", "index_segmenter", "mover", "prefix_lister", "prefix_segments_lister", "reacher", "segmenter", "statistician"]
24+
implement = ["copier", "fetcher", "index_segmenter", "mover", "reacher", "segmenter", "segments_lister", "statistician"]
2525

2626
new {
2727
required = ["name"]
2828
optional = ["disable_uri_cleaning", "http_client_options", "location", "pair_policy", "work_dir"]
2929
}
3030

31+
op "list" {
32+
optional = ["list_type"]
33+
}
3134
op "reach" {
3235
required = ["expire"]
3336
}

‎storage.go‎

Lines changed: 18 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -43,21 +43,27 @@ func (i *listObjectInput) ContinuationToken() string {
4343
return convert.StringValue(i.Marker)
4444
}
4545

46-
func (s *Storage) listDir(ctx context.Context, dir string, opt *pairStorageListDir) (oi *typ.ObjectIterator, err error) {
46+
func (s *Storage) list(ctx context.Context, dir string, opt *pairStorageList) (oi *typ.ObjectIterator, err error) {
4747
marker := ""
48-
delimiter := "/"
4948
limit := 200
5049

5150
rp := s.getAbsPath(dir)
5251

5352
input := &listObjectInput{
54-
Limit: &limit,
55-
Marker: &marker,
56-
Prefix: &rp,
57-
Delimiter: &delimiter,
53+
Limit: &limit,
54+
Marker: &marker,
55+
Prefix: &rp,
56+
}
57+
58+
var nextFn typ.NextObjectFunc
59+
if opt.HasListType && opt.ListType == typ.ListTypeDir {
60+
input.Delimiter = convert.String("/")
61+
nextFn = s.listNextDir
62+
} else {
63+
nextFn = s.listNextPrefix
5864
}
5965

60-
return typ.NewObjectIterator(ctx, s.listNextDir, input), nil
66+
return typ.NewObjectIterator(ctx, nextFn, input), nil
6167
}
6268
func (s *Storage) listNextDir(ctx context.Context, page *typ.ObjectPage) error {
6369
input := page.Status.(*listObjectInput)
@@ -69,11 +75,10 @@ func (s *Storage) listNextDir(ctx context.Context, page *typ.ObjectPage) error {
6975
}
7076

7177
for _, v := range output.CommonPrefixes {
72-
o := &typ.Object{
73-
ID: *v,
74-
Name: s.getRelPath(*v),
75-
Type: typ.ObjectTypeDir,
76-
}
78+
o := s.newObject(true)
79+
o.ID = *v
80+
o.Name = s.getRelPath(*v)
81+
o.Type = typ.ObjectTypeDir
7782

7883
page.Data = append(page.Data, o)
7984
}
@@ -104,20 +109,7 @@ func (s *Storage) listNextDir(ctx context.Context, page *typ.ObjectPage) error {
104109
input.Marker = output.NextMarker
105110
return nil
106111
}
107-
func (s *Storage) listPrefix(ctx context.Context, prefix string, opt *pairStorageListPrefix) (oi *typ.ObjectIterator, err error) {
108-
marker := ""
109-
limit := 200
110112

111-
rp := s.getAbsPath(prefix)
112-
113-
input := &listObjectInput{
114-
Limit: &limit,
115-
Marker: &marker,
116-
Prefix: &rp,
117-
}
118-
119-
return typ.NewObjectIterator(ctx, s.listNextPrefix, input), nil
120-
}
121113
func (s *Storage) listNextPrefix(ctx context.Context, page *typ.ObjectPage) error {
122114
input := page.Status.(*listObjectInput)
123115
serviceInput := service.ListObjectsInput(*input)
@@ -155,7 +147,7 @@ type listMultipartUploadsInput service.ListMultipartUploadsInput
155147
func (i *listMultipartUploadsInput) ContinuationToken() string {
156148
return convert.StringValue(i.UploadIDMarker)
157149
}
158-
func (s *Storage) listPrefixSegments(ctx context.Context, prefix string, opt *pairStorageListPrefixSegments) (si *typ.SegmentIterator, err error) {
150+
func (s *Storage) listSegments(ctx context.Context, prefix string, opt *pairStorageListSegments) (si *typ.SegmentIterator, err error) {
159151
limit := 200
160152

161153
rp := s.getAbsPath(prefix)

‎storager_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@ func TestStorage_ListPrefix(t *testing.T) {
325325
bucket: mockBucket,
326326
}
327327

328-
it, err := client.ListPrefix(path)
328+
it, err := client.List(path, pairs.WithListType(ListTypePrefix))
329329
if err != nil {
330330
t.Error(err)
331331
}
@@ -625,7 +625,7 @@ func TestStorage_ListPrefixSegments(t *testing.T) {
625625

626626
items := make([]Segment, 0)
627627

628-
it, err := client.ListPrefixSegments(path)
628+
it, err := client.ListSegments(path)
629629
if err != nil {
630630
t.Error(err)
631631
}

‎utils.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -310,11 +310,10 @@ func (s *Storage) newObject(done bool) *typ.Object {
310310
}
311311

312312
func (s *Storage) formatFileObject(v *service.KeyType) (o *typ.Object, err error) {
313-
o = &typ.Object{
314-
ID: *v.Key,
315-
Name: s.getRelPath(*v.Key),
316-
Type: typ.ObjectTypeFile,
317-
}
313+
o = s.newObject(false)
314+
o.ID = *v.Key
315+
o.Name = s.getRelPath(*v.Key)
316+
o.Type = typ.ObjectTypeFile
318317

319318
o.SetSize(service.Int64Value(v.Size))
320319
o.SetUpdatedAt(convertUnixTimestampToTime(service.IntValue(v.Modified)))

0 commit comments

Comments
 (0)