Skip to content

Commit c7a913c

Browse files
berndverstberndverst
authored
Minor Az Storage Binding improvements (#3126)
Signed-off-by: Bernd Verst <[email protected]>
1 parent fb4d55b commit c7a913c

File tree

10 files changed

+88
-67
lines changed

10 files changed

+88
-67
lines changed

‎.build-tools/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ require (
1313
)
1414

1515
require (
16-
github.com/dapr/kit v0.11.4-0.20230807225040-b6b141aa3e32 // indirect
16+
github.com/dapr/kit v0.12.1 // indirect
1717
github.com/gogo/protobuf v1.3.2 // indirect
1818
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
1919
github.com/inconshreveable/mousetrap v1.0.1 // indirect

‎.build-tools/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
2-
github.com/dapr/kit v0.11.4-0.20230807225040-b6b141aa3e32 h1:ctqg12Eqd9SE88pwlJTupsIJowtaQcyH4s5xRMoweUU=
3-
github.com/dapr/kit v0.11.4-0.20230807225040-b6b141aa3e32/go.mod h1:bRyy09hwD03VEJ6b4weVNZWDjC5K3XY1+oGI5JCdZnU=
2+
github.com/dapr/kit v0.12.1 h1:XT0CJQQaKRYSzIzZo15O1PAHGUrMGoAavdFRcNVZ+UE=
3+
github.com/dapr/kit v0.12.1/go.mod h1:eNYjsudq3Ij0x8CLWsPturHor56sZRNu5tk2hUiJT80=
44
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
55
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
66
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=

‎bindings/azure/blobstorage/blobstorage.go‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ const (
4646
metadataKeyMarker = "marker"
4747
// The number of blobs that will be returned in a list operation.
4848
metadataKeyNumber = "number"
49+
// Defines the response metadata key for the number of pages traversed in a list response.
50+
metadataKeyPagesTraversed = "pagesTraversed"
4951
// Defines if the user defined metadata should be returned in the get operation.
5052
metadataKeyIncludeMetadata = "includeMetadata"
5153
// Defines the delete snapshots option for the delete operation.
@@ -295,30 +297,34 @@ func (a *AzureBlobStorage) list(ctx context.Context, req *bindings.InvokeRequest
295297
}
296298
options.Marker = &initialMarker
297299

298-
metadata := map[string]string{}
300+
metadata := make(map[string]string, 3)
299301
blobs := []*container.BlobItem{}
300302
pager := a.containerClient.NewListBlobsFlatPager(&options)
301303

304+
metadata[metadataKeyMarker] = ""
305+
numBlobs := 0
306+
pagesTraversed := 0
302307
for pager.More() {
303308
resp, err := pager.NextPage(ctx)
304309
if err != nil {
305310
return nil, fmt.Errorf("error listing blobs: %w", err)
306311
}
312+
pagesTraversed++
307313

308314
blobs = append(blobs, resp.Segment.BlobItems...)
309-
numBlobs := len(blobs)
310-
metadata[metadataKeyNumber] = strconv.FormatInt(int64(numBlobs), 10)
311-
metadata[metadataKeyMarker] = ""
315+
numBlobs += len(resp.Segment.BlobItems)
312316
if resp.Marker != nil {
313317
metadata[metadataKeyMarker] = *resp.Marker
318+
} else {
319+
metadata[metadataKeyMarker] = ""
314320
}
315321

316-
if *options.MaxResults-maxResults > 0 {
317-
*options.MaxResults -= maxResults
318-
} else {
322+
if numBlobs >= int(*options.MaxResults) {
319323
break
320324
}
321325
}
326+
metadata[metadataKeyNumber] = strconv.FormatInt(int64(numBlobs), 10)
327+
metadata[metadataKeyPagesTraversed] = strconv.FormatInt(int64(pagesTraversed), 10)
322328

323329
jsonResponse, err := json.Marshal(blobs)
324330
if err != nil {

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ require (
5151
github.com/cloudwego/kitex-examples v0.1.1
5252
github.com/cyphar/filepath-securejoin v0.2.3
5353
github.com/dancannon/gorethink v4.0.0+incompatible
54-
github.com/dapr/kit v0.11.4-0.20230807225040-b6b141aa3e32
54+
github.com/dapr/kit v0.12.1
5555
github.com/didip/tollbooth/v7 v7.0.1
5656
github.com/eclipse/paho.mqtt.golang v1.4.3
5757
github.com/fasthttp-contrib/sessions v0.0.0-20160905201309-74f6ac73d5d5

‎go.sum‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -777,8 +777,8 @@ github.com/dancannon/gorethink v4.0.0+incompatible h1:KFV7Gha3AuqT+gr0B/eKvGhbjm
777777
github.com/dancannon/gorethink v4.0.0+incompatible/go.mod h1:BLvkat9KmZc1efyYwhz3WnybhRZtgF1K929FD8z1avU=
778778
github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0=
779779
github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0=
780-
github.com/dapr/kit v0.11.4-0.20230807225040-b6b141aa3e32 h1:ctqg12Eqd9SE88pwlJTupsIJowtaQcyH4s5xRMoweUU=
781-
github.com/dapr/kit v0.11.4-0.20230807225040-b6b141aa3e32/go.mod h1:bRyy09hwD03VEJ6b4weVNZWDjC5K3XY1+oGI5JCdZnU=
780+
github.com/dapr/kit v0.12.1 h1:XT0CJQQaKRYSzIzZo15O1PAHGUrMGoAavdFRcNVZ+UE=
781+
github.com/dapr/kit v0.12.1/go.mod h1:eNYjsudq3Ij0x8CLWsPturHor56sZRNu5tk2hUiJT80=
782782
github.com/dave/jennifer v1.4.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
783783
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
784784
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -3010,4 +3010,4 @@ sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
30103010
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
30113011
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
30123012
stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=
3013-
stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0=
3013+
stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0=

‎tests/certification/bindings/azure/blobstorage/blobstorage_test.go‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -81,12 +81,12 @@ func listBlobRequest(ctx flow.Context, client daprsdk.Client, prefix string, mar
8181
requestOptions["maxResults"] = maxResults
8282
}
8383
includeOptions := make(map[string]interface{})
84-
includeOptions["Snapshots"] = includeSnapshots
85-
includeOptions["UncommittedBlobs"] = includeUncommittedBlobs
86-
includeOptions["Copy"] = includeCopy
87-
includeOptions["Deleted"] = includeDeleted
88-
includeOptions["Metadata"] = includeMetadata
89-
requestOptions["Include"] = includeOptions
84+
includeOptions["snapshots"] = includeSnapshots
85+
includeOptions["uncommittedBlobs"] = includeUncommittedBlobs
86+
includeOptions["copy"] = includeCopy
87+
includeOptions["deleted"] = includeDeleted
88+
includeOptions["metadata"] = includeMetadata
89+
requestOptions["include"] = includeOptions
9090

9191
optionsBytes, marshalErr := json.Marshal(requestOptions)
9292
if marshalErr != nil {
@@ -515,17 +515,28 @@ func TestBlobStorage(t *testing.T) {
515515
assert.Contains(t, output[0]["Name"], "prefixA")
516516

517517
nextMarker := out.Metadata["marker"]
518+
assert.Empty(t, nextMarker)
518519

519-
// list the contents of the container with a marker.
520-
out2, listErr2 := listBlobRequest(ctx, client, "prefixA", nextMarker, 1, false, false, false, false, false)
521-
assert.NoError(t, listErr2)
520+
assert.Equal(t, "1", out.Metadata["pagesTraversed"])
521+
assert.Equal(t, "1", out.Metadata["number"])
522+
523+
// Commenting this out for now. We do not have enough data to for a second page of results, so cannot test this.
524+
525+
// // list the contents of the container with a marker.
526+
// out2, listErr2 := listBlobRequest(ctx, client, "prefix", nextMarker, 1, false, false, false, false, false)
527+
// assert.NoError(t, listErr2)
528+
529+
// var output2 []map[string]interface{}
530+
// err2 := json.Unmarshal(out2.Data, &output2)
531+
// assert.NoError(t, err2)
532+
533+
// assert.Equal(t, 1, len(output2))
534+
// assert.Contains(t, output2[0]["Name"], "prefixA")
522535

523-
var output2 []map[string]interface{}
524-
err2 := json.Unmarshal(out2.Data, &output2)
525-
assert.NoError(t, err2)
536+
// nextMarker2 := out2.Metadata["marker"]
537+
// assert.Empty(t, nextMarker2)
526538

527-
assert.Equal(t, 1, len(output2))
528-
assert.Contains(t, output2[0]["Name"], "prefixA")
539+
// assert.Equal(t, "1", out2.Metadata["pagesTraversed"])
529540

530541
// cleanup.
531542
_, invokeDeleteErr1 := deleteBlobRequest(ctx, client, "prefixA/filename.txt", nil)

‎tests/certification/go.mod‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,10 @@ require (
1616
github.com/cenkalti/backoff/v4 v4.2.1
1717
github.com/cloudwego/kitex v0.5.0
1818
github.com/cloudwego/kitex-examples v0.1.1
19-
github.com/dapr/components-contrib v1.11.3-0.20230825194223-81539d0b5377
20-
github.com/dapr/dapr v1.11.3-0.20230828162238-cc25c181bc45
19+
github.com/dapr/components-contrib v1.12.0-rc.1
20+
github.com/dapr/dapr v1.12.0-rc.1
2121
github.com/dapr/go-sdk v1.8.0
22-
github.com/dapr/kit v0.11.4-0.20230807225040-b6b141aa3e32
22+
github.com/dapr/kit v0.12.1
2323
github.com/eclipse/paho.mqtt.golang v1.4.3
2424
github.com/go-chi/chi/v5 v5.0.10
2525
github.com/go-redis/redis/v8 v8.11.5
@@ -36,7 +36,7 @@ require (
3636
go.mongodb.org/mongo-driver v1.12.1
3737
go.uber.org/multierr v1.11.0
3838
go.uber.org/ratelimit v0.3.0
39-
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
39+
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
4040
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
4141
modernc.org/sqlite v1.25.0
4242
)
@@ -81,7 +81,7 @@ require (
8181
github.com/beorn7/perks v1.0.1 // indirect
8282
github.com/bits-and-blooms/bitset v1.4.0 // indirect
8383
github.com/bradfitz/gomemcache v0.0.0-20230611145640-acc696258285 // indirect
84-
github.com/bufbuild/protocompile v0.4.0 // indirect
84+
github.com/bufbuild/protocompile v0.6.0 // indirect
8585
github.com/bytedance/gopkg v0.0.0-20220817015305-b879a72dc90f // indirect
8686
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
8787
github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -109,7 +109,7 @@ require (
109109
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
110110
github.com/eapache/queue v1.1.0 // indirect
111111
github.com/emicklei/go-restful/v3 v3.10.2 // indirect
112-
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
112+
github.com/evanphx/json-patch/v5 v5.7.0 // indirect
113113
github.com/fatih/color v1.15.0 // indirect
114114
github.com/fsnotify/fsnotify v1.6.0 // indirect
115115
github.com/go-chi/cors v1.2.1 // indirect
@@ -172,7 +172,7 @@ require (
172172
github.com/jcmturner/gofork v1.7.6 // indirect
173173
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
174174
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
175-
github.com/jhump/protoreflect v1.15.1 // indirect
175+
github.com/jhump/protoreflect v1.15.2 // indirect
176176
github.com/jinzhu/copier v0.3.5 // indirect
177177
github.com/jmespath/go-jmespath v0.4.0 // indirect
178178
github.com/josharian/intern v1.0.0 // indirect
@@ -197,7 +197,7 @@ require (
197197
github.com/mattn/go-colorable v0.1.13 // indirect
198198
github.com/mattn/go-isatty v0.0.19 // indirect
199199
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
200-
github.com/microsoft/durabletask-go v0.3.0 // indirect
200+
github.com/microsoft/durabletask-go v0.3.1 // indirect
201201
github.com/microsoft/go-mssqldb v1.5.0 // indirect
202202
github.com/miekg/dns v1.1.50 // indirect
203203
github.com/mitchellh/copystructure v1.2.0 // indirect
@@ -269,16 +269,16 @@ require (
269269
go.uber.org/atomic v1.10.0 // indirect
270270
go.uber.org/zap v1.24.0 // indirect
271271
golang.org/x/arch v0.3.0 // indirect
272-
golang.org/x/crypto v0.12.0 // indirect
272+
golang.org/x/crypto v0.13.0 // indirect
273273
golang.org/x/mod v0.12.0 // indirect
274-
golang.org/x/net v0.14.0 // indirect
274+
golang.org/x/net v0.15.0 // indirect
275275
golang.org/x/oauth2 v0.11.0 // indirect
276276
golang.org/x/sync v0.3.0 // indirect
277-
golang.org/x/sys v0.11.0 // indirect
278-
golang.org/x/term v0.11.0 // indirect
279-
golang.org/x/text v0.12.0 // indirect
277+
golang.org/x/sys v0.12.0 // indirect
278+
golang.org/x/term v0.12.0 // indirect
279+
golang.org/x/text v0.13.0 // indirect
280280
golang.org/x/time v0.3.0 // indirect
281-
golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
281+
golang.org/x/tools v0.13.0 // indirect
282282
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
283283
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
284284
google.golang.org/api v0.138.0 // indirect

0 commit comments

Comments
 (0)