Skip to content

Commit b655149

Browse files
stefanprodanstefanprodan
committed
bucket: Add prefix filtering capability
Signed-off-by: Stefan Prodan <[email protected]>
1 parent 3073c2c commit b655149

File tree

10 files changed

+48
-12
lines changed

10 files changed

+48
-12
lines changed

‎api/v1beta2/bucket_types.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323

2424
"github.com/fluxcd/pkg/apis/acl"
2525
"github.com/fluxcd/pkg/apis/meta"
26+
2627
apiv1 "github.com/fluxcd/source-controller/api/v1"
2728
)
2829

@@ -73,6 +74,10 @@ type BucketSpec struct {
7374
// +optional
7475
Region string `json:"region,omitempty"`
7576

77+
// Prefix to use for server-side filtering of files in the Bucket.
78+
// +optional
79+
Prefix string `json:"prefix,omitempty"`
80+
7681
// SecretRef specifies the Secret containing authentication credentials
7782
// for the Bucket.
7883
// +optional

‎config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,10 @@ spec:
331331
to ensure efficient use of resources.
332332
pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$
333333
type: string
334+
prefix:
335+
description: Prefix to use for server-side filtering of files in the
336+
Bucket.
337+
type: string
334338
provider:
335339
default: generic
336340
description: Provider of the object storage bucket. Defaults to 'generic',

‎docs/api/v1beta2/source.md

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,18 @@ string
138138
</tr>
139139
<tr>
140140
<td>
141+
<code>prefix</code><br>
142+
<em>
143+
string
144+
</em>
145+
</td>
146+
<td>
147+
<em>(Optional)</em>
148+
<p>Prefix to use for server-side filtering of files in the Bucket.</p>
149+
</td>
150+
</tr>
151+
<tr>
152+
<td>
141153
<code>secretRef</code><br>
142154
<em>
143155
<a href="https://pkg.go.dev/github.com/fluxcd/pkg/apis/meta#LocalObjectReference">
@@ -1422,6 +1434,18 @@ string
14221434
</tr>
14231435
<tr>
14241436
<td>
1437+
<code>prefix</code><br>
1438+
<em>
1439+
string
1440+
</em>
1441+
</td>
1442+
<td>
1443+
<em>(Optional)</em>
1444+
<p>Prefix to use for server-side filtering of files in the Bucket.</p>
1445+
</td>
1446+
</tr>
1447+
<tr>
1448+
<td>
14251449
<code>secretRef</code><br>
14261450
<em>
14271451
<a href="https://pkg.go.dev/github.com/fluxcd/pkg/apis/meta#LocalObjectReference">

‎internal/controller/bucket_controller.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ type BucketProvider interface {
145145
// bucket, calling visit for every item.
146146
// If the underlying client or the visit callback returns an error,
147147
// it returns early.
148-
VisitObjects(ctx context.Context, bucketName string, visit func(key, etag string) error) error
148+
VisitObjects(ctx context.Context, bucketName string, prefix string, visit func(key, etag string) error) error
149149
// ObjectIsNotFound returns true if the given error indicates an object
150150
// could not be found.
151151
ObjectIsNotFound(error) bool
@@ -742,7 +742,7 @@ func fetchEtagIndex(ctx context.Context, provider BucketProvider, obj *bucketv1.
742742
matcher := sourceignore.NewMatcher(ps)
743743

744744
// Build up index
745-
err = provider.VisitObjects(ctxTimeout, obj.Spec.BucketName, func(key, etag string) error {
745+
err = provider.VisitObjects(ctxTimeout, obj.Spec.BucketName, obj.Spec.Prefix, func(key, etag string) error {
746746
if strings.HasSuffix(key, "/") || key == sourceignore.IgnoreFile {
747747
return nil
748748
}

‎internal/controller/bucket_controller_fetch_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ func (m mockBucketClient) ObjectIsNotFound(e error) bool {
6969
return e == errMockNotFound
7070
}
7171

72-
func (m mockBucketClient) VisitObjects(_ context.Context, _ string, f func(key, etag string) error) error {
72+
func (m mockBucketClient) VisitObjects(_ context.Context, _ string, _ string, f func(key, etag string) error) error {
7373
for key, obj := range m.objects {
7474
if err := f(key, obj.etag); err != nil {
7575
return err

‎pkg/azure/blob.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ func (c *BlobClient) FGetObject(ctx context.Context, bucketName, objectName, loc
265265
// bucket, calling visit for every item.
266266
// If the underlying client or the visit callback returns an error,
267267
// it returns early.
268-
func (c *BlobClient) VisitObjects(ctx context.Context, bucketName string, visit func(path, etag string) error) error {
268+
func (c *BlobClient) VisitObjects(ctx context.Context, bucketName string, prefix string, visit func(path, etag string) error) error {
269269
items := c.NewListBlobsFlatPager(bucketName, nil)
270270
for items.More() {
271271
resp, err := items.NextPage(ctx)

‎pkg/gcp/gcp.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ func (c *GCSClient) FGetObject(ctx context.Context, bucketName, objectName, loca
165165
// bucket, calling visit for every item.
166166
// If the underlying client or the visit callback returns an error,
167167
// it returns early.
168-
func (c *GCSClient) VisitObjects(ctx context.Context, bucketName string, visit func(path, etag string) error) error {
168+
func (c *GCSClient) VisitObjects(ctx context.Context, bucketName string, prefix string, visit func(path, etag string) error) error {
169169
items := c.Client.Bucket(bucketName).Objects(ctx, nil)
170170
for {
171171
object, err := items.Next()

‎pkg/gcp/gcp_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ func TestVisitObjects(t *testing.T) {
170170
}
171171
keys := []string{}
172172
etags := []string{}
173-
err := gcpClient.VisitObjects(context.Background(), bucketName, func(key, etag string) error {
173+
err := gcpClient.VisitObjects(context.Background(), bucketName, "", func(key, etag string) error {
174174
keys = append(keys, key)
175175
etags = append(etags, etag)
176176
return nil
@@ -185,7 +185,7 @@ func TestVisitObjectsErr(t *testing.T) {
185185
Client: client,
186186
}
187187
badBucketName := "bad-bucket"
188-
err := gcpClient.VisitObjects(context.Background(), badBucketName, func(key, etag string) error {
188+
err := gcpClient.VisitObjects(context.Background(), badBucketName, "", func(key, etag string) error {
189189
return nil
190190
})
191191
assert.Error(t, err, fmt.Sprintf("listing objects from bucket '%s' failed: storage: bucket doesn't exist", badBucketName))
@@ -196,7 +196,7 @@ func TestVisitObjectsCallbackErr(t *testing.T) {
196196
Client: client,
197197
}
198198
mockErr := fmt.Errorf("mock")
199-
err := gcpClient.VisitObjects(context.Background(), bucketName, func(key, etag string) error {
199+
err := gcpClient.VisitObjects(context.Background(), bucketName, "", func(key, etag string) error {
200200
return mockErr
201201
})
202202
assert.Error(t, err, mockErr.Error())

‎pkg/minio/minio.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,9 +105,10 @@ func (c *MinioClient) FGetObject(ctx context.Context, bucketName, objectName, lo
105105
// bucket, calling visit for every item.
106106
// If the underlying client or the visit callback returns an error,
107107
// it returns early.
108-
func (c *MinioClient) VisitObjects(ctx context.Context, bucketName string, visit func(key, etag string) error) error {
108+
func (c *MinioClient) VisitObjects(ctx context.Context, bucketName string, prefix string, visit func(key, etag string) error) error {
109109
for object := range c.Client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{
110110
Recursive: true,
111+
Prefix: prefix,
111112
UseV1: s3utils.IsGoogleEndpoint(*c.Client.EndpointURL()),
112113
}) {
113114
if object.Err != nil {

‎pkg/minio/minio_test.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import (
3636

3737
"github.com/fluxcd/pkg/apis/meta"
3838
"github.com/fluxcd/pkg/sourceignore"
39+
3940
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
4041
)
4142

@@ -62,6 +63,7 @@ var (
6263

6364
var (
6465
bucketName = "test-bucket-minio" + uuid.New().String()
66+
prefix = ""
6567
secret = corev1.Secret{
6668
ObjectMeta: v1.ObjectMeta{
6769
Name: "minio-secret",
@@ -228,7 +230,7 @@ func TestFGetObjectNotExists(t *testing.T) {
228230
func TestVisitObjects(t *testing.T) {
229231
keys := []string{}
230232
etags := []string{}
231-
err := testMinioClient.VisitObjects(context.TODO(), bucketName, func(key, etag string) error {
233+
err := testMinioClient.VisitObjects(context.TODO(), bucketName, prefix, func(key, etag string) error {
232234
keys = append(keys, key)
233235
etags = append(etags, etag)
234236
return nil
@@ -241,15 +243,15 @@ func TestVisitObjects(t *testing.T) {
241243
func TestVisitObjectsErr(t *testing.T) {
242244
ctx := context.Background()
243245
badBucketName := "bad-bucket"
244-
err := testMinioClient.VisitObjects(ctx, badBucketName, func(string, string) error {
246+
err := testMinioClient.VisitObjects(ctx, badBucketName, prefix, func(string, string) error {
245247
return nil
246248
})
247249
assert.Error(t, err, fmt.Sprintf("listing objects from bucket '%s' failed: The specified bucket does not exist", badBucketName))
248250
}
249251

250252
func TestVisitObjectsCallbackErr(t *testing.T) {
251253
mockErr := fmt.Errorf("mock")
252-
err := testMinioClient.VisitObjects(context.TODO(), bucketName, func(key, etag string) error {
254+
err := testMinioClient.VisitObjects(context.TODO(), bucketName, prefix, func(key, etag string) error {
253255
return mockErr
254256
})
255257
assert.Error(t, err, mockErr.Error())

0 commit comments

Comments
 (0)