Skip to content

Commit 149b295

Browse files
JRBANCELJRBANCEL
authored
Make the number of repository fetched at once configurable to handle large registries (#353)
Make the number of repositories fetched at once configurable to better handle large registries
1 parent 36804ec commit 149b295

File tree

5 files changed

+44
-26
lines changed

5 files changed

+44
-26
lines changed

‎README.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,18 @@ acr purge \
196196
--ago 30d \
197197
--concurrency 4
198198
```
199+
200+
#### Repository page size flag
201+
To control the number of repositories fetched in a single page, the `--repository-page-size` flag should be set. A default value of 100 will be used if `--repository-page-size` is not specified.
202+
This is useful when the number of artifacts in the registry is very large and listing too many repositories at once can timeout.
203+
```sh
204+
acr purge \
205+
--registry <Registry Name> \
206+
--filter <Repository Filter/Name>:<Regex Filter> \
207+
--ago 30d \
208+
--repository-page-size 10
209+
```
210+
199211
### Integration with ACR Tasks
200212

201213
To run a locally built version of the ACR-CLI using ACR Tasks follow these steps:

‎cmd/acr/annotate.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ func newAnnotateCmd(rootParams *rootParameters) *cobra.Command {
8282
}
8383

8484
// A map is used to collect the regex tags for every repository.
85-
tagFilters, err := common.CollectTagFilters(ctx, annotateParams.filters, acrClient.AutorestClient, annotateParams.filterTimeout)
85+
tagFilters, err := common.CollectTagFilters(ctx, annotateParams.filters, acrClient.AutorestClient, annotateParams.filterTimeout, defaultRepoPageSize)
8686
if err != nil {
8787
return err
8888
}

‎cmd/acr/purge.go

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,14 +46,19 @@ const (
4646
4747
- Delete all tags older than 1 day in the example.azurecr.io registry inside the hello-world repository, with 4 purge tasks running concurrently
4848
acr purge -r example --filter "hello-world:.*" --ago 1d --concurrency 4
49+
50+
- Delete all tags that are older than 7 days in the example.azurecr.io registry inside all repositories, with a page size of 50 repositories
51+
acr purge -r example --filter ".*:.*" --ago 7d --repository-page-size 50
4952
`
5053
maxPoolSize = 32 // The max number of parallel delete requests recommended by ACR server
5154
headerLink = "Link"
5255
)
5356

5457
var (
55-
defaultPoolSize = runtime.GOMAXPROCS(0)
56-
concurrencyDescription = fmt.Sprintf("Number of concurrent purge tasks. Range: [1 - %d]", maxPoolSize)
58+
defaultPoolSize = runtime.GOMAXPROCS(0)
59+
defaultRepoPageSize = int32(100)
60+
repoPageSizeDescription = "Number of repositories queried at once"
61+
concurrencyDescription = fmt.Sprintf("Number of concurrent purge tasks. Range: [1 - %d]", maxPoolSize)
5762
)
5863

5964
// Default settings for regexp2
@@ -71,6 +76,7 @@ type purgeParameters struct {
7176
untagged bool
7277
dryRun bool
7378
concurrency int
79+
repoPageSize int32
7480
}
7581

7682
// newPurgeCmd defines the purge command.
@@ -95,7 +101,7 @@ func newPurgeCmd(rootParams *rootParameters) *cobra.Command {
95101
return err
96102
}
97103
// A map is used to collect the regex tags for every repository.
98-
tagFilters, err := common.CollectTagFilters(ctx, purgeParams.filters, acrClient.AutorestClient, purgeParams.filterTimeout)
104+
tagFilters, err := common.CollectTagFilters(ctx, purgeParams.filters, acrClient.AutorestClient, purgeParams.filterTimeout, purgeParams.repoPageSize)
99105
if err != nil {
100106
return err
101107
}
@@ -162,6 +168,7 @@ func newPurgeCmd(rootParams *rootParameters) *cobra.Command {
162168
cmd.Flags().StringArrayVarP(&purgeParams.configs, "config", "c", nil, "Authentication config paths (e.g. C://Users/docker/config.json)")
163169
cmd.Flags().Int64Var(&purgeParams.filterTimeout, "filter-timeout-seconds", defaultRegexpMatchTimeoutSeconds, "This limits the evaluation of the regex filter, and will return a timeout error if this duration is exceeded during a single evaluation. If written incorrectly a regexp filter with backtracking can result in an infinite loop.")
164170
cmd.Flags().IntVar(&purgeParams.concurrency, "concurrency", defaultPoolSize, concurrencyDescription)
171+
cmd.Flags().Int32Var(&purgeParams.repoPageSize, "repository-page-size", defaultRepoPageSize, repoPageSizeDescription)
165172
cmd.Flags().BoolP("help", "h", false, "Print usage")
166173
cmd.MarkFlagRequired("filter")
167174
cmd.MarkFlagRequired("ago")

‎cmd/acr/purge_test.go

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -713,7 +713,7 @@ func TestCollectTagFilters(t *testing.T) {
713713
mockClient := &mocks.BaseClientAPI{}
714714
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
715715
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
716-
filters, err := common.CollectTagFilters(testCtx, []string{".+:.*-?local[.].+"}, mockClient, 60)
716+
filters, err := common.CollectTagFilters(testCtx, []string{".+:.*-?local[.].+"}, mockClient, 60, defaultRepoPageSize)
717717
assert.Equal(4, len(filters), "Number of found should be 4")
718718
assert.Equal(".*-?local[.].+", filters[testRepo], "Filter for test repo should be .*-?local[.].+")
719719
assert.Equal(".*-?local[.].+", filters["bar"], "Filter for bar repo should be .*-?local[.].+")
@@ -726,7 +726,7 @@ func TestCollectTagFilters(t *testing.T) {
726726
mockClient := &mocks.BaseClientAPI{}
727727
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
728728
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
729-
filters, err := common.CollectTagFilters(testCtx, []string{".+:.*-?local\\..+"}, mockClient, 60)
729+
filters, err := common.CollectTagFilters(testCtx, []string{".+:.*-?local\\..+"}, mockClient, 60, defaultRepoPageSize)
730730
assert.Equal(4, len(filters), "Number of found should be 4")
731731
assert.Equal(".*-?local\\..+", filters[testRepo], "Filter for test repo should be .*-?local\\..+")
732732
assert.Equal(".*-?local\\..+", filters["bar"], "Filter for bar repo should be .*-?local\\..+")
@@ -739,7 +739,7 @@ func TestCollectTagFilters(t *testing.T) {
739739
mockClient := &mocks.BaseClientAPI{}
740740
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
741741
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
742-
filters, err := common.CollectTagFilters(testCtx, []string{testRepo + ":.*"}, mockClient, 60)
742+
filters, err := common.CollectTagFilters(testCtx, []string{testRepo + ":.*"}, mockClient, 60, defaultRepoPageSize)
743743
assert.Equal(1, len(filters), "Number of found should be one")
744744
assert.Equal(".*", filters[testRepo], "Filter for test repo should be .*")
745745
assert.Equal(nil, err, "Error should be nil")
@@ -751,7 +751,7 @@ func TestCollectTagFilters(t *testing.T) {
751751
mockClient := &mocks.BaseClientAPI{}
752752
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
753753
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
754-
filters, err := common.CollectTagFilters(testCtx, []string{".*:.*"}, mockClient, 60)
754+
filters, err := common.CollectTagFilters(testCtx, []string{".*:.*"}, mockClient, 60, defaultRepoPageSize)
755755
assert.Equal(4, len(filters), "Number of found should be 4")
756756
assert.Equal(".*", filters[testRepo], "Filter for test repo should be .*")
757757
assert.Equal(".*", filters["bar"], "Filter for bar repo should be .*")
@@ -764,7 +764,7 @@ func TestCollectTagFilters(t *testing.T) {
764764
mockClient := &mocks.BaseClientAPI{}
765765
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
766766
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
767-
filters, err := common.CollectTagFilters(testCtx, []string{"ba:.*"}, mockClient, 60)
767+
filters, err := common.CollectTagFilters(testCtx, []string{"ba:.*"}, mockClient, 60, defaultRepoPageSize)
768768
assert.Equal(0, len(filters), "Number of found repos should be zero")
769769
assert.Equal(nil, err, "Error should be nil")
770770
mockClient.AssertExpectations(t)
@@ -775,7 +775,7 @@ func TestCollectTagFilters(t *testing.T) {
775775
mockClient := &mocks.BaseClientAPI{}
776776
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
777777
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
778-
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar:.*"}, mockClient, 60)
778+
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar:.*"}, mockClient, 60, defaultRepoPageSize)
779779
assert.Equal(1, len(filters), "Number of found repos should be one")
780780
assert.Equal(nil, err, "Error should be nil")
781781
mockClient.AssertExpectations(t)
@@ -786,7 +786,7 @@ func TestCollectTagFilters(t *testing.T) {
786786
mockClient := &mocks.BaseClientAPI{}
787787
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
788788
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
789-
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar:(?:.*)"}, mockClient, 60)
789+
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar:(?:.*)"}, mockClient, 60, defaultRepoPageSize)
790790
assert.Equal(1, len(filters), "Number of found repos should be one")
791791
assert.Equal(nil, err, "Error should be nil")
792792
mockClient.AssertExpectations(t)
@@ -797,7 +797,7 @@ func TestCollectTagFilters(t *testing.T) {
797797
mockClient := &mocks.BaseClientAPI{}
798798
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
799799
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
800-
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*):(?:.*)"}, mockClient, 60)
800+
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*):(?:.*)"}, mockClient, 60, defaultRepoPageSize)
801801
assert.Equal(1, len(filters), "Number of found repos should be one")
802802
assert.Equal(nil, err, "Error should be nil")
803803
mockClient.AssertExpectations(t)
@@ -808,7 +808,7 @@ func TestCollectTagFilters(t *testing.T) {
808808
mockClient := &mocks.BaseClientAPI{}
809809
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
810810
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
811-
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*)?:(?:.*)"}, mockClient, 60)
811+
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*)?:(?:.*)"}, mockClient, 60, defaultRepoPageSize)
812812
assert.Equal(1, len(filters), "Number of found repos should be one")
813813
assert.Equal(nil, err, "Error should be nil")
814814
mockClient.AssertExpectations(t)
@@ -819,7 +819,7 @@ func TestCollectTagFilters(t *testing.T) {
819819
mockClient := &mocks.BaseClientAPI{}
820820
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
821821
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
822-
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*):.(?:.*)"}, mockClient, 60)
822+
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*):.(?:.*)"}, mockClient, 60, defaultRepoPageSize)
823823
assert.Equal(1, len(filters), "Number of found repos should be one")
824824
assert.Equal(nil, err, "Error should be nil")
825825
mockClient.AssertExpectations(t)
@@ -830,7 +830,7 @@ func TestCollectTagFilters(t *testing.T) {
830830
mockClient := &mocks.BaseClientAPI{}
831831
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
832832
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
833-
filters, err := common.CollectTagFilters(testCtx, []string{"foo/b[[:alpha:]]r(?:.*):.(?:.*)"}, mockClient, 60)
833+
filters, err := common.CollectTagFilters(testCtx, []string{"foo/b[[:alpha:]]r(?:.*):.(?:.*)"}, mockClient, 60, defaultRepoPageSize)
834834
assert.Equal(1, len(filters), "Number of found repos should be one")
835835
assert.Equal(nil, err, "Error should be nil")
836836
mockClient.AssertExpectations(t)
@@ -840,7 +840,7 @@ func TestCollectTagFilters(t *testing.T) {
840840
assert := assert.New(t)
841841
mockClient := &mocks.BaseClientAPI{}
842842
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(NoRepositoriesResult, nil).Once()
843-
filters, err := common.CollectTagFilters(testCtx, []string{testRepo + ":.*"}, mockClient, 60)
843+
filters, err := common.CollectTagFilters(testCtx, []string{testRepo + ":.*"}, mockClient, 60, defaultRepoPageSize)
844844
assert.Equal(0, len(filters), "Number of found repos should be zero")
845845
assert.Equal(nil, err, "Error should be nil")
846846
mockClient.AssertExpectations(t)
@@ -851,7 +851,7 @@ func TestCollectTagFilters(t *testing.T) {
851851
mockClient := &mocks.BaseClientAPI{}
852852
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
853853
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
854-
_, err := common.CollectTagFilters(testCtx, []string{":.*"}, mockClient, 60)
854+
_, err := common.CollectTagFilters(testCtx, []string{":.*"}, mockClient, 60, defaultRepoPageSize)
855855
assert.NotEqual(nil, err, "Error should not be nil")
856856
mockClient.AssertExpectations(t)
857857
})
@@ -861,7 +861,7 @@ func TestCollectTagFilters(t *testing.T) {
861861
mockClient := &mocks.BaseClientAPI{}
862862
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
863863
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
864-
_, err := common.CollectTagFilters(testCtx, []string{testRepo + ".*:"}, mockClient, 60)
864+
_, err := common.CollectTagFilters(testCtx, []string{testRepo + ".*:"}, mockClient, 60, defaultRepoPageSize)
865865
assert.NotEqual(nil, err, "Error should not be nil")
866866
mockClient.AssertExpectations(t)
867867
})
@@ -873,7 +873,7 @@ func TestGetAllRepositoryNames(t *testing.T) {
873873
mockClient := &mocks.BaseClientAPI{}
874874
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
875875
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
876-
allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient)
876+
allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient, defaultRepoPageSize)
877877
assert.Equal(4, len(allRepoNames), "Number of all repo names should be 4")
878878
assert.Equal(nil, err, "Error should be nil")
879879
mockClient.AssertExpectations(t)
@@ -885,7 +885,7 @@ func TestGetAllRepositoryNames(t *testing.T) {
885885
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
886886
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(MoreRepositoriesResult, nil).Once()
887887
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
888-
allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient)
888+
allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient, defaultRepoPageSize)
889889
assert.Equal(7, len(allRepoNames), "Number of all repo names should be 7")
890890
assert.Equal(nil, err, "Error should be nil")
891891
mockClient.AssertExpectations(t)
@@ -895,7 +895,7 @@ func TestGetAllRepositoryNames(t *testing.T) {
895895
assert := assert.New(t)
896896
mockClient := &mocks.BaseClientAPI{}
897897
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(NoRepositoriesResult, nil).Once()
898-
allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient)
898+
allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient, defaultRepoPageSize)
899899
assert.Equal(0, len(allRepoNames), "Number of all repo names should be 7")
900900
assert.Equal(nil, err, "Error should be nil")
901901
mockClient.AssertExpectations(t)

‎cmd/common/image_functions.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,11 @@ const (
2929
mediaTypeArtifactManifest = "application/vnd.oci.artifact.manifest.v1+json"
3030
)
3131

32-
func GetAllRepositoryNames(ctx context.Context, client acrapi.BaseClientAPI) ([]string, error) {
32+
func GetAllRepositoryNames(ctx context.Context, client acrapi.BaseClientAPI, pageSize int32) ([]string, error) {
3333
allRepoNames := make([]string, 0)
3434
lastName := ""
35-
var batchSize int32 = 100
3635
for {
37-
repos, err := client.GetRepositories(ctx, lastName, &batchSize)
36+
repos, err := client.GetRepositories(ctx, lastName, &pageSize)
3837
if err != nil {
3938
return nil, err
4039
}
@@ -97,8 +96,8 @@ func GetRepositoryAndTagRegex(filter string) (string, string, error) {
9796
}
9897

9998
// CollectTagFilters collects all matching repos and collects the associated tag filters
100-
func CollectTagFilters(ctx context.Context, rawFilters []string, client acrapi.BaseClientAPI, regexMatchTimeout int64) (map[string]string, error) {
101-
allRepoNames, err := GetAllRepositoryNames(ctx, client)
99+
func CollectTagFilters(ctx context.Context, rawFilters []string, client acrapi.BaseClientAPI, regexMatchTimeout int64, repoPageSize int32) (map[string]string, error) {
100+
allRepoNames, err := GetAllRepositoryNames(ctx, client, repoPageSize)
102101
if err != nil {
103102
return nil, err
104103
}

0 commit comments

Comments
 (0)