diff --git a/README.md b/README.md index 5cdb488b..50fac020 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,18 @@ acr purge \ --repository-page-size 10 ``` +#### Untag Limit flag +To control the maximum number of tags that can be untagged in a single purge operation, the `--untag-limit` flag should be set. A zero or negative value means no limit. A default value of 0 will be used if `--untag-limit` is not specified. + +This is useful when the number of untagged manifests is very large and untagging too many manifests at once can timeout. +```sh +acr purge \ + --registry \ + --filter : \ + --ago 30d \ + --untag-limit 10000 +``` + ### Integration with ACR Tasks To run a locally built version of the ACR-CLI using ACR Tasks follow these steps: diff --git a/cmd/acr/annotate.go b/cmd/acr/annotate.go index 9c4e47bf..4403a8be 100644 --- a/cmd/acr/annotate.go +++ b/cmd/acr/annotate.go @@ -287,7 +287,7 @@ func annotateUntaggedManifests(ctx context.Context, // Contrary to getTagsToAnnotate, getManifests gets all the manifests at once. // This was done because if there is a manifest that has no tag but is referenced by a multiarch manifest that has tags then it // should not be annotated. - manifestsToAnnotate, err := common.GetUntaggedManifests(ctx, acrClient, loginURL, repoName, dryRun, false) + manifestsToAnnotate, err := common.GetUntaggedManifests(ctx, acrClient, loginURL, repoName, dryRun, false, defaultUntagLimit) if err != nil { return -1, err } diff --git a/cmd/acr/purge.go b/cmd/acr/purge.go index 94b369a1..666096b3 100644 --- a/cmd/acr/purge.go +++ b/cmd/acr/purge.go @@ -49,6 +49,8 @@ const ( - Delete all tags that are older than 7 days in the example.azurecr.io registry inside all repositories, with a page size of 50 repositories acr purge -r example --filter ".*:.*" --ago 7d --repository-page-size 50 + - Delete all tags that are older than 7 days in the example.azurecr.io registry inside all repositories, with a maximum of 100 manifests untagged in a single request + acr purge -r example --filter ".*:.*" --ago 7d --untag-limit 10000 ` maxPoolSize = 32 // The max number of parallel delete requests recommended by ACR server headerLink = "Link" @@ -57,6 +59,7 @@ const ( var ( defaultPoolSize = runtime.GOMAXPROCS(0) defaultRepoPageSize = int32(100) + defaultUntagLimit = 0 repoPageSizeDescription = "Number of repositories queried at once" concurrencyDescription = fmt.Sprintf("Number of concurrent purge tasks. Range: [1 - %d]", maxPoolSize) ) @@ -77,6 +80,7 @@ type purgeParameters struct { dryRun bool concurrency int repoPageSize int32 + UntagLimit int } // newPurgeCmd defines the purge command. @@ -130,7 +134,7 @@ func newPurgeCmd(rootParams *rootParameters) *cobra.Command { singleDeletedManifestsCount := 0 // If the untagged flag is set then also manifests are deleted. if purgeParams.untagged { - singleDeletedManifestsCount, err = purgeDanglingManifests(ctx, acrClient, poolSize, loginURL, repoName) + singleDeletedManifestsCount, err = purgeDanglingManifests(ctx, acrClient, poolSize, loginURL, repoName, purgeParams.UntagLimit) if err != nil { return errors.Wrap(err, "failed to purge manifests") } @@ -169,6 +173,7 @@ func newPurgeCmd(rootParams *rootParameters) *cobra.Command { cmd.Flags().Int64Var(&purgeParams.filterTimeout, "filter-timeout-seconds", defaultRegexpMatchTimeoutSeconds, "This limits the evaluation of the regex filter, and will return a timeout error if this duration is exceeded during a single evaluation. If written incorrectly a regexp filter with backtracking can result in an infinite loop.") cmd.Flags().IntVar(&purgeParams.concurrency, "concurrency", defaultPoolSize, concurrencyDescription) cmd.Flags().Int32Var(&purgeParams.repoPageSize, "repository-page-size", defaultRepoPageSize, repoPageSizeDescription) + cmd.Flags().IntVar(&purgeParams.UntagLimit, "untag-limit", defaultUntagLimit, "The maximum number of manifests to untag in a single requess.") cmd.Flags().BoolP("help", "h", false, "Print usage") cmd.MarkFlagRequired("filter") cmd.MarkFlagRequired("ago") @@ -318,12 +323,12 @@ func getTagsToDelete(ctx context.Context, // purgeDanglingManifests deletes all manifests that do not have any tags associated with them. // except the ones that are referenced by a multiarch manifest or that have subject. -func purgeDanglingManifests(ctx context.Context, acrClient api.AcrCLIClientInterface, poolSize int, loginURL string, repoName string) (int, error) { +func purgeDanglingManifests(ctx context.Context, acrClient api.AcrCLIClientInterface, poolSize int, loginURL string, repoName string, limit int) (int, error) { fmt.Printf("Deleting manifests for repository: %s\n", repoName) // Contrary to getTagsToDelete, getManifestsToDelete gets all the Manifests at once, this was done because if there is a manifest that has no // tag but is referenced by a multiarch manifest that has tags then it should not be deleted. Or if a manifest has no tag, but it has subject, // then it should not be deleted. - manifestsToDelete, err := common.GetUntaggedManifests(ctx, acrClient, loginURL, repoName, false, true) + manifestsToDelete, err := common.GetUntaggedManifests(ctx, acrClient, loginURL, repoName, false, true, limit) if err != nil { return -1, err } diff --git a/cmd/acr/purge_test.go b/cmd/acr/purge_test.go index a9f8c6e1..0e9522b3 100644 --- a/cmd/acr/purge_test.go +++ b/cmd/acr/purge_test.go @@ -282,7 +282,7 @@ func TestPurgeManifests(t *testing.T) { assert := assert.New(t) mockClient := &mocks.AcrCLIClientInterface{} mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "").Return(notFoundManifestResponse, errors.New("testRepo not found")).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(0, deletedTags, "Number of deleted elements should be 0") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -293,7 +293,7 @@ func TestPurgeManifests(t *testing.T) { assert := assert.New(t) mockClient := &mocks.AcrCLIClientInterface{} mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "").Return(nil, errors.New("unauthorized")).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(-1, deletedTags, "Number of deleted elements should be -1") assert.NotEqual(nil, err, "Error should not be nil") mockClient.AssertExpectations(t) @@ -306,7 +306,7 @@ func TestPurgeManifests(t *testing.T) { mockClient := &mocks.AcrCLIClientInterface{} mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "").Return(singleManifestV2WithTagsResult, nil).Once() mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "sha256:2830cc0fcddc1bc2bd4aeab0ed5ee7087dab29a49e65151c77553e46a7ed5283").Return(EmptyListManifestsResult, nil).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(0, deletedTags, "Number of deleted elements should be 0") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -318,7 +318,7 @@ func TestPurgeManifests(t *testing.T) { mockClient := &mocks.AcrCLIClientInterface{} mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "").Return(singleManifestV2WithTagsResult, nil).Once() mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "sha256:2830cc0fcddc1bc2bd4aeab0ed5ee7087dab29a49e65151c77553e46a7ed5283").Return(nil, errors.New("error getting manifests")).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(-1, deletedTags, "Number of deleted elements should be -1") assert.NotEqual(nil, err, "Error should not be nil") mockClient.AssertExpectations(t) @@ -331,7 +331,7 @@ func TestPurgeManifests(t *testing.T) { mockClient := &mocks.AcrCLIClientInterface{} mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "").Return(singleMultiArchManifestV2WithTagsResult, nil).Once() mockClient.On("GetManifest", mock.Anything, testRepo, "sha256:d88fb54ba4424dada7c928c6af332ed1c49065ad85eafefb6f26664695015119").Return(nil, errors.New("error getting manifest")).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(-1, deletedTags, "Number of deleted elements should be -1") assert.NotEqual(nil, err, "Error not should be nil") mockClient.AssertExpectations(t) @@ -343,7 +343,7 @@ func TestPurgeManifests(t *testing.T) { mockClient := &mocks.AcrCLIClientInterface{} mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "").Return(singleMultiArchManifestV2WithTagsResult, nil).Once() mockClient.On("GetManifest", mock.Anything, testRepo, "sha256:d88fb54ba4424dada7c928c6af332ed1c49065ad85eafefb6f26664695015119").Return([]byte("invalid manifest"), nil).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(-1, deletedTags, "Number of deleted elements should be -1") assert.NotEqual(nil, err, "Error not should be nil") mockClient.AssertExpectations(t) @@ -360,7 +360,7 @@ func TestPurgeManifests(t *testing.T) { mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "sha256:6305e31b9b0081d2532397a1e08823f843f329a7af2ac98cb1d7f0355a3e3696").Return(EmptyListManifestsResult, nil).Once() mockClient.On("DeleteManifest", mock.Anything, testRepo, "sha256:63532043b5af6247377a472ad075a42bde35689918de1cf7f807714997e0e683").Return(nil, nil).Once() mockClient.On("DeleteManifest", mock.Anything, testRepo, "sha256:6305e31b9b0081d2532397a1e08823f843f329a7af2ac98cb1d7f0355a3e3696").Return(nil, nil).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(2, deletedTags, "Number of deleted elements should be 2") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -376,7 +376,7 @@ func TestPurgeManifests(t *testing.T) { mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "sha256:6305e31b9b0081d2532397a1e08823f843f329a7af2ac98cb1d7f0355a3e3696").Return(EmptyListManifestsResult, nil).Once() mockClient.On("DeleteManifest", mock.Anything, testRepo, "sha256:63532043b5af6247377a472ad075a42bde35689918de1cf7f807714997e0e683").Return(nil, nil).Once() mockClient.On("DeleteManifest", mock.Anything, testRepo, "sha256:6305e31b9b0081d2532397a1e08823f843f329a7af2ac98cb1d7f0355a3e3696").Return(¬FoundResponse, errors.New("manifest not found")).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(2, deletedTags, "Number of deleted elements should be 2") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -391,7 +391,7 @@ func TestPurgeManifests(t *testing.T) { mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "sha256:6305e31b9b0081d2532397a1e08823f843f329a7af2ac98cb1d7f0355a3e3696").Return(EmptyListManifestsResult, nil).Once() mockClient.On("DeleteManifest", mock.Anything, testRepo, "sha256:63532043b5af6247377a472ad075a42bde35689918de1cf7f807714997e0e683").Return(nil, errors.New("error deleting manifest")).Once() mockClient.On("DeleteManifest", mock.Anything, testRepo, "sha256:6305e31b9b0081d2532397a1e08823f843f329a7af2ac98cb1d7f0355a3e3696").Return(nil, nil).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(-1, deletedTags, "Number of deleted elements should be -1") assert.NotEqual(nil, err, "Error should not be nil") mockClient.AssertExpectations(t) @@ -407,7 +407,7 @@ func TestPurgeManifests(t *testing.T) { mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "sha256:6305e31b9b0081d2532397a1e08823f843f329a7af2ac98cb1d7f0355a3e3696").Return(EmptyListManifestsResult, nil).Once() mockClient.On("DeleteManifest", mock.Anything, testRepo, "sha256:63532043b5af6247377a472ad075a42bde35689918de1cf7f807714997e0e683").Return(nil, nil).Once() mockClient.On("DeleteManifest", mock.Anything, testRepo, "sha256:6305e31b9b0081d2532397a1e08823f843f329a7af2ac98cb1d7f0355a3e3696").Return(nil, errors.New("error deleting manifest")).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(-1, deletedTags, "Number of deleted elements should be -1") assert.NotEqual(nil, err, "Error should not be nil") mockClient.AssertExpectations(t) @@ -424,7 +424,7 @@ func TestPurgeManifests(t *testing.T) { mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "sha256:d88fb54ba4424dada7c928c6af332ed1c49065ad85eafefb6f26664695015119").Return(doubleManifestV2WithoutTagsResult, nil).Once() mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "sha256:6305e31b9b0081d2532397a1e08823f843f329a7af2ac98cb1d7f0355a3e3696").Return(EmptyListManifestsResult, nil).Once() mockClient.On("DeleteManifest", mock.Anything, testRepo, "sha256:6305e31b9b0081d2532397a1e08823f843f329a7af2ac98cb1d7f0355a3e3696").Return(nil, nil).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(1, deletedTags, "Number of deleted elements should be 1") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -442,7 +442,7 @@ func TestPurgeManifests(t *testing.T) { mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "sha256:d88fb54ba4424dada7c928c6af332ed1c49065ad85eafefb6f26664695015119").Return(doubleOCIWithoutTagsResult, nil).Once() mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "sha256:6305e31b9b0081d2532397a1e08823f843f329a7af2ac98cb1d7f0355a3e3696").Return(EmptyListManifestsResult, nil).Once() mockClient.On("DeleteManifest", mock.Anything, testRepo, "sha256:6305e31b9b0081d2532397a1e08823f843f329a7af2ac98cb1d7f0355a3e3696").Return(nil, nil).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(1, deletedTags, "Number of deleted elements should be 1") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -455,7 +455,7 @@ func TestPurgeManifests(t *testing.T) { mockClient := &mocks.AcrCLIClientInterface{} mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "").Return(deleteDisabledOneManifestResult, nil).Once() mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", digest).Return(EmptyListManifestsResult, nil).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(0, deletedTags, "Number of deleted elements should be 0") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -468,7 +468,7 @@ func TestPurgeManifests(t *testing.T) { mockClient := &mocks.AcrCLIClientInterface{} mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "").Return(writeDisabledOneManifestResult, nil).Once() mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", digest).Return(EmptyListManifestsResult, nil).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(0, deletedTags, "Number of deleted elements should be 0") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -481,7 +481,7 @@ func TestPurgeManifests(t *testing.T) { mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "").Return(singleManifestWithSubjectWithoutTagResult, nil).Once() mockClient.On("GetManifest", mock.Anything, testRepo, "sha256:118811b833e6ca4f3c65559654ca6359410730e97c719f5090d0bfe4db0ab588").Return(manifestWithSubjectOCIArtificate, nil).Once() mockClient.On("GetAcrManifests", mock.Anything, testRepo, "", "sha256:118811b833e6ca4f3c65559654ca6359410730e97c719f5090d0bfe4db0ab588").Return(EmptyListManifestsResult, nil).Once() - deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo) + deletedTags, err := purgeDanglingManifests(testCtx, mockClient, defaultPoolSize, testLoginURL, testRepo, defaultUntagLimit) assert.Equal(0, deletedTags, "Number of deleted elements should be 0") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) diff --git a/cmd/common/image_functions.go b/cmd/common/image_functions.go index e3179caf..f776d5c0 100644 --- a/cmd/common/image_functions.go +++ b/cmd/common/image_functions.go @@ -149,7 +149,7 @@ func GetLastTagFromResponse(resultTags *acr.RepositoryTagsType) string { // GetUntaggedManifests gets all the manifests for the command to be executed on. The command will be executed on this manifest if it does not // have any tag and does not form part of a manifest list that has tags referencing it. If the purge command is to be executed, // the manifest should also not have a tag and not have a subject manifest. -func GetUntaggedManifests(ctx context.Context, acrClient api.AcrCLIClientInterface, loginURL string, repoName string, dryRun bool, ignoreReferrerManifests bool) (*[]string, error) { +func GetUntaggedManifests(ctx context.Context, acrClient api.AcrCLIClientInterface, loginURL string, repoName string, dryRun bool, ignoreReferrerManifests bool, UntagLimit int) (*[]string, error) { lastManifestDigest := "" var manifestsForCommand []string resultManifests, err := acrClient.GetAcrManifests(ctx, repoName, "", lastManifestDigest) @@ -196,6 +196,10 @@ func GetUntaggedManifests(ctx context.Context, acrClient api.AcrCLIClientInterfa if err != nil { return nil, err } + if UntagLimit > 0 && len(candidates) >= UntagLimit { + // If the number of candidates is greater than the limit, we stop fetching more manifests. + break + } } // Remove all manifests that should not be deleted for i := 0; i < len(candidates); i++ {