Skip to content

feat: added exclude filter for purge command #428

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: added exclude filter for purge command #428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,18 @@ Examples of filters
| Untag all tags that are older than the duration in app repository | --filter `"app:.*"` |
| Untag all tags that are older than the duration in all repositories | --filter `".*:.*"` |

##### Exclude filter tag
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can keep this at 4#


To exclude tags from the purging, use the --exclude-filter parameter. Exclude filters take precedence over filters.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you note that these are optional?


```sh
acr purge \
--registry <Registry Name> \
--filter <Repository Name>:<Regex filter> \
--exclude-filter <Regex exclude>
...
```

#### Ago flag

The ago flag can be used to change the default expiration time of a tag, for example, the following command would purge all tags that are older than 30 days:
Expand Down
65 changes: 58 additions & 7 deletions cmd/acr/purge.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ const (

- Delete all tags that are older than 7 days in the example.azurecr.io registry inside all repositories, with a page size of 50 repositories
acr purge -r example --filter ".*:.*" --ago 7d --repository-page-size 50

- Delete all tags that starts with hello but not if the tag is a production tag ending on -prod
acr purge -r example --filter "hello-world:hello.*" --exclude-filter ".*-prod$"
`
maxPoolSize = 32 // The max number of parallel delete requests recommended by ACR server
headerLink = "Link"
Expand All @@ -73,6 +76,7 @@ type purgeParameters struct {
keep int
filters []string
filterTimeout int64
excludeFilter string
untagged bool
dryRun bool
concurrency int
Expand Down Expand Up @@ -123,7 +127,7 @@ func newPurgeCmd(rootParams *rootParameters) *cobra.Command {
fmt.Printf("Specified concurrency value too large. Set to maximum value: %d \n", maxPoolSize)
}

singleDeletedTagsCount, err := purgeTags(ctx, acrClient, poolSize, loginURL, repoName, purgeParams.ago, tagRegex, purgeParams.keep, purgeParams.filterTimeout)
singleDeletedTagsCount, err := purgeTags(ctx, acrClient, poolSize, loginURL, repoName, purgeParams.ago, tagRegex, purgeParams.keep, purgeParams.filterTimeout, purgeParams.excludeFilter)
if err != nil {
return errors.Wrap(err, "failed to purge tags")
}
Expand All @@ -140,7 +144,7 @@ func newPurgeCmd(rootParams *rootParameters) *cobra.Command {
deletedManifestsCount += singleDeletedManifestsCount
} else {
// No tag or manifest will be deleted but the counters still will be updated.
singleDeletedTagsCount, singleDeletedManifestsCount, err := dryRunPurge(ctx, acrClient, loginURL, repoName, purgeParams.ago, tagRegex, purgeParams.untagged, purgeParams.keep, purgeParams.filterTimeout)
singleDeletedTagsCount, singleDeletedManifestsCount, err := dryRunPurge(ctx, acrClient, loginURL, repoName, purgeParams.ago, tagRegex, purgeParams.untagged, purgeParams.keep, purgeParams.filterTimeout, purgeParams.excludeFilter)
if err != nil {
return errors.Wrap(err, "failed to dry-run purge")
}
Expand All @@ -165,6 +169,7 @@ func newPurgeCmd(rootParams *rootParameters) *cobra.Command {
cmd.Flags().StringVar(&purgeParams.ago, "ago", "", "The tags that were last updated before this duration will be deleted, the format is [number]d[string] where the first number represents an amount of days and the string is in a Go duration format (e.g. 2d3h6m selects images older than 2 days, 3 hours and 6 minutes)")
cmd.Flags().IntVar(&purgeParams.keep, "keep", 0, "Number of latest to-be-deleted tags to keep, use this when you want to keep at least x number of latest tags that could be deleted meeting all other filter criteria")
cmd.Flags().StringArrayVarP(&purgeParams.filters, "filter", "f", nil, "Specify the repository and a regular expression filter for the tag name, if a tag matches the filter and is older than the duration specified in ago it will be deleted. Note: If backtracking is used in the regexp it's possible for the expression to run into an infinite loop. The default timeout is set to 1 minute for evaluation of any filter expression. Use the '--filter-timeout-seconds' option to set a different value.")
cmd.Flags().StringVar(&purgeParams.excludeFilter, "exclude-filter", "", "Specify a regular expression filter for the tag name (without repository!) to be excluded from purge.")
cmd.Flags().StringArrayVarP(&purgeParams.configs, "config", "c", nil, "Authentication config paths (e.g. C://Users/docker/config.json)")
cmd.Flags().Int64Var(&purgeParams.filterTimeout, "filter-timeout-seconds", defaultRegexpMatchTimeoutSeconds, "This limits the evaluation of the regex filter, and will return a timeout error if this duration is exceeded during a single evaluation. If written incorrectly a regexp filter with backtracking can result in an infinite loop.")
cmd.Flags().IntVar(&purgeParams.concurrency, "concurrency", defaultPoolSize, concurrencyDescription)
Expand All @@ -176,8 +181,19 @@ func newPurgeCmd(rootParams *rootParameters) *cobra.Command {
}

// purgeTags deletes all tags that are older than the ago value and that match the tagFilter string.
func purgeTags(ctx context.Context, acrClient api.AcrCLIClientInterface, poolSize int, loginURL string, repoName string, ago string, tagFilter string, keep int, regexpMatchTimeoutSeconds int64) (int, error) {
func purgeTags(ctx context.Context, acrClient api.AcrCLIClientInterface, poolSize int, loginURL string, repoName string, ago string, tagFilter string, keep int, regexpMatchTimeoutSeconds int64, excludeFilter string) (int, error) {
fmt.Printf("Deleting tags for repository: %s\n", repoName)

var excludeRegex *regexp2.Regexp = nil

if excludeFilter != "" {
res, err := common.BuildRegexFilter(excludeFilter, regexpMatchTimeoutSeconds)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add an error wrap, just to make it clearer? Like fmt.Errorf("could not build excludeFilter regex: %w:" same for the actual filter since this change will make it a little unclear as to which failed to build.

if err != nil {
return -1, err
}
excludeRegex = res
}

agoDuration, err := parseDuration(ago)
if err != nil {
return -1, err
Expand All @@ -197,8 +213,9 @@ func purgeTags(ctx context.Context, acrClient api.AcrCLIClientInterface, poolSiz
// In order to only have a limited amount of http requests, a purger is used that will start goroutines to delete tags.
purger := worker.NewPurger(poolSize, acrClient, loginURL, repoName)
// GetTagsToDelete will return an empty lastTag when there are no more tags.

for {
tagsToDelete, newLastTag, newSkippedTagsCount, err := getTagsToDelete(ctx, acrClient, repoName, tagRegex, timeToCompare, lastTag, keep, skippedTagsCount)
tagsToDelete, newLastTag, newSkippedTagsCount, err := getTagsToDelete(ctx, acrClient, repoName, tagRegex, timeToCompare, lastTag, keep, skippedTagsCount, excludeRegex)
if err != nil {
return -1, err
}
Expand Down Expand Up @@ -257,7 +274,8 @@ func getTagsToDelete(ctx context.Context,
timeToCompare time.Time,
lastTag string,
keep int,
skippedTagsCount int) (*[]acr.TagAttributesBase, string, int, error) {
skippedTagsCount int,
excludeFilter *regexp2.Regexp) (*[]acr.TagAttributesBase, string, int, error) {

var matches bool
var lastUpdateTime time.Time
Expand All @@ -275,6 +293,18 @@ func getTagsToDelete(ctx context.Context,
tags := *resultTags.TagsAttributes
tagsEligibleForDeletion := []acr.TagAttributesBase{}
for _, tag := range tags {
if excludeFilter != nil {
matches, err = excludeFilter.MatchString(*tag.Name)
if err != nil {
// The only error that regexp2 will return is a timeout error
return nil, "", skippedTagsCount, err
}
if matches {
fmt.Printf("Tag \"%s\" skipped due to exclude filter\n", *tag.Name)
continue
}
}

matches, err = filter.MatchString(*tag.Name)
if err != nil {
// The only error that regexp2 will return is a timeout error
Expand Down Expand Up @@ -337,7 +367,18 @@ func purgeDanglingManifests(ctx context.Context, acrClient api.AcrCLIClientInter
}

// dryRunPurge outputs everything that would be deleted if the purge command was executed
func dryRunPurge(ctx context.Context, acrClient api.AcrCLIClientInterface, loginURL string, repoName string, ago string, filter string, untagged bool, keep int, regexMatchTimeout int64) (int, int, error) {
func dryRunPurge(
ctx context.Context,
acrClient api.AcrCLIClientInterface,
loginURL string,
repoName string,
ago string,
filter string,
untagged bool,
keep int,
regexMatchTimeout int64,
excludeFilter string) (int, int, error) {

deletedTagsCount := 0
deletedManifestsCount := 0
// In order to keep track if a manifest would get deleted a map is defined that as a key has the manifest
Expand All @@ -355,11 +396,21 @@ func dryRunPurge(ctx context.Context, acrClient api.AcrCLIClientInterface, login
return -1, -1, err
}

var excludeRegex *regexp2.Regexp = nil
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit. Assigning nil to a pointer is unnecessary


if excludeFilter != "" {
res, err := common.BuildRegexFilter(excludeFilter, regexMatchTimeout)
if err != nil {
return -1, -1, err
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as above, can you add the error wrap?

}
excludeRegex = res
}

lastTag := ""
skippedTagsCount := 0
// The loop to get the deleted tags follows the same logic as the one in the purgeTags function
for {
tagsToDelete, newLastTag, newSkippedTagsCount, err := getTagsToDelete(ctx, acrClient, repoName, regex, timeToCompare, lastTag, keep, skippedTagsCount)
tagsToDelete, newLastTag, newSkippedTagsCount, err := getTagsToDelete(ctx, acrClient, repoName, regex, timeToCompare, lastTag, keep, skippedTagsCount, excludeRegex)
if err != nil {
return -1, -1, err
}
Expand Down
Loading