Skip to content

Commit 9011d6d

Browse files
author
yeyuanjie
committed
when using rules to delete packages, remove unclean bugs
1 parent 74a0178 commit 9011d6d

File tree

3 files changed

+116
-84
lines changed

3 files changed

+116
-84
lines changed

‎models/packages/package_version.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ type PackageSearchOptions struct {
180180
RepoID int64
181181
Type Type
182182
PackageID int64
183+
LtVersionID int64
183184
Name SearchValue // only results with the specific name are found
184185
Version SearchValue // only results with the specific version are found
185186
Properties map[string]string // only results are found which contain all listed version properties with the specific value
@@ -210,6 +211,9 @@ func (opts *PackageSearchOptions) ToConds() builder.Cond {
210211
if opts.PackageID != 0 {
211212
cond = cond.And(builder.Eq{"package.id": opts.PackageID})
212213
}
214+
if opts.LtVersionID != 0 {
215+
cond = cond.And(builder.Lt{"package_version.id": opts.LtVersionID})
216+
}
213217
if opts.Name.Value != "" {
214218
if opts.Name.ExactMatch {
215219
cond = cond.And(builder.Eq{"package.lower_name": strings.ToLower(opts.Name.Value)})

‎routers/web/shared/packages/packages.go‎

Lines changed: 49 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -154,46 +154,60 @@ func SetRulePreviewContext(ctx *context.Context, owner *user_model.User) {
154154

155155
versionsToRemove := make([]*packages_model.PackageDescriptor, 0, 10)
156156

157+
limit := 200
157158
for _, p := range packages {
158-
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
159-
PackageID: p.ID,
160-
IsInternal: optional.Some(false),
161-
Sort: packages_model.SortCreatedDesc,
162-
Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200),
163-
})
164-
if err != nil {
165-
ctx.ServerError("SearchVersions", err)
166-
return
167-
}
168-
for _, pv := range pvs {
169-
if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
170-
ctx.ServerError("ShouldBeSkipped", err)
159+
lastVersionID := int64(0)
160+
for {
161+
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
162+
PackageID: p.ID,
163+
IsInternal: optional.Some(false),
164+
Sort: packages_model.SortCreatedDesc,
165+
Paginator: db.NewAbsoluteListOptions(func() int {
166+
if lastVersionID > 0 {
167+
return 0
168+
}
169+
return pcr.KeepCount
170+
}(), limit),
171+
LtVersionID: lastVersionID,
172+
})
173+
if err != nil {
174+
ctx.ServerError("SearchVersions", err)
171175
return
172-
} else if skip {
173-
continue
174176
}
175-
176-
toMatch := pv.LowerVersion
177-
if pcr.MatchFullName {
178-
toMatch = p.LowerName + "/" + pv.LowerVersion
179-
}
180-
181-
if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
182-
continue
177+
for _, pv := range pvs {
178+
lastVersionID = pv.ID
179+
if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
180+
ctx.ServerError("ShouldBeSkipped", err)
181+
return
182+
} else if skip {
183+
continue
184+
}
185+
186+
toMatch := pv.LowerVersion
187+
if pcr.MatchFullName {
188+
toMatch = p.LowerName + "/" + pv.LowerVersion
189+
}
190+
191+
if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
192+
continue
193+
}
194+
if pv.CreatedUnix.AsLocalTime().After(olderThan) {
195+
continue
196+
}
197+
if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
198+
continue
199+
}
200+
201+
pd, err := packages_model.GetPackageDescriptor(ctx, pv)
202+
if err != nil {
203+
ctx.ServerError("GetPackageDescriptor", err)
204+
return
205+
}
206+
versionsToRemove = append(versionsToRemove, pd)
183207
}
184-
if pv.CreatedUnix.AsLocalTime().After(olderThan) {
185-
continue
186-
}
187-
if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
188-
continue
189-
}
190-
191-
pd, err := packages_model.GetPackageDescriptor(ctx, pv)
192-
if err != nil {
193-
ctx.ServerError("GetPackageDescriptor", err)
194-
return
208+
if len(pvs) < limit {
209+
break
195210
}
196-
versionsToRemove = append(versionsToRemove, pd)
197211
}
198212
}
199213

‎services/packages/cleanup/cleanup.go‎

Lines changed: 63 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -58,65 +58,79 @@ func ExecuteCleanupRules(outerCtx context.Context) error {
5858
}
5959

6060
anyVersionDeleted := false
61+
limit := 200
6162
for _, p := range packages {
62-
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
63-
PackageID: p.ID,
64-
IsInternal: optional.Some(false),
65-
Sort: packages_model.SortCreatedDesc,
66-
Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200),
67-
})
68-
if err != nil {
69-
return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
70-
}
71-
versionDeleted := false
72-
for _, pv := range pvs {
73-
if pcr.Type == packages_model.TypeContainer {
74-
if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
75-
return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
76-
} else if skip {
77-
log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
78-
continue
79-
}
63+
lastVersionID := int64(0)
64+
for {
65+
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
66+
PackageID: p.ID,
67+
IsInternal: optional.Some(false),
68+
Sort: packages_model.SortCreatedDesc,
69+
Paginator: db.NewAbsoluteListOptions(func() int {
70+
if lastVersionID > 0 {
71+
return 0
72+
}
73+
return pcr.KeepCount
74+
}(), limit),
75+
LtVersionID: lastVersionID,
76+
})
77+
if err != nil {
78+
return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
8079
}
80+
versionDeleted := false
81+
for _, pv := range pvs {
82+
lastVersionID = pv.ID
83+
if pcr.Type == packages_model.TypeContainer {
84+
if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
85+
return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
86+
} else if skip {
87+
log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
88+
continue
89+
}
90+
}
8191

82-
toMatch := pv.LowerVersion
83-
if pcr.MatchFullName {
84-
toMatch = p.LowerName + "/" + pv.LowerVersion
85-
}
92+
toMatch := pv.LowerVersion
93+
if pcr.MatchFullName {
94+
toMatch = p.LowerName + "/" + pv.LowerVersion
95+
}
8696

87-
if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
88-
log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
89-
continue
90-
}
91-
if pv.CreatedUnix.AsLocalTime().After(olderThan) {
92-
log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version)
93-
continue
94-
}
95-
if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
96-
log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
97-
continue
98-
}
97+
if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
98+
log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
99+
continue
100+
}
101+
if pv.CreatedUnix.AsLocalTime().After(olderThan) {
102+
log.Debug("Rule[%d]: keep '%s/%s' (remove days) %v", pcr.ID, p.Name, pv.Version, pv.CreatedUnix.FormatDate())
103+
continue
104+
}
105+
if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
106+
log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
107+
continue
108+
}
99109

100-
log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
110+
log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
101111

102-
if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
103-
return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err)
104-
}
112+
if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
113+
return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err)
114+
}
105115

106-
versionDeleted = true
107-
anyVersionDeleted = true
108-
}
116+
versionDeleted = true
117+
anyVersionDeleted = true
118+
}
109119

110-
if versionDeleted {
111-
if pcr.Type == packages_model.TypeCargo {
112-
owner, err := user_model.GetUserByID(ctx, pcr.OwnerID)
113-
if err != nil {
114-
return fmt.Errorf("GetUserByID failed: %w", err)
115-
}
116-
if err := cargo_service.UpdatePackageIndexIfExists(ctx, owner, owner, p.ID); err != nil {
117-
return fmt.Errorf("CleanupRule [%d]: cargo.UpdatePackageIndexIfExists failed: %w", pcr.ID, err)
120+
if versionDeleted {
121+
if pcr.Type == packages_model.TypeCargo {
122+
owner, err := user_model.GetUserByID(ctx, pcr.OwnerID)
123+
if err != nil {
124+
return fmt.Errorf("GetUserByID failed: %w", err)
125+
}
126+
if err := cargo_service.UpdatePackageIndexIfExists(ctx, owner, owner, p.ID); err != nil {
127+
return fmt.Errorf("CleanupRule [%d]: cargo.UpdatePackageIndexIfExists failed: %w", pcr.ID, err)
128+
}
118129
}
119130
}
131+
if len(pvs) < limit {
132+
break
133+
}
120134
}
121135
}
122136

0 commit comments

Comments
 (0)