Skip to content

Commit 91ff87c

Browse files
axifivetechknowlogick
axifive
authored and
techknowlogick
committed
Fixed violation of the unique constraint for v68 migration (#4297)
1 parent a9ffbeb commit 91ff87c

File tree

2 files changed

+106
-53
lines changed

2 files changed

+106
-53
lines changed

‎models/migrations/v68.go

Lines changed: 100 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -5,99 +5,147 @@
55
package migrations
66

77
import (
8+
"fmt"
9+
"regexp"
810
"strings"
911

10-
"code.gitea.io/gitea/models"
1112
"code.gitea.io/gitea/modules/log"
1213

1314
"github.com/go-xorm/xorm"
1415
)
1516

17+
var topicPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
18+
19+
func validateTopic(topic string) bool {
20+
return len(topic) <= 35 && topicPattern.MatchString(topic)
21+
}
22+
1623
func reformatAndRemoveIncorrectTopics(x *xorm.Engine) (err error) {
1724
log.Info("This migration could take up to minutes, please be patient.")
25+
1826
type Topic struct {
19-
ID int64
20-
Name string `xorm:"unique"`
27+
ID int64
28+
Name string `xorm:"UNIQUE"`
29+
RepoCount int
30+
CreatedUnix int64 `xorm:"INDEX created"`
31+
UpdatedUnix int64 `xorm:"INDEX updated"`
32+
}
33+
34+
type RepoTopic struct {
35+
RepoID int64 `xorm:"UNIQUE(s)"`
36+
TopicID int64 `xorm:"UNIQUE(s)"`
37+
}
38+
39+
type Repository struct {
40+
ID int64 `xorm:"pk autoincr"`
41+
Topics []string `xorm:"TEXT JSON"`
42+
}
43+
44+
if err := x.Sync2(new(Topic)); err != nil {
45+
return fmt.Errorf("Sync2: %v", err)
46+
}
47+
if err := x.Sync2(new(RepoTopic)); err != nil {
48+
return fmt.Errorf("Sync2: %v", err)
2149
}
2250

2351
sess := x.NewSession()
2452
defer sess.Close()
2553

2654
const batchSize = 100
2755
touchedRepo := make(map[int64]struct{})
28-
topics := make([]*Topic, 0, batchSize)
2956
delTopicIDs := make([]int64, 0, batchSize)
30-
ids := make([]int64, 0, 30)
3157

58+
log.Info("Validating existed topics...")
3259
if err := sess.Begin(); err != nil {
3360
return err
3461
}
35-
log.Info("Validating existed topics...")
3662
for start := 0; ; start += batchSize {
37-
topics = topics[:0]
38-
if err := sess.Asc("id").Limit(batchSize, start).Find(&topics); err != nil {
63+
topics := make([]*Topic, 0, batchSize)
64+
if err := x.Cols("id", "name").Asc("id").Limit(batchSize, start).Find(&topics); err != nil {
3965
return err
4066
}
4167
if len(topics) == 0 {
4268
break
4369
}
4470
for _, topic := range topics {
45-
if models.ValidateTopic(topic.Name) {
71+
if validateTopic(topic.Name) {
4672
continue
4773
}
74+
log.Info("Incorrect topic: id = %v, name = %q", topic.ID, topic.Name)
75+
4876
topic.Name = strings.Replace(strings.TrimSpace(strings.ToLower(topic.Name)), " ", "-", -1)
4977

78+
ids := make([]int64, 0, 30)
5079
if err := sess.Table("repo_topic").Cols("repo_id").
5180
Where("topic_id = ?", topic.ID).Find(&ids); err != nil {
5281
return err
5382
}
83+
log.Info("Touched repo ids: %v", ids)
5484
for _, id := range ids {
5585
touchedRepo[id] = struct{}{}
5686
}
5787

58-
if models.ValidateTopic(topic.Name) {
59-
log.Info("Updating topic: id = %v, name = %v", topic.ID, topic.Name)
60-
if _, err := sess.Table("topic").ID(topic.ID).
61-
Update(&Topic{Name: topic.Name}); err != nil {
88+
if validateTopic(topic.Name) {
89+
unifiedTopic := Topic{Name: topic.Name}
90+
exists, err := sess.Cols("id", "name").Get(&unifiedTopic)
91+
log.Info("Exists topic with the name %q? %v, id = %v", topic.Name, exists, unifiedTopic.ID)
92+
if err != nil {
6293
return err
6394
}
64-
} else {
65-
delTopicIDs = append(delTopicIDs, topic.ID)
95+
if exists {
96+
log.Info("Updating repo_topic rows with topic_id = %v to topic_id = %v", topic.ID, unifiedTopic.ID)
97+
if _, err := sess.Where("topic_id = ? AND repo_id NOT IN "+
98+
"(SELECT rt1.repo_id FROM repo_topic rt1 INNER JOIN repo_topic rt2 "+
99+
"ON rt1.repo_id = rt2.repo_id WHERE rt1.topic_id = ? AND rt2.topic_id = ?)",
100+
topic.ID, topic.ID, unifiedTopic.ID).Update(&RepoTopic{TopicID: unifiedTopic.ID}); err != nil {
101+
return err
102+
}
103+
log.Info("Updating topic `repo_count` field")
104+
if _, err := sess.Exec(
105+
"UPDATE topic SET repo_count = (SELECT COUNT(*) FROM repo_topic WHERE topic_id = ? GROUP BY topic_id) WHERE id = ?",
106+
unifiedTopic.ID, unifiedTopic.ID); err != nil {
107+
return err
108+
}
109+
} else {
110+
log.Info("Updating topic: id = %v, name = %q", topic.ID, topic.Name)
111+
if _, err := sess.Table("topic").ID(topic.ID).
112+
Update(&Topic{Name: topic.Name}); err != nil {
113+
return err
114+
}
115+
continue
116+
}
66117
}
118+
delTopicIDs = append(delTopicIDs, topic.ID)
67119
}
68120
}
121+
if err := sess.Commit(); err != nil {
122+
return err
123+
}
69124

70-
log.Info("Deleting incorrect topics...")
71-
for start := 0; ; start += batchSize {
72-
if (start + batchSize) < len(delTopicIDs) {
73-
ids = delTopicIDs[start:(start + batchSize)]
74-
} else {
75-
ids = delTopicIDs[start:]
76-
}
77-
78-
log.Info("Deleting 'repo_topic' rows for topics with ids = %v", ids)
79-
if _, err := sess.In("topic_id", ids).Delete(&models.RepoTopic{}); err != nil {
80-
return err
81-
}
82-
83-
log.Info("Deleting topics with id = %v", ids)
84-
if _, err := sess.In("id", ids).Delete(&Topic{}); err != nil {
85-
return err
86-
}
125+
sess.Init()
87126

88-
if len(ids) < batchSize {
89-
break
90-
}
127+
log.Info("Deleting incorrect topics...")
128+
if err := sess.Begin(); err != nil {
129+
return err
130+
}
131+
log.Info("Deleting 'repo_topic' rows for topics with ids = %v", delTopicIDs)
132+
if _, err := sess.In("topic_id", delTopicIDs).Delete(&RepoTopic{}); err != nil {
133+
return err
134+
}
135+
log.Info("Deleting topics with id = %v", delTopicIDs)
136+
if _, err := sess.In("id", delTopicIDs).Delete(&Topic{}); err != nil {
137+
return err
138+
}
139+
if err := sess.Commit(); err != nil {
140+
return err
91141
}
92142

93-
repoTopics := make([]*models.RepoTopic, 0, batchSize)
94-
delRepoTopics := make([]*models.RepoTopic, 0, batchSize)
95-
tmpRepoTopics := make([]*models.RepoTopic, 0, 30)
143+
delRepoTopics := make([]*RepoTopic, 0, batchSize)
96144

97145
log.Info("Checking the number of topics in the repositories...")
98146
for start := 0; ; start += batchSize {
99-
repoTopics = repoTopics[:0]
100-
if err := sess.Cols("repo_id").Asc("repo_id").Limit(batchSize, start).
147+
repoTopics := make([]*RepoTopic, 0, batchSize)
148+
if err := x.Cols("repo_id").Asc("repo_id").Limit(batchSize, start).
101149
GroupBy("repo_id").Having("COUNT(*) > 25").Find(&repoTopics); err != nil {
102150
return err
103151
}
@@ -109,8 +157,8 @@ func reformatAndRemoveIncorrectTopics(x *xorm.Engine) (err error) {
109157
for _, repoTopic := range repoTopics {
110158
touchedRepo[repoTopic.RepoID] = struct{}{}
111159

112-
tmpRepoTopics = tmpRepoTopics[:0]
113-
if err := sess.Where("repo_id = ?", repoTopic.RepoID).Find(&tmpRepoTopics); err != nil {
160+
tmpRepoTopics := make([]*RepoTopic, 0, 30)
161+
if err := x.Where("repo_id = ?", repoTopic.RepoID).Find(&tmpRepoTopics); err != nil {
114162
return err
115163
}
116164

@@ -122,13 +170,18 @@ func reformatAndRemoveIncorrectTopics(x *xorm.Engine) (err error) {
122170
}
123171
}
124172

173+
sess.Init()
174+
125175
log.Info("Deleting superfluous topics for repositories (more than 25 topics)...")
176+
if err := sess.Begin(); err != nil {
177+
return err
178+
}
126179
for _, repoTopic := range delRepoTopics {
127180
log.Info("Deleting 'repo_topic' rows for 'repository' with id = %v. Topic id = %v",
128181
repoTopic.RepoID, repoTopic.TopicID)
129182

130183
if _, err := sess.Where("repo_id = ? AND topic_id = ?", repoTopic.RepoID,
131-
repoTopic.TopicID).Delete(&models.RepoTopic{}); err != nil {
184+
repoTopic.TopicID).Delete(&RepoTopic{}); err != nil {
132185
return err
133186
}
134187
if _, err := sess.Exec(
@@ -138,17 +191,17 @@ func reformatAndRemoveIncorrectTopics(x *xorm.Engine) (err error) {
138191
}
139192
}
140193

141-
topicNames := make([]string, 0, 30)
142194
log.Info("Updating repositories 'topics' fields...")
143195
for repoID := range touchedRepo {
196+
topicNames := make([]string, 0, 30)
144197
if err := sess.Table("topic").Cols("name").
145-
Join("INNER", "repo_topic", "topic.id = repo_topic.topic_id").
146-
Where("repo_topic.repo_id = ?", repoID).Find(&topicNames); err != nil {
198+
Join("INNER", "repo_topic", "repo_topic.topic_id = topic.id").
199+
Where("repo_topic.repo_id = ?", repoID).Desc("topic.repo_count").Find(&topicNames); err != nil {
147200
return err
148201
}
149202
log.Info("Updating 'topics' field for repository with id = %v", repoID)
150203
if _, err := sess.ID(repoID).Cols("topics").
151-
Update(&models.Repository{Topics: topicNames}); err != nil {
204+
Update(&Repository{Topics: topicNames}); err != nil {
152205
return err
153206
}
154207
}

‎models/topic.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,16 @@ var topicPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
2626
// Topic represents a topic of repositories
2727
type Topic struct {
2828
ID int64
29-
Name string `xorm:"unique"`
29+
Name string `xorm:"UNIQUE"`
3030
RepoCount int
3131
CreatedUnix util.TimeStamp `xorm:"INDEX created"`
3232
UpdatedUnix util.TimeStamp `xorm:"INDEX updated"`
3333
}
3434

3535
// RepoTopic represents associated repositories and topics
3636
type RepoTopic struct {
37-
RepoID int64 `xorm:"unique(s)"`
38-
TopicID int64 `xorm:"unique(s)"`
37+
RepoID int64 `xorm:"UNIQUE(s)"`
38+
TopicID int64 `xorm:"UNIQUE(s)"`
3939
}
4040

4141
// ErrTopicNotExist represents an error that a topic is not exist
@@ -190,10 +190,10 @@ func SaveTopics(repoID int64, topicNames ...string) error {
190190
}
191191
}
192192

193-
topicNames = topicNames[:0]
193+
topicNames = make([]string, 0, 25)
194194
if err := sess.Table("topic").Cols("name").
195-
Join("INNER", "repo_topic", "topic.id = repo_topic.topic_id").
196-
Where("repo_topic.repo_id = ?", repoID).Find(&topicNames); err != nil {
195+
Join("INNER", "repo_topic", "repo_topic.topic_id = topic.id").
196+
Where("repo_topic.repo_id = ?", repoID).Desc("topic.repo_count").Find(&topicNames); err != nil {
197197
return err
198198
}
199199

0 commit comments

Comments
 (0)