Skip to content

Commit 6d0f33e

Browse files
tbphptbphp
authored
Merge pull request #26 from tbphp/fix-request-logs-data
fix: 修复迁移日志表数据
2 parents 313d147 + 2c4315d commit 6d0f33e

File tree

1 file changed

+101
-91
lines changed

1 file changed

+101
-91
lines changed

‎internal/db/migrations/v1.0.13_fix_request_logs.go‎

Lines changed: 101 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -7,124 +7,134 @@ import (
77
"gorm.io/gorm"
88
)
99

10+
// TODO: 更新迁移，待多个版本后旧版本都升级差不多之后移除。
1011
func V1_0_13_FixRequestLogs(db *gorm.DB) error {
11-
return db.Transaction(func(tx *gorm.DB) error {
12-
// 如果有key_id，就执行修复
13-
if !tx.Migrator().HasColumn(&models.RequestLog{}, "key_id") {
14-
return nil
15-
}
12+
// 如果有key_id，就执行修复
13+
if !db.Migrator().HasColumn(&models.RequestLog{}, "key_id") {
14+
return nil
15+
}
1616

17-
logrus.Info("Old schema detected. Starting data migration for request_logs...")
17+
logrus.Info("Old schema detected. Starting data migration for request_logs...")
1818

19-
if !tx.Migrator().HasColumn(&models.RequestLog{}, "group_name") {
20-
logrus.Info("Adding 'group_name' column to request_logs table...")
21-
if err := tx.Migrator().AddColumn(&models.RequestLog{}, "group_name"); err != nil {
22-
return err
23-
}
19+
if !db.Migrator().HasColumn(&models.RequestLog{}, "group_name") {
20+
logrus.Info("Adding 'group_name' column to request_logs table...")
21+
if err := db.Migrator().AddColumn(&models.RequestLog{}, "group_name"); err != nil {
22+
return err // Column addition is critical
2423
}
25-
if !tx.Migrator().HasColumn(&models.RequestLog{}, "key_value") {
26-
logrus.Info("Adding 'key_value' column to request_logs table...")
27-
if err := tx.Migrator().AddColumn(&models.RequestLog{}, "key_value"); err != nil {
28-
return err
29-
}
24+
}
25+
if !db.Migrator().HasColumn(&models.RequestLog{}, "key_value") {
26+
logrus.Info("Adding 'key_value' column to request_logs table...")
27+
if err := db.Migrator().AddColumn(&models.RequestLog{}, "key_value"); err != nil {
28+
return err // Column addition is critical
3029
}
31-
32-
type OldRequestLog struct {
33-
ID string
34-
KeyID uint `gorm:"column:key_id"`
35-
GroupID uint
30+
}
31+
32+
type OldRequestLog struct {
33+
ID string
34+
KeyID uint `gorm:"column:key_id"`
35+
GroupID uint
36+
}
37+
38+
batchSize := 1000
39+
for i := 0; ; i++ {
40+
logrus.Infof("Processing batch %d...", i+1)
41+
var oldLogs []OldRequestLog
42+
43+
result := db.Model(&models.RequestLog{}).
44+
Select("id", "key_id", "group_id").
45+
Where("key_value IS NULL OR group_name IS NULL").
46+
Limit(batchSize).
47+
Find(&oldLogs)
48+
49+
if result.Error != nil {
50+
logrus.WithError(result.Error).Error("Failed to fetch batch of logs. Skipping to next batch.")
51+
continue
3652
}
3753

38-
batchSize := 1000
39-
for i := 0; ; i++ {
40-
logrus.Infof("Processing batch %d...", i+1)
41-
var oldLogs []OldRequestLog
42-
43-
result := tx.Model(&models.RequestLog{}).
44-
Select("id", "key_id", "group_id").
45-
Where("key_value IS NULL OR group_name IS NULL").
46-
Limit(batchSize).
47-
Find(&oldLogs)
54+
if len(oldLogs) == 0 {
55+
logrus.Info("All batches processed.")
56+
break
57+
}
4858

49-
if result.Error != nil {
50-
return result.Error
59+
keyIDMap := make(map[uint]bool)
60+
groupIDMap := make(map[uint]bool)
61+
for _, logEntry := range oldLogs {
62+
if logEntry.KeyID > 0 {
63+
keyIDMap[logEntry.KeyID] = true
5164
}
52-
53-
if len(oldLogs) == 0 {
54-
logrus.Info("All batches processed.")
55-
break
65+
if logEntry.GroupID > 0 {
66+
groupIDMap[logEntry.GroupID] = true
5667
}
68+
}
5769

58-
keyIDMap := make(map[uint]bool)
59-
groupIDMap := make(map[uint]bool)
60-
for _, logEntry := range oldLogs {
61-
if logEntry.KeyID > 0 {
62-
keyIDMap[logEntry.KeyID] = true
63-
}
64-
if logEntry.GroupID > 0 {
65-
groupIDMap[logEntry.GroupID] = true
66-
}
70+
var apiKeys []models.APIKey
71+
if len(keyIDMap) > 0 {
72+
var keyIDs []uint
73+
for id := range keyIDMap {
74+
keyIDs = append(keyIDs, id)
75+
}
76+
if err := db.Model(&models.APIKey{}).Where("id IN ?", keyIDs).Find(&apiKeys).Error; err != nil {
77+
logrus.WithError(err).Warn("Failed to fetch API keys for the current batch. Some logs may not be updated.")
6778
}
79+
}
80+
keyValueMapping := make(map[uint]string)
81+
for _, key := range apiKeys {
82+
keyValueMapping[key.ID] = key.KeyValue
83+
}
6884

69-
var apiKeys []models.APIKey
70-
if len(keyIDMap) > 0 {
71-
var keyIDs []uint
72-
for id := range keyIDMap {
73-
keyIDs = append(keyIDs, id)
74-
}
75-
if err := tx.Model(&models.APIKey{}).Where("id IN ?", keyIDs).Find(&apiKeys).Error; err != nil {
76-
return err
77-
}
85+
var groups []models.Group
86+
if len(groupIDMap) > 0 {
87+
var groupIDs []uint
88+
for id := range groupIDMap {
89+
groupIDs = append(groupIDs, id)
7890
}
79-
keyValueMapping := make(map[uint]string)
80-
for _, key := range apiKeys {
81-
keyValueMapping[key.ID] = key.KeyValue
91+
if err := db.Model(&models.Group{}).Where("id IN ?", groupIDs).Find(&groups).Error; err != nil {
92+
logrus.WithError(err).Warn("Failed to fetch groups for the current batch. Some logs may not be updated.")
8293
}
94+
}
95+
groupNameMapping := make(map[uint]string)
96+
for _, group := range groups {
97+
groupNameMapping[group.ID] = group.Name
98+
}
8399

84-
var groups []models.Group
85-
if len(groupIDMap) > 0 {
86-
var groupIDs []uint
87-
for id := range groupIDMap {
88-
groupIDs = append(groupIDs, id)
89-
}
90-
if err := tx.Model(&models.Group{}).Where("id IN ?", groupIDs).Find(&groups).Error; err != nil {
91-
return err
92-
}
93-
}
94-
groupNameMapping := make(map[uint]string)
95-
for _, group := range groups {
96-
groupNameMapping[group.ID] = group.Name
100+
updateGroups := make(map[string]map[string][]string)
101+
102+
for _, logEntry := range oldLogs {
103+
groupName, gExists := groupNameMapping[logEntry.GroupID]
104+
if !gExists {
105+
logrus.Warnf("Log ID %s: Could not find Group for group_id %d. Setting group_name to empty string.", logEntry.ID, logEntry.GroupID)
97106
}
98107

99-
for _, logEntry := range oldLogs {
100-
groupName, gExists := groupNameMapping[logEntry.GroupID]
101-
if !gExists {
102-
logrus.Warnf("Log ID %s: Could not find Group for group_id %d. Setting group_name to empty string.", logEntry.ID, logEntry.GroupID)
103-
}
108+
keyValue, kExists := keyValueMapping[logEntry.KeyID]
109+
if !kExists {
110+
logrus.Warnf("Log ID %s: Could not find APIKey for key_id %d. Setting key_value to empty string.", logEntry.ID, logEntry.KeyID)
111+
}
104112

105-
keyValue, kExists := keyValueMapping[logEntry.KeyID]
106-
if !kExists {
107-
logrus.Warnf("Log ID %s: Could not find APIKey for key_id %d. Setting key_value to empty string.", logEntry.ID, logEntry.KeyID)
108-
}
113+
if _, ok := updateGroups[groupName]; !ok {
114+
updateGroups[groupName] = make(map[string][]string)
115+
}
116+
updateGroups[groupName][keyValue] = append(updateGroups[groupName][keyValue], logEntry.ID)
117+
}
109118

119+
for groupName, keyMap := range updateGroups {
120+
for keyValue, ids := range keyMap {
110121
updates := map[string]any{
111122
"group_name": groupName,
112123
"key_value": keyValue,
113124
}
114-
if err := tx.Model(&models.RequestLog{}).Where("id = ?", logEntry.ID).UpdateColumns(updates).Error; err != nil {
115-
logrus.WithError(err).Errorf("Failed to update log entry with ID: %s", logEntry.ID)
116-
continue
125+
if err := db.Model(&models.RequestLog{}).Where("id IN ?", ids).UpdateColumns(updates).Error; err != nil {
126+
logrus.WithError(err).Errorf("Failed to update a batch of log entries. Skipping this batch.")
117127
}
118128
}
119-
logrus.Infof("Successfully updated %d log entries in batch %d.", len(oldLogs), i+1)
120129
}
130+
logrus.Infof("Finished processing batch %d. Updated %d log entries.", i+1, len(oldLogs))
131+
}
121132

122-
logrus.Info("Data migration complete. Dropping 'key_id' column from request_logs table...")
123-
if err := tx.Migrator().DropColumn(&models.RequestLog{}, "key_id"); err != nil {
124-
logrus.WithError(err).Warn("Failed to drop 'key_id' column. This can be done manually.")
125-
}
133+
logrus.Info("Data migration complete. Dropping 'key_id' column from request_logs table...")
134+
if err := db.Migrator().DropColumn(&models.RequestLog{}, "key_id"); err != nil {
135+
logrus.WithError(err).Warn("Failed to drop 'key_id' column. This can be done manually.")
136+
}
126137

127-
logrus.Info("Database migration successful!")
128-
return nil
129-
})
138+
logrus.Info("Database migration finished!")
139+
return nil
130140
}

0 commit comments

Comments
 (0)