Skip to content

Commit 3b99d6e

Browse files
BoxBoxJasonBoxBoxJason
committed
feat: add full group resync
added a function to check if a group data does not match and update it if needed also moved some functions for better maintainability
1 parent 6d300f1 commit 3b99d6e

File tree

2 files changed

+148
-84
lines changed

2 files changed

+148
-84
lines changed

‎internal/mirroring/post.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func (destinationGitlab *GitlabInstance) createGroups(sourceGitlab *GitlabInstan
2424
// Reverse the mirror mapping to get the source group path for each destination group
2525
reversedMirrorMap, destinationGroupPaths := sourceGitlab.reverseGroupMirrorMap(mirrorMapping)
2626

27-
errorChan := make(chan error, len(destinationGroupPaths))
27+
errorChan := make(chan []error, len(destinationGroupPaths))
2828
// Iterate over the groups in alphabetical order (little hack to ensure parent groups are created before children)
2929
for _, destinationGroupPath := range destinationGroupPaths {
3030
_, err := destinationGitlab.createGroup(destinationGroupPath, sourceGitlab, mirrorMapping, &reversedMirrorMap)
@@ -39,20 +39,20 @@ func (destinationGitlab *GitlabInstance) createGroups(sourceGitlab *GitlabInstan
3939
// createGroup creates a GitLab group in the destination GitLab instance based on the source group and mirror mapping.
4040
// It checks if the group already exists in the destination instance and creates it if not.
4141
// The function also handles the copying of group avatars from the source to the destination instance.
42-
func (destinationGitlab *GitlabInstance) createGroup(destinationGroupPath string, sourceGitlab *GitlabInstance, mirrorMapping *utils.MirrorMapping, reversedMirrorMap *map[string]string) (*gitlab.Group, error) {
42+
func (destinationGitlab *GitlabInstance) createGroup(destinationGroupPath string, sourceGitlab *GitlabInstance, mirrorMapping *utils.MirrorMapping, reversedMirrorMap *map[string]string) (*gitlab.Group, []error) {
4343
// Retrieve the corresponding source group path
4444
sourceGroupPath := (*reversedMirrorMap)[destinationGroupPath]
4545
zap.L().Debug("Mirroring group", zap.String(ROLE_SOURCE, sourceGroupPath), zap.String(ROLE_DESTINATION, destinationGroupPath))
4646

4747
sourceGroup := sourceGitlab.Groups[sourceGroupPath]
4848
if sourceGroup == nil {
49-
return nil, fmt.Errorf("group %s not found in destination GitLab instance (internal error, please review script)", sourceGroupPath)
49+
return nil, []error{fmt.Errorf("group %s not found in destination GitLab instance (internal error, please review script)", sourceGroupPath)}
5050
}
5151

5252
// Retrieve the corresponding group creation options from the mirror mapping
5353
groupCreationOptions, ok := mirrorMapping.GetGroup(sourceGroupPath)
5454
if !ok {
55-
return nil, fmt.Errorf("source group %s not found in mirror mapping (internal error, please review script)", sourceGroupPath)
55+
return nil, []error{fmt.Errorf("source group %s not found in mirror mapping (internal error, please review script)", sourceGroupPath)}
5656
}
5757

5858
// Check if the group already exists in the destination GitLab instance
@@ -62,12 +62,12 @@ func (destinationGitlab *GitlabInstance) createGroup(destinationGroupPath string
6262
zap.L().Debug("Group not found, creating new group in GitLab Instance", zap.String("group", destinationGroupPath), zap.String(ROLE, ROLE_DESTINATION))
6363
destinationGroup, err = destinationGitlab.createGroupFromSource(sourceGroup, groupCreationOptions)
6464
if err != nil {
65-
return nil, fmt.Errorf("failed to create group %s in destination GitLab instance: %s", destinationGroupPath, err)
65+
return nil, []error{fmt.Errorf("failed to create group %s in destination GitLab instance: %s", destinationGroupPath, err)}
6666
} else {
6767
// Copy the group avatar from the source to the destination instance
68-
err = sourceGitlab.copyGroupAvatar(destinationGitlab, destinationGroup, sourceGroup)
69-
if err != nil {
70-
return destinationGroup, fmt.Errorf("failed to copy group avatar for %s: %s", destinationGroupPath, err)
68+
errArray := sourceGitlab.updateGroupFromSource(destinationGitlab, destinationGroup, sourceGroup, groupCreationOptions)
69+
if errArray != nil {
70+
return destinationGroup, errArray
7171
}
7272
}
7373
}

‎internal/mirroring/put.go‎

Lines changed: 140 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -11,82 +11,9 @@ import (
1111
"go.uber.org/zap"
1212
)
1313

14-
// enableProjectMirrorPull enables the pull mirror for a project in the destination GitLab instance.
15-
// It sets the source project URL, enables mirroring, and configures other options like triggering builds and overwriting diverged branches.
16-
func (g *GitlabInstance) enableProjectMirrorPull(sourceProject *gitlab.Project, destinationProject *gitlab.Project, mirrorOptions *utils.MirroringOptions) error {
17-
zap.L().Debug("Enabling project mirror pull", zap.String("sourceProject", sourceProject.HTTPURLToRepo), zap.String("destinationProject", destinationProject.HTTPURLToRepo))
18-
_, _, err := g.Gitlab.Projects.ConfigureProjectPullMirror(destinationProject.ID, &gitlab.ConfigureProjectPullMirrorOptions{
19-
URL: &sourceProject.HTTPURLToRepo,
20-
OnlyMirrorProtectedBranches: gitlab.Ptr(true),
21-
Enabled: gitlab.Ptr(true),
22-
MirrorOverwritesDivergedBranches: gitlab.Ptr(true),
23-
MirrorTriggerBuilds: gitlab.Ptr(mirrorOptions.MirrorTriggerBuilds),
24-
})
25-
return err
26-
}
27-
28-
// copyProjectAvatar copies the avatar from the source project to the destination project.
29-
// It first checks if the destination project already has an avatar set. If not, it downloads the avatar from the source project
30-
// and uploads it to the destination project.
31-
// The avatar is saved with a unique filename based on the current timestamp.
32-
// The function returns an error if any step fails, including downloading or uploading the avatar.
33-
func (sourceGitlabInstance *GitlabInstance) copyProjectAvatar(destinationGitlabInstance *GitlabInstance, destinationProject *gitlab.Project, sourceProject *gitlab.Project) error {
34-
zap.L().Debug("Checking if project avatar is already set", zap.String("project", destinationProject.HTTPURLToRepo))
35-
36-
// Check if the destination project already has an avatar
37-
if destinationProject.AvatarURL != "" {
38-
zap.L().Debug("Project already has an avatar set, skipping.", zap.String("project", destinationProject.HTTPURLToRepo), zap.String("path", destinationProject.AvatarURL))
39-
return nil
40-
}
41-
42-
zap.L().Debug("Copying project avatar", zap.String(ROLE_SOURCE, sourceProject.HTTPURLToRepo), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo))
43-
44-
// Download the source project avatar
45-
sourceProjectAvatar, _, err := sourceGitlabInstance.Gitlab.Projects.DownloadAvatar(sourceProject.ID)
46-
if err != nil {
47-
return fmt.Errorf("failed to download avatar for project %s: %s", sourceProject.HTTPURLToRepo, err)
48-
}
49-
50-
// Upload the avatar to the destination project
51-
_, _, err = destinationGitlabInstance.Gitlab.Projects.UploadAvatar(destinationProject.ID, sourceProjectAvatar, fmt.Sprintf("avatar-%d.png", time.Now().Unix()))
52-
if err != nil {
53-
return fmt.Errorf("failed to upload avatar for project %s: %s", destinationProject.HTTPURLToRepo, err)
54-
}
55-
56-
return nil
57-
}
58-
59-
// copyGroupAvatar copies the avatar from the source group to the destination group.
60-
// It first checks if the destination group already has an avatar set. If not, it downloads the avatar from the source group
61-
// and uploads it to the destination group.
62-
// The avatar is saved with a unique filename based on the current timestamp.
63-
// The function returns an error if any step fails, including downloading or uploading the avatar.
64-
func (sourceGitlabInstance *GitlabInstance) copyGroupAvatar(destinationGitlabInstance *GitlabInstance, destinationGroup *gitlab.Group, sourceGroup *gitlab.Group) error {
65-
zap.L().Debug("Checking if group avatar is already set", zap.String("group", destinationGroup.WebURL))
66-
67-
// Check if the destination group already has an avatar
68-
if destinationGroup.AvatarURL != "" {
69-
zap.L().Debug("Group avatar already set", zap.String("group", destinationGroup.WebURL), zap.String("path", destinationGroup.AvatarURL))
70-
return nil
71-
}
72-
73-
zap.L().Debug("Copying group avatar", zap.String(ROLE_SOURCE, sourceGroup.WebURL), zap.String(ROLE_DESTINATION, destinationGroup.WebURL))
74-
75-
// Download the source group avatar
76-
sourceGroupAvatar, _, err := sourceGitlabInstance.Gitlab.Groups.DownloadAvatar(sourceGroup.ID)
77-
if err != nil {
78-
return fmt.Errorf("failed to download avatar for group %s: %s", sourceGroup.WebURL, err)
79-
}
80-
81-
// Upload the avatar to the destination group
82-
filename := fmt.Sprintf("avatar-%d.png", time.Now().Unix())
83-
_, _, err = destinationGitlabInstance.Gitlab.Groups.UploadAvatar(destinationGroup.ID, sourceGroupAvatar, filename)
84-
if err != nil {
85-
return fmt.Errorf("failed to upload avatar for group %s: %s", destinationGroup.WebURL, err)
86-
}
87-
88-
return nil
89-
}
14+
// ===========================================================================
15+
// PROJECTS PUT FUNCTIONS //
16+
// ===========================================================================
9017

9118
// updateProjectFromSource updates the destination project with settings from the source project.
9219
// It enables the project mirror pull, copies the project avatar, and optionally adds the project to the CI/CD catalog.
@@ -195,3 +122,140 @@ func (destinationGitlabInstance *GitlabInstance) syncProjectAttributes(sourcePro
195122
}
196123
return nil
197124
}
125+
126+
// enableProjectMirrorPull enables the pull mirror for a project in the destination GitLab instance.
127+
// It sets the source project URL, enables mirroring, and configures other options like triggering builds and overwriting diverged branches.
128+
func (g *GitlabInstance) enableProjectMirrorPull(sourceProject *gitlab.Project, destinationProject *gitlab.Project, mirrorOptions *utils.MirroringOptions) error {
129+
zap.L().Debug("Enabling project mirror pull", zap.String("sourceProject", sourceProject.HTTPURLToRepo), zap.String("destinationProject", destinationProject.HTTPURLToRepo))
130+
_, _, err := g.Gitlab.Projects.ConfigureProjectPullMirror(destinationProject.ID, &gitlab.ConfigureProjectPullMirrorOptions{
131+
URL: &sourceProject.HTTPURLToRepo,
132+
OnlyMirrorProtectedBranches: gitlab.Ptr(true),
133+
Enabled: gitlab.Ptr(true),
134+
MirrorOverwritesDivergedBranches: gitlab.Ptr(true),
135+
MirrorTriggerBuilds: gitlab.Ptr(mirrorOptions.MirrorTriggerBuilds),
136+
})
137+
return err
138+
}
139+
140+
// copyProjectAvatar copies the avatar from the source project to the destination project.
141+
// It first checks if the destination project already has an avatar set. If not, it downloads the avatar from the source project
142+
// and uploads it to the destination project.
143+
// The avatar is saved with a unique filename based on the current timestamp.
144+
// The function returns an error if any step fails, including downloading or uploading the avatar.
145+
func (sourceGitlabInstance *GitlabInstance) copyProjectAvatar(destinationGitlabInstance *GitlabInstance, destinationProject *gitlab.Project, sourceProject *gitlab.Project) error {
146+
zap.L().Debug("Checking if project avatar is already set", zap.String("project", destinationProject.HTTPURLToRepo))
147+
148+
// Check if the destination project already has an avatar
149+
if destinationProject.AvatarURL != "" {
150+
zap.L().Debug("Project already has an avatar set, skipping.", zap.String("project", destinationProject.HTTPURLToRepo), zap.String("path", destinationProject.AvatarURL))
151+
return nil
152+
}
153+
154+
zap.L().Debug("Copying project avatar", zap.String(ROLE_SOURCE, sourceProject.HTTPURLToRepo), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo))
155+
156+
// Download the source project avatar
157+
sourceProjectAvatar, _, err := sourceGitlabInstance.Gitlab.Projects.DownloadAvatar(sourceProject.ID)
158+
if err != nil {
159+
return fmt.Errorf("failed to download avatar for project %s: %s", sourceProject.HTTPURLToRepo, err)
160+
}
161+
162+
// Upload the avatar to the destination project
163+
_, _, err = destinationGitlabInstance.Gitlab.Projects.UploadAvatar(destinationProject.ID, sourceProjectAvatar, fmt.Sprintf("avatar-%d.png", time.Now().Unix()))
164+
if err != nil {
165+
return fmt.Errorf("failed to upload avatar for project %s: %s", destinationProject.HTTPURLToRepo, err)
166+
}
167+
168+
return nil
169+
}
170+
171+
// ===========================================================================
172+
// GROUPS PUT FUNCTIONS //
173+
// ===========================================================================
174+
175+
// updateGroupFromSource updates the destination group with settings from the source group.
176+
// It copies the group avatar and updates the group attributes.
177+
func (destinationGitlabInstance *GitlabInstance) updateGroupFromSource(sourceGitlabInstance *GitlabInstance, sourceGroup *gitlab.Group, destinationGroup *gitlab.Group, copyOptions *utils.MirroringOptions) []error {
178+
wg := sync.WaitGroup{}
179+
maxErrors := 2
180+
wg.Add(maxErrors)
181+
errorChan := make(chan error, maxErrors)
182+
183+
go func() {
184+
defer wg.Done()
185+
errorChan <- destinationGitlabInstance.syncGroupAttributes(sourceGroup, destinationGroup, copyOptions)
186+
}()
187+
188+
go func() {
189+
defer wg.Done()
190+
errorChan <- sourceGitlabInstance.copyGroupAvatar(destinationGitlabInstance, destinationGroup, sourceGroup)
191+
}()
192+
193+
wg.Wait()
194+
close(errorChan)
195+
196+
return utils.MergeErrors(errorChan)
197+
}
198+
199+
// copyGroupAvatar copies the avatar from the source group to the destination group.
200+
// It first checks if the destination group already has an avatar set. If not, it downloads the avatar from the source group
201+
// and uploads it to the destination group.
202+
// The avatar is saved with a unique filename based on the current timestamp.
203+
// The function returns an error if any step fails, including downloading or uploading the avatar.
204+
func (sourceGitlabInstance *GitlabInstance) copyGroupAvatar(destinationGitlabInstance *GitlabInstance, destinationGroup *gitlab.Group, sourceGroup *gitlab.Group) error {
205+
zap.L().Debug("Checking if group avatar is already set", zap.String("group", destinationGroup.WebURL))
206+
207+
// Check if the destination group already has an avatar
208+
if destinationGroup.AvatarURL != "" {
209+
zap.L().Debug("Group avatar already set", zap.String("group", destinationGroup.WebURL), zap.String("path", destinationGroup.AvatarURL))
210+
return nil
211+
}
212+
213+
zap.L().Debug("Copying group avatar", zap.String(ROLE_SOURCE, sourceGroup.WebURL), zap.String(ROLE_DESTINATION, destinationGroup.WebURL))
214+
215+
// Download the source group avatar
216+
sourceGroupAvatar, _, err := sourceGitlabInstance.Gitlab.Groups.DownloadAvatar(sourceGroup.ID)
217+
if err != nil {
218+
return fmt.Errorf("failed to download avatar for group %s: %s", sourceGroup.WebURL, err)
219+
}
220+
221+
// Upload the avatar to the destination group
222+
filename := fmt.Sprintf("avatar-%d.png", time.Now().Unix())
223+
_, _, err = destinationGitlabInstance.Gitlab.Groups.UploadAvatar(destinationGroup.ID, sourceGroupAvatar, filename)
224+
if err != nil {
225+
return fmt.Errorf("failed to upload avatar for group %s: %s", destinationGroup.WebURL, err)
226+
}
227+
228+
return nil
229+
}
230+
231+
// syncGroupAttributes updates the destination group with settings from the source group.
232+
// It checks if any diverged group data exists and if so, it overwrites it.
233+
func (destinationGitlabInstance *GitlabInstance) syncGroupAttributes(sourceGroup *gitlab.Group, destinationGroup *gitlab.Group, copyOptions *utils.MirroringOptions) error {
234+
zap.L().Debug("Checking if group requires attributes resync", zap.String(ROLE_SOURCE, sourceGroup.FullPath), zap.String(ROLE_DESTINATION, destinationGroup.FullPath))
235+
gitlabEditOptions := &gitlab.UpdateGroupOptions{}
236+
missmatched := false
237+
if sourceGroup.Name != destinationGroup.Name {
238+
gitlabEditOptions.Name = &sourceGroup.Name
239+
missmatched = true
240+
}
241+
if sourceGroup.Description != destinationGroup.Description {
242+
gitlabEditOptions.Description = &sourceGroup.Description
243+
missmatched = true
244+
}
245+
if copyOptions.Visibility != string(destinationGroup.Visibility) {
246+
visibilityValue := utils.ConvertVisibility(copyOptions.Visibility)
247+
gitlabEditOptions.Visibility = &visibilityValue
248+
missmatched = true
249+
}
250+
251+
if missmatched {
252+
destinationGroup, _, err := destinationGitlabInstance.Gitlab.Groups.UpdateGroup(destinationGroup.ID, gitlabEditOptions)
253+
if err != nil {
254+
return fmt.Errorf("failed to edit group %s: %s", destinationGroup.FullPath, err)
255+
}
256+
zap.L().Debug("Group attributes resync completed", zap.String(ROLE_SOURCE, sourceGroup.FullPath), zap.String(ROLE_DESTINATION, destinationGroup.FullPath))
257+
} else {
258+
zap.L().Debug("Group attributes are already in sync, skipping", zap.String(ROLE_SOURCE, sourceGroup.FullPath), zap.String(ROLE_DESTINATION, destinationGroup.FullPath))
259+
}
260+
return nil
261+
}

0 commit comments

Comments
 (0)