@@ -2,207 +2,29 @@ package mirroring
22
33import (
44 "fmt"
5+ "gitlab-sync/internal/utils"
56 "path/filepath"
67 "strings"
78 "sync"
89
9- "gitlab-sync/internal/utils"
10-
11- gitlab "gitlab.com/gitlab-org/api/client-go"
1210 "go.uber.org/zap"
1311)
1412
15- // fetchProjects retrieves all projects that match the filters from the GitLab instance and stores them in the instance cache.
16- // It also updates the mirror mapping with the corresponding group creation options.
17- //
18- // The function is run in a goroutine for each project, and a wait group is used to wait for all goroutines to finish.
19- func (g * GitlabInstance ) fetchProjects (projectFilters * map [string ]bool , groupFilters * map [string ]bool , mirrorMapping * utils.MirrorMapping , isSource bool ) error {
20- sourceString := "source"
21- if ! isSource {
22- sourceString = "destination"
23- }
24- zap .L ().Debug ("Fetching all projects from GitLab instance" , zap .String ("role" , sourceString ))
25- projects , _ , err := g .Gitlab .Projects .ListProjects (nil )
26- if err != nil {
27- return err
28- }
29-
30- zap .L ().Debug ("Processing projects from GitLab instance" , zap .String ("role" , sourceString ), zap .Int ("projects" , len (projects )))
31-
32- // Create a wait group to wait for all goroutines to finish
33- var wg sync.WaitGroup
34-
35- for _ , project := range projects {
36- wg .Add (1 )
37-
38- go func (project * gitlab.Project ) {
39- defer wg .Done ()
40-
41- group , matches := g .checkPathMatchesFilters (project .PathWithNamespace , projectFilters , groupFilters )
42- if matches {
43- g .storeProject (project , group , mirrorMapping , isSource )
44- }
45-
46- }(project )
47- }
48-
49- wg .Wait ()
50-
51- zap .L ().Debug ("Found matching projects in the GitLab instance" , zap .String ("role" , sourceString ), zap .Int ("projects" , len (g .Projects )))
52- return nil
53- }
54-
55- // storeProject stores the project in the Gitlab instance projects cache
56- // and updates the mirror mapping with the corresponding group creation options.
57- func (g * GitlabInstance ) storeProject (project * gitlab.Project , parentGroupPath string , mirrorMapping * utils.MirrorMapping , isSource bool ) {
58- // Add the project to the gitlab instance projects cache
59- g .addProject (project .PathWithNamespace , project )
60-
61- if isSource {
62- zap .L ().Debug ("Storing project in mirror mapping" , zap .String ("project" , project .HTTPURLToRepo ), zap .String ("group" , parentGroupPath ))
63- // Retrieve the corresponding group creation options from the mirror mapping
64- groupCreationOptions , ok := mirrorMapping .GetGroup (parentGroupPath )
65- if ! ok {
66- zap .L ().Error ("Group not found in mirror mapping" , zap .String ("group" , parentGroupPath ))
67- return
68- }
69-
70- // Calculate the relative path between the project and the group
71- relativePath , err := filepath .Rel (parentGroupPath , project .PathWithNamespace )
72- if err != nil {
73- zap .L ().Error ("Failed to calculate relative path for project" , zap .String ("project" , project .HTTPURLToRepo ), zap .String ("group" , parentGroupPath ), zap .Error (err ))
74- return
75- }
76-
77- // Add the project to the mirror mapping with the corresponding group creation options
78- mirrorMapping .AddProject (project .PathWithNamespace , & utils.MirroringOptions {
79- DestinationPath : filepath .Join (groupCreationOptions .DestinationPath , relativePath ),
80- CI_CD_Catalog : groupCreationOptions .CI_CD_Catalog ,
81- Issues : groupCreationOptions .Issues ,
82- MirrorTriggerBuilds : groupCreationOptions .MirrorTriggerBuilds ,
83- Visibility : groupCreationOptions .Visibility ,
84- MirrorReleases : groupCreationOptions .MirrorReleases ,
85- })
86- }
87- }
88-
89- // fetchGroups retrieves all groups that match the filters from the GitLab instance and stores them in the instance cache.
90- // It also updates the mirror mapping with the corresponding group creation options.
91- //
92- // The function is run in a goroutine for each group, and a wait group is used to wait for all goroutines to finish.
93- func (g * GitlabInstance ) fetchGroups (groupFilters * map [string ]bool , mirrorMapping * utils.MirrorMapping , isSource bool ) error {
94- sourceString := "source"
95- if ! isSource {
96- sourceString = "destination"
97- }
98- zap .L ().Debug ("Fetching all groups from GitLab instance" , zap .String ("role" , sourceString ))
99- groups , _ , err := g .Gitlab .Groups .ListGroups (nil )
100- if err != nil {
101- return err
102- }
103-
104- zap .L ().Debug ("Processing groups from GitLab instance" , zap .String ("role" , sourceString ), zap .Int ("groups" , len (groups )))
105-
106- // Create a wait group to wait for all goroutines to finish
107- var wg sync.WaitGroup
108-
109- for _ , group := range groups {
110- wg .Add (1 )
111-
112- go func (group * gitlab.Group ) {
113- defer wg .Done ()
114-
115- groupPath , matches := g .checkPathMatchesFilters (group .FullPath , nil , groupFilters )
116- if matches {
117- g .storeGroup (group , groupPath , mirrorMapping , isSource )
118- }
119- }(group )
120- }
121-
122- wg .Wait ()
123-
124- zap .L ().Debug ("Found matching groups in the GitLab instance" , zap .String ("role" , sourceString ), zap .Int ("groups" , len (g .Groups )))
125-
126- return nil
127- }
128-
129- // storeGroup stores the group in the Gitlab instance groups cache
130- // and updates the mirror mapping with the corresponding group creation options.
131- func (g * GitlabInstance ) storeGroup (group * gitlab.Group , parentGroupPath string , mirrorMapping * utils.MirrorMapping , isSource bool ) {
132- if group != nil {
133- // Add the group to the gitlab instance groups cache
134- g .addGroup (group .FullPath , group )
135-
136- if isSource {
137- zap .L ().Debug ("Storing group in mirror mapping" , zap .String ("group" , group .FullPath ), zap .String ("parentGroup" , parentGroupPath ))
138- // Retrieve the corresponding group creation options from the mirror mapping
139- groupCreationOptions , ok := mirrorMapping .Groups [parentGroupPath ]
140- if ! ok {
141- zap .L ().Error ("Group not found in mirror mapping" , zap .String ("group" , parentGroupPath ))
142- return
143- }
144-
145- // Calculate the relative path between the group and the parent group
146- relativePath , err := filepath .Rel (parentGroupPath , group .FullPath )
147- if err != nil {
148- zap .L ().Error ("Failed to calculate relative path for group" , zap .String ("group" , group .FullPath ), zap .String ("parentGroup" , parentGroupPath ), zap .Error (err ))
149- return
150- }
151-
152- // Add the group to the mirror mapping
153- mirrorMapping .AddGroup (group .FullPath , & utils.MirroringOptions {
154- DestinationPath : filepath .Join (groupCreationOptions .DestinationPath , relativePath ),
155- CI_CD_Catalog : groupCreationOptions .CI_CD_Catalog ,
156- Issues : groupCreationOptions .Issues ,
157- MirrorTriggerBuilds : groupCreationOptions .MirrorTriggerBuilds ,
158- Visibility : groupCreationOptions .Visibility ,
159- MirrorReleases : groupCreationOptions .MirrorReleases ,
160- })
161- }
162- } else {
163- zap .L ().Error ("Failed to store group in mirror mapping: nil group" )
164- }
165- }
166-
167- // checkPathMatchesFilters checks if the resources matches the filters
168- // - either is in the projects map
169- // - or path starts with any of the groups in the groups map
170- //
171- // In the case of a match with a group, it returns the group path
172- func (g * GitlabInstance ) checkPathMatchesFilters (resourcePath string , projectFilters * map [string ]bool , groupFilters * map [string ]bool ) (string , bool ) {
173- zap .L ().Debug ("Checking if path matches filters" , zap .String ("path" , resourcePath ))
174- if projectFilters != nil {
175- if _ , ok := (* projectFilters )[resourcePath ]; ok {
176- zap .L ().Debug ("Resource path matches project filter" , zap .String ("project" , resourcePath ))
177- return "" , true
178- }
179- }
180- if groupFilters != nil {
181- for groupPath := range * groupFilters {
182- if strings .HasPrefix (resourcePath , groupPath ) {
183- zap .L ().Debug ("Resource path matches group filter" , zap .String ("resource" , resourcePath ), zap .String ("group" , groupPath ))
184- return groupPath , true
185- }
186- }
187- }
188- return "" , false
189- }
190-
19113// fetchAll retrieves all projects and groups from the GitLab instance
19214// that match the filters and stores them in the instance cache.
193- func fetchAll (gitlabInstance * GitlabInstance , projectFilters map [string ]bool , groupFilters map [string ]bool , mirrorMapping * utils.MirrorMapping , isSource bool ) error {
15+ func fetchAll (gitlabInstance * GitlabInstance , projectFilters map [string ]struct {} , groupFilters map [string ]struct {} , mirrorMapping * utils.MirrorMapping ) error {
19416 wg := sync.WaitGroup {}
19517 errCh := make (chan error , 2 )
19618 wg .Add (2 )
19719 go func () {
19820 defer wg .Done ()
199- if err := gitlabInstance .fetchGroups (& groupFilters , mirrorMapping , isSource ); err != nil {
21+ if err := gitlabInstance .fetchAndProcessGroups (& groupFilters , mirrorMapping ); err != nil {
20022 errCh <- err
20123 }
20224 }()
20325 go func () {
20426 defer wg .Done ()
205- if err := gitlabInstance .fetchProjects (& projectFilters , & groupFilters , mirrorMapping , isSource ); err != nil {
27+ if err := gitlabInstance .fetchAndProcessProjects (& projectFilters , & groupFilters , mirrorMapping ); err != nil {
20628 errCh <- err
20729 }
20830 }()
@@ -230,3 +52,27 @@ func (g *GitlabInstance) getParentNamespaceID(projectOrGroupPath string) (int, e
23052 }
23153 return parentGroupID , err
23254}
55+
56+ // checkPathMatchesFilters checks if the resources matches the filters
57+ // - either is in the projects map
58+ // - or path starts with any of the groups in the groups map
59+ //
60+ // In the case of a match with a group, it returns the group path
61+ func (g * GitlabInstance ) checkPathMatchesFilters (resourcePath string , projectFilters * map [string ]struct {}, groupFilters * map [string ]struct {}) (string , bool ) {
62+ zap .L ().Debug ("Checking if path matches filters" , zap .String ("path" , resourcePath ))
63+ if projectFilters != nil {
64+ if _ , ok := (* projectFilters )[resourcePath ]; ok {
65+ zap .L ().Debug ("Resource path matches project filter" , zap .String ("project" , resourcePath ))
66+ return "" , true
67+ }
68+ }
69+ if groupFilters != nil {
70+ for groupPath := range * groupFilters {
71+ if strings .HasPrefix (resourcePath , groupPath ) {
72+ zap .L ().Debug ("Resource path matches group filter" , zap .String ("resource" , resourcePath ), zap .String ("group" , groupPath ))
73+ return groupPath , true
74+ }
75+ }
76+ }
77+ return "" , false
78+ }
0 commit comments