Skip to content

Commit 1b573c7

Browse files
BoxBoxJasonBoxBoxJason
committed
test: add unit tests on get methods
Create a helper module to mock a gitlab server with responses for group / projects get and post requests Update the MergeError function to support both error slices and channels Fix all errors detected by unit tests (thread locks on large instances for instance)
1 parent 0986273 commit 1b573c7

File tree

12 files changed

+562
-63
lines changed

12 files changed

+562
-63
lines changed

‎internal/mirroring/get.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,19 @@ import (
1212

1313
// fetchAll retrieves all projects and groups from the GitLab instance
1414
// that match the filters and stores them in the instance cache.
15-
func fetchAll(gitlabInstance *GitlabInstance, projectFilters map[string]struct{}, groupFilters map[string]struct{}, mirrorMapping *utils.MirrorMapping) error {
15+
func (g *GitlabInstance) fetchAll(projectFilters map[string]struct{}, groupFilters map[string]struct{}, mirrorMapping *utils.MirrorMapping) error {
1616
wg := sync.WaitGroup{}
1717
errCh := make(chan error, 2)
1818
wg.Add(2)
1919
go func() {
2020
defer wg.Done()
21-
if err := gitlabInstance.fetchAndProcessGroups(&groupFilters, mirrorMapping); err != nil {
21+
if err := g.fetchAndProcessGroups(&groupFilters, mirrorMapping); err != nil {
2222
errCh <- err
2323
}
2424
}()
2525
go func() {
2626
defer wg.Done()
27-
if err := gitlabInstance.fetchAndProcessProjects(&projectFilters, &groupFilters, mirrorMapping); err != nil {
27+
if err := g.fetchAndProcessProjects(&projectFilters, &groupFilters, mirrorMapping); err != nil {
2828
errCh <- err
2929
}
3030
}()
@@ -58,7 +58,7 @@ func (g *GitlabInstance) getParentNamespaceID(projectOrGroupPath string) (int, e
5858
// - or path starts with any of the groups in the groups map
5959
//
6060
// In the case of a match with a group, it returns the group path
61-
func (g *GitlabInstance) checkPathMatchesFilters(resourcePath string, projectFilters *map[string]struct{}, groupFilters *map[string]struct{}) (string, bool) {
61+
func checkPathMatchesFilters(resourcePath string, projectFilters *map[string]struct{}, groupFilters *map[string]struct{}) (string, bool) {
6262
zap.L().Debug("Checking if path matches filters", zap.String("path", resourcePath))
6363
if projectFilters != nil {
6464
if _, ok := (*projectFilters)[resourcePath]; ok {

‎internal/mirroring/get_group.go‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ func (g *GitlabInstance) processGroupsSmallInstance(allGroups []*gitlab.Group, g
119119
go func(group *gitlab.Group) {
120120
defer wg.Done()
121121

122-
groupPath, matches := g.checkPathMatchesFilters(group.FullPath, nil, groupFilters)
122+
groupPath, matches := checkPathMatchesFilters(group.FullPath, nil, groupFilters)
123123
if matches {
124124
g.storeGroup(group, groupPath, mirrorMapping)
125125
}
@@ -141,14 +141,31 @@ func (g *GitlabInstance) fetchAndProcessGroupsLargeInstance(groupFilters *map[st
141141
var wg sync.WaitGroup
142142
wg.Add(len(*groupFilters))
143143

144+
var (
145+
errs []error
146+
errMu sync.Mutex
147+
)
148+
149+
// Start an error collector goroutine.
150+
go func() {
151+
// This goroutine will run until errChan is closed.
152+
for err := range errChan {
153+
if err != nil {
154+
errMu.Lock()
155+
errs = append(errs, err)
156+
errMu.Unlock()
157+
}
158+
}
159+
}()
160+
144161
for groupPath := range *groupFilters {
145162
go g.fetchAndProcessGroupRecursive(groupPath, groupPath, mirrorMapping, errChan, &wg)
146163
}
147164

148165
// Wait for all goroutines to finish
149166
wg.Wait()
150167
close(errChan)
151-
return utils.MergeErrors(errChan, 2)
168+
return utils.MergeErrors(errs, 2)
152169
}
153170

154171
// fetchAndProcessGroupRecursive fetches a group and its projects recursively
@@ -166,7 +183,7 @@ func (g *GitlabInstance) fetchAndProcessGroupRecursive(gid any, fetchOriginPath
166183
case int, string:
167184
group, _, err = g.Gitlab.Groups.GetGroup(gid, &gitlab.GetGroupOptions{WithProjects: gitlab.Ptr(false)})
168185
if err != nil {
169-
errChan <- err
186+
errChan <- fmt.Errorf("failed to retrieve group %s: %v", gid, err)
170187
}
171188
case *gitlab.Group:
172189
group = v
@@ -200,7 +217,7 @@ func (g *GitlabInstance) fetchAndProcessGroupSubgroups(group *gitlab.Group, fetc
200217
for {
201218
subgroups, resp, err := g.Gitlab.Groups.ListSubGroups(group.ID, fetchOpts)
202219
if err != nil {
203-
errChan <- err
220+
errChan <- fmt.Errorf("failed to retrieve subgroups for group %s: %v", group.FullPath, err)
204221
return
205222
}
206223
for _, subgroup := range subgroups {

‎internal/mirroring/get_project.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package mirroring
22

33
import (
4+
"fmt"
45
"path/filepath"
56
"sync"
67

@@ -124,7 +125,7 @@ func (g *GitlabInstance) processProjectsSmallInstance(allProjects []*gitlab.Proj
124125
go func(project *gitlab.Project) {
125126
defer wg.Done()
126127

127-
group, matches := g.checkPathMatchesFilters(project.PathWithNamespace, projectFilters, groupFilters)
128+
group, matches := checkPathMatchesFilters(project.PathWithNamespace, projectFilters, groupFilters)
128129
if matches {
129130
g.storeProject(project, group, mirrorMapping)
130131
}
@@ -187,7 +188,7 @@ func (g *GitlabInstance) fetchAndProcessGroupProjects(group *gitlab.Group, fetch
187188
for {
188189
projects, resp, err := g.Gitlab.Groups.ListGroupProjects(group.ID, opt)
189190
if err != nil {
190-
errChan <- err
191+
errChan <- fmt.Errorf("failed to retrieve projects for group %s: %v", group.Name, err)
191192
}
192193

193194
for _, project := range projects {

‎internal/mirroring/get_test.go‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
package mirroring
2+
3+
import (
4+
"gitlab-sync/internal/utils"
5+
"testing"
6+
)
7+
8+
func TestCheckPathMatchesFilters(t *testing.T) {
9+
tests := []struct {
10+
name string
11+
path string
12+
projectFilters map[string]struct{}
13+
groupFilters map[string]struct{}
14+
expected bool
15+
}{
16+
{
17+
name: "Project in project filters AND group filters",
18+
path: TEST_PROJECT.PathWithNamespace,
19+
projectFilters: map[string]struct{}{
20+
TEST_PROJECT.PathWithNamespace: {},
21+
},
22+
groupFilters: map[string]struct{}{
23+
TEST_GROUP.FullPath: {},
24+
},
25+
expected: true,
26+
},
27+
{
28+
name: "Project in project filters but not in group filters",
29+
path: TEST_PROJECT.PathWithNamespace,
30+
projectFilters: map[string]struct{}{
31+
TEST_PROJECT.PathWithNamespace: {},
32+
},
33+
groupFilters: map[string]struct{}{},
34+
expected: true,
35+
},
36+
{
37+
name: "Project not in project filters but in group filters",
38+
path: TEST_PROJECT.PathWithNamespace,
39+
projectFilters: map[string]struct{}{},
40+
groupFilters: map[string]struct{}{
41+
TEST_GROUP.FullPath: {},
42+
},
43+
expected: true,
44+
},
45+
{
46+
name: "Project not in project filters and not in group filters",
47+
path: TEST_PROJECT.PathWithNamespace,
48+
projectFilters: map[string]struct{}{},
49+
groupFilters: map[string]struct{}{},
50+
expected: false,
51+
},
52+
}
53+
54+
// Iterate over the test cases
55+
for _, test := range tests {
56+
t.Run(test.name, func(t *testing.T) {
57+
t.Parallel()
58+
// Call the function with the test case parameters
59+
_, got := checkPathMatchesFilters(test.path, &test.projectFilters, &test.groupFilters)
60+
61+
// Check if the result matches the expected value
62+
if got != test.expected {
63+
t.Errorf("expected %v, got %v", test.expected, got)
64+
}
65+
})
66+
}
67+
68+
}
69+
70+
func TestGetParentNamespaceID(t *testing.T) {
71+
gitlabInstance := setupTestGitlabInstance(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL)
72+
gitlabInstance.addGroup(TEST_GROUP)
73+
gitlabInstance.addProject(TEST_PROJECT)
74+
75+
tests := []struct {
76+
name string
77+
path string
78+
expectedID int
79+
expectedError bool
80+
}{
81+
{
82+
name: "Valid parent path",
83+
path: TEST_PROJECT.PathWithNamespace,
84+
expectedID: TEST_GROUP.ID,
85+
expectedError: false,
86+
},
87+
{
88+
name: "Invalid parent path",
89+
path: "invalid/path",
90+
expectedID: -1,
91+
expectedError: true,
92+
},
93+
{
94+
name: "Existing resource with no parent path",
95+
path: TEST_GROUP.FullPath,
96+
expectedID: -1,
97+
expectedError: true,
98+
},
99+
}
100+
101+
// Iterate over the test cases
102+
for _, test := range tests {
103+
t.Run(test.name, func(t *testing.T) {
104+
t.Parallel()
105+
// Call the function with the test case parameters
106+
gotID, err := gitlabInstance.getParentNamespaceID(test.path)
107+
108+
// Check if the result matches the expected value
109+
if gotID != test.expectedID {
110+
t.Errorf("expected %d, got %d", test.expectedID, gotID)
111+
}
112+
113+
// Check if an error was expected
114+
if (err != nil) != test.expectedError {
115+
t.Errorf("expected error: %v, got: %v", test.expectedError, err)
116+
}
117+
})
118+
}
119+
}
120+
121+
func TestFetchAll(t *testing.T) {
122+
123+
tests := []struct {
124+
name string
125+
instanceSize string
126+
role string
127+
expectedError bool
128+
}{
129+
{
130+
name: "Fetch all Destination with small instance size",
131+
instanceSize: INSTANCE_SIZE_SMALL,
132+
role: ROLE_DESTINATION,
133+
expectedError: false,
134+
},
135+
{
136+
name: "Fetch all Destination with big instance size",
137+
instanceSize: INSTANCE_SIZE_BIG,
138+
role: ROLE_DESTINATION,
139+
expectedError: false,
140+
},
141+
{
142+
name: "Fetch all Source with small instance size",
143+
instanceSize: INSTANCE_SIZE_SMALL,
144+
role: ROLE_SOURCE,
145+
expectedError: false,
146+
},
147+
{
148+
name: "Fetch all Source with big instance size",
149+
instanceSize: INSTANCE_SIZE_BIG,
150+
role: ROLE_SOURCE,
151+
expectedError: false,
152+
},
153+
}
154+
155+
projectFilters := map[string]struct{}{
156+
TEST_PROJECT.PathWithNamespace: {},
157+
}
158+
groupFilters := map[string]struct{}{
159+
TEST_GROUP_2.FullPath: {},
160+
}
161+
gitlabMirrorArgs := &utils.MirrorMapping{
162+
Projects: map[string]*utils.MirroringOptions{},
163+
Groups: map[string]*utils.MirroringOptions{},
164+
}
165+
gitlabMirrorArgs.Projects[TEST_PROJECT.PathWithNamespace] = &utils.MirroringOptions{
166+
DestinationPath: TEST_PROJECT.PathWithNamespace,
167+
}
168+
gitlabMirrorArgs.Groups[TEST_GROUP_2.FullPath] = &utils.MirroringOptions{
169+
DestinationPath: TEST_GROUP_2.FullPath,
170+
}
171+
172+
// Iterate over the test cases
173+
for _, test := range tests {
174+
t.Run(test.name, func(t *testing.T) {
175+
t.Parallel()
176+
_, gitlabInstance := setupTestServer(t, test.role, test.instanceSize)
177+
178+
// Call the function with the test case parameters
179+
err := gitlabInstance.fetchAll(projectFilters, groupFilters, gitlabMirrorArgs)
180+
181+
// Check if an error was expected
182+
if (err != nil) != test.expectedError {
183+
t.Errorf("expected error: %v, got: %v", test.expectedError, err)
184+
}
185+
186+
//Check if the instance cache contains the expected projects and groups
187+
if _, ok := gitlabInstance.Projects[TEST_PROJECT.PathWithNamespace]; !ok {
188+
t.Errorf("expected project %s not found in %s %s instance cache", TEST_PROJECT.PathWithNamespace, gitlabInstance.Role, gitlabInstance.InstanceSize)
189+
}
190+
if _, ok := gitlabInstance.Groups[TEST_GROUP_2.FullPath]; !ok {
191+
t.Errorf("expected group %s not found in %s %s instance cache", TEST_GROUP_2.FullPath, gitlabInstance.Role, gitlabInstance.InstanceSize)
192+
}
193+
194+
})
195+
}
196+
197+
}

0 commit comments

Comments
 (0)