test: add unit tests on post / put methods

Upgrade helper module to support more API response mocks
Fix all errors detected by unit tests

Author: BoxBoxJason

internal/mirroring/get_group.go

@@ -169,7 +169,7 @@ func (g *GitlabInstance) fetchAndProcessGroupsLargeInstance(groupFilters *map[st
 }
 
 // fetchAndProcessGroupRecursive fetches a group and its projects recursively
-// It uses a wait group to ensure that all goroutines finish before returning
+// It uses a wait group to ensure that all goroutines finish befopidpre returning
 // It sends the fetched group to the allGroupsChannel and the projects to the allProjectsChanel
 //
 // gid can be either an int, a string or a *gitlab.Group
@@ -188,7 +188,8 @@ func (g *GitlabInstance) fetchAndProcessGroupRecursive(gid any, fetchOriginPath
 	case *gitlab.Group:
 		group = v
 	default:
-		errChan <- fmt.Errorf("invalid group ID type %T (%s)", gid, gid)
+		errChan <- fmt.Errorf("invalid group ID type %T (%v)", gid, gid)
+		return
 	}
 	if group != nil {
 		g.storeGroup(group, fetchOriginPath, mirrorMapping)

internal/mirroring/get_project.go

@@ -149,14 +149,14 @@ func (g *GitlabInstance) fetchAndProcessProjectsBigInstance(projectFilters *map[
 	// Fetch each project in parallel
 	var wg sync.WaitGroup
 	projectsChan := make(chan *gitlab.Project, len(*projectFilters))
-	errCh := make(chan error)
+	errCh := make(chan error, len(*projectFilters))
 	wg.Add(len(*projectFilters))
 	for project := range *projectFilters {
 		go func(projectPath string) {
 			defer wg.Done()
 			p, _, err := g.Gitlab.Projects.GetProject(projectPath, &gitlab.GetProjectOptions{})
 			if err != nil {
-				errCh <- err
+				errCh <- fmt.Errorf("failed to retrieve project %s: %v", projectPath, err)
 				return
 			}
 			projectsChan <- p

internal/mirroring/get_project_test.go

@@ -0,0 +1,103 @@
+package mirroring
+
+import (
+	"gitlab-sync/internal/utils"
+	"testing"
+)
+
+func TestFetchAndProcessProjectsBigInstance(t *testing.T) {
+	tests := []struct {
+		name             string
+		role             string
+		projectFilters   map[string]struct{}
+		expectedProjects map[string]struct{}
+		expectError      bool
+	}{
+		{
+			name: "Test with source role, 1 project only, no error",
+			role: ROLE_SOURCE,
+			projectFilters: map[string]struct{}{
+				TEST_PROJECT.PathWithNamespace: {},
+			},
+			expectedProjects: map[string]struct{}{
+				TEST_PROJECT.PathWithNamespace: {},
+			},
+		},
+		{
+			name: "Test with destination role, 1 project only, no error",
+			role: ROLE_DESTINATION,
+			projectFilters: map[string]struct{}{
+				TEST_PROJECT.PathWithNamespace: {},
+			},
+			expectedProjects: map[string]struct{}{
+				TEST_PROJECT.PathWithNamespace: {},
+			},
+		},
+		{
+			name: "Test with source role, 2 projects, no error",
+			role: ROLE_SOURCE,
+			projectFilters: map[string]struct{}{
+				TEST_PROJECT.PathWithNamespace:   {},
+				TEST_PROJECT_2.PathWithNamespace: {},
+			},
+			expectedProjects: map[string]struct{}{
+				TEST_PROJECT.PathWithNamespace:   {},
+				TEST_PROJECT_2.PathWithNamespace: {},
+			},
+		},
+		{
+			name: "Test with destination role, 2 projects, no error",
+			role: ROLE_DESTINATION,
+			projectFilters: map[string]struct{}{
+				TEST_PROJECT.PathWithNamespace:   {},
+				TEST_PROJECT_2.PathWithNamespace: {},
+			},
+			expectedProjects: map[string]struct{}{
+				TEST_PROJECT.PathWithNamespace:   {},
+				TEST_PROJECT_2.PathWithNamespace: {},
+			},
+		},
+		{
+			name: "Test with source role, 1 project, 1 error",
+			role: ROLE_SOURCE,
+			projectFilters: map[string]struct{}{
+				TEST_PROJECT.PathWithNamespace:    {},
+				INVALID_PROJECT.PathWithNamespace: {},
+			},
+			expectedProjects: map[string]struct{}{
+				TEST_PROJECT.PathWithNamespace: {},
+			},
+			expectError: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+
+			gitlabMirrorArgs := &utils.MirrorMapping{
+				Projects: map[string]*utils.MirroringOptions{
+					TEST_PROJECT.PathWithNamespace: {
+						DestinationPath: TEST_PROJECT.PathWithNamespace,
+					},
+				},
+			}
+
+			_, gitlabInstance := setupTestServer(t, test.role, INSTANCE_SIZE_BIG)
+
+			err := gitlabInstance.fetchAndProcessProjectsBigInstance(&test.projectFilters, gitlabMirrorArgs)
+			if (err != nil) != test.expectError {
+				t.Fatalf("Expected error: %v, got: %v", test.expectError, err)
+			}
+			if len(gitlabInstance.Projects) != len(test.expectedProjects) {
+				t.Fatalf("Expected %d projects, got %d", len(test.expectedProjects), len(gitlabInstance.Projects))
+			}
+			for project := range test.expectedProjects {
+				if _, ok := gitlabInstance.Projects[project]; !ok {
+					t.Errorf("Expected project %s to be in the cache", project)
+				}
+			}
+		})
+	}
+
+}

internal/mirroring/get_test.go

@@ -190,7 +190,6 @@ func TestFetchAll(t *testing.T) {
 			if _, ok := gitlabInstance.Groups[TEST_GROUP_2.FullPath]; !ok {
 				t.Errorf("expected group %s not found in %s %s instance cache", TEST_GROUP_2.FullPath, gitlabInstance.Role, gitlabInstance.InstanceSize)
 			}
-
 		})
 	}
 

internal/mirroring/helper.go

@@ -0,0 +1,32 @@
+package mirroring
+
+import (
+	"gitlab-sync/internal/utils"
+	"sort"
+
+	"go.uber.org/zap"
+)
+
+// reverseGroupMirrorMap reverses the mirror mapping to get the source group path for each destination group.
+// It creates a map where the keys are the destination group paths and the values are the source group paths.
+// The function also returns a sorted list of destination group paths.
+func (g *GitlabInstance) reverseGroupMirrorMap(mirrorMapping *utils.MirrorMapping) (map[string]string, []string) {
+	var reversedMirrorMap map[string]string
+	destinationGroupPaths := make([]string, 0, len(g.Groups))
+	if mirrorMapping != nil {
+		// Reverse the mirror mapping to get the source group path for each destination group
+		reversedMirrorMap = make(map[string]string, len(mirrorMapping.Groups))
+		// Extract the keys (group paths) and sort them
+		// This ensures that the parent groups are created before their children
+		for sourceGroupPath, createOptions := range mirrorMapping.Groups {
+			if _, ok := reversedMirrorMap[createOptions.DestinationPath]; ok {
+				zap.L().Error("duplicate destination path found in mirror mapping", zap.String("destinationPath", createOptions.DestinationPath))
+				continue
+			}
+			reversedMirrorMap[createOptions.DestinationPath] = sourceGroupPath
+			destinationGroupPaths = append(destinationGroupPaths, createOptions.DestinationPath)
+		}
+		sort.Strings(destinationGroupPaths)
+	}
+	return reversedMirrorMap, destinationGroupPaths
+}