build: concurrency upgrade

Fix the improper initial logic
Add concurrency
Add verbosity

Author: BoxBoxJason

.gitignore

@@ -16,6 +16,7 @@
 # Compile output
 dist/
 build/
+bin/
 
 # Node.js
 node_modules/
@@ -52,3 +53,6 @@ Desktop.ini
 *.cer
 *.jks
 *.pub
+
+mirroring.json
+go.sum

cmd/main.go

@@ -4,9 +4,10 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"path"
+	"path/filepath"
 	"strings"
 
+	"github.com/boxboxjason/gitlab-sync/mirroring"
 	"github.com/boxboxjason/gitlab-sync/utils"
 	"github.com/spf13/cobra"
 )
@@ -46,15 +47,21 @@ func main() {
 
 			// Check if the Mirror Mapping file path is provided
 			mirrorMappingPath = promptForMandatoryInput(mirrorMappingPath, "Input Mirror Mapping file path (MANDATORY)", "Mirror Mapping file path is mandatory", "Mirror Mapping file path set", args.NoPrompt, false)
-			utils.LogVerbose("Mirror Mapping file resolved path: " + path.Clean(mirrorMappingPath))
+			utils.LogVerbose("Mirror Mapping file resolved path: " + filepath.Clean(mirrorMappingPath))
 
 			utils.LogVerbose("Parsing mirror mapping file")
 			mapping, err := utils.OpenMirrorMapping(mirrorMappingPath)
 			if err != nil {
-				log.Fatalf("Error opening mirror mapping file: %v", err)
+				log.Fatalf("Error opening mirror mapping file: %s", err)
 			}
-			args.MirrorMapping = *mapping
 			utils.LogVerbose("Mirror mapping file parsed successfully")
+
+			err = mirroring.MirrorGitlabs(args.SourceGitlabURL, args.SourceGitlabToken, args.DestinationGitlabURL, args.DestinationGitlabToken, mapping)
+			if err != nil {
+				fmt.Println("Error during mirroring process:")
+				fmt.Println(err)
+			}
+			log.Println("Mirroring completed")
 		},
 	}
 
@@ -70,7 +77,6 @@ func main() {
 		fmt.Println(err)
 		os.Exit(1)
 	}
-
 }
 
 func promptForInput(prompt string) string {
@@ -80,10 +86,10 @@ func promptForInput(prompt string) string {
 	return input
 }
 
-func promptForMandatoryInput(defaultValue string, prompt string, errorMsg string, loggerMsg string, promptsEnabled bool, hideOutput bool) string {
+func promptForMandatoryInput(defaultValue string, prompt string, errorMsg string, loggerMsg string, promptsDisabled bool, hideOutput bool) string {
 	input := strings.TrimSpace(defaultValue)
 	if input == "" {
-		if promptsEnabled {
+		if !promptsDisabled {
 			input = strings.TrimSpace(promptForInput(prompt))
 			if input == "" {
 				log.Fatal(errorMsg)

mirroring/get.go

@@ -2,72 +2,193 @@ package mirroring
 
 import (
 	"fmt"
-	"path"
+	"log"
+	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/boxboxjason/gitlab-sync/utils"
+	gitlab "gitlab.com/gitlab-org/api/client-go"
 )
 
-func (g *GitlabInstance) fetchProjects(filters *utils.MirrorMapping) error {
+func (g *GitlabInstance) fetchProjects(projectFilters *map[string]bool, groupFilters *map[string]bool, mirrorMapping *utils.MirrorMapping, isSource bool) error {
+	sourceString := "source"
+	if !isSource {
+		sourceString = "destination"
+	}
+	utils.LogVerbosef("Fetching all projects from %s GitLab instance", sourceString)
 	projects, _, err := g.Gitlab.Projects.ListProjects(nil)
 	if err != nil {
 		return err
 	}
 
+	utils.LogVerbosef("Processing %d projects from %s GitLab instance", len(projects), sourceString)
+
+	// Create a wait group to wait for all goroutines to finish
+	var wg sync.WaitGroup
+
+	// Create a channel to limit the number of concurrent goroutines
+	concurrencyLimit := 10
+	sem := make(chan struct{}, concurrencyLimit)
+
 	for _, project := range projects {
-		// Check if the project matches the filters:
-		//   - either is in the projects map
-		//   - or path starts with any of the groups in the groups map
-		if _, ok := filters.Projects[project.PathWithNamespace]; ok {
-			g.addProject(project.PathWithNamespace, project)
-		} else {
-			for group := range filters.Groups {
-				if strings.HasPrefix(project.PathWithNamespace, group) {
-					g.addProject(project.PathWithNamespace, project)
-					break
+		wg.Add(1)
+		// Acquire a token from the semaphore
+		sem <- struct{}{}
+
+		go func(project *gitlab.Project) {
+			defer wg.Done()
+			// Release the token back to the semaphore
+			defer func() { <-sem }()
+
+			// Check if the project matches the filters:
+			//   - either is in the projects map
+			//   - or path starts with any of the groups in the groups map
+			if _, ok := (*projectFilters)[project.PathWithNamespace]; ok {
+				g.addProject(project.PathWithNamespace, project)
+			} else {
+				for group := range *groupFilters {
+					if strings.HasPrefix(project.PathWithNamespace, group) {
+						// Add the project to the gitlab instance projects cache
+						g.addProject(project.PathWithNamespace, project)
+
+						if isSource {
+							// Retrieve the corresponding group creation options from the mirror mapping
+							groupCreationOptions, ok := mirrorMapping.Groups[group]
+							if !ok {
+								log.Fatalf("Group %s not found in mirror mapping (internal error, please review script)", group)
+							}
+
+							// Calculate the relative path between the project and the group
+							relativePath, err := filepath.Rel(group, project.PathWithNamespace)
+							if err != nil {
+								log.Fatalf("Failed to calculate relative path for project %s: %s", project.PathWithNamespace, err)
+							}
+
+							// Add the project to the mirror mapping
+							mirrorMapping.AddProject(project.PathWithNamespace, &utils.ProjectMirroringOptions{
+								CI_CD_Catalog:  groupCreationOptions.CI_CD_Catalog,
+								Issues:         groupCreationOptions.Issues,
+								DestinationURL: filepath.Join(groupCreationOptions.DestinationURL, relativePath),
+							})
+						}
+						break
+					}
 				}
 			}
-		}
+		}(project)
 	}
 
-	utils.LogVerbosef("Found %d projects to mirror in the source GitLab instance", len(g.Projects))
+	wg.Wait()
+
+	utils.LogVerbosef("Found %d projects to mirror in the %s GitLab instance", len(g.Projects), sourceString)
 
 	return nil
 }
 
-func (g *GitlabInstance) fetchGroups(filters *utils.MirrorMapping) error {
+func (g *GitlabInstance) fetchGroups(groupFilters *map[string]bool, mirrorMapping *utils.MirrorMapping, isSource bool) error {
+	sourceString := "source"
+	if !isSource {
+		sourceString = "destination"
+	}
+	utils.LogVerbosef("Fetching all groups from %s GitLab instance", sourceString)
 	groups, _, err := g.Gitlab.Groups.ListGroups(nil)
 	if err != nil {
 		return err
 	}
 
+	utils.LogVerbosef("Processing %d groups from %s GitLab instance", len(groups), sourceString)
+
+	// Create a wait group to wait for all goroutines to finish
+	var wg sync.WaitGroup
+
+	// Create a channel to limit the number of concurrent goroutines
+	concurrencyLimit := 10
+	sem := make(chan struct{}, concurrencyLimit)
+
 	for _, group := range groups {
-		// Check if the group matches the filters:
-		//   - either is in the groups map
-		//   - or path starts with any of the groups in the groups map
-		//   - or is a subgroup of any of the groups in the groups map
-		if _, ok := filters.Groups[group.FullPath]; ok {
-			g.addGroup(group.FullPath, group)
-		} else {
-			for groupPath := range filters.Groups {
-				if strings.HasPrefix(group.FullPath, groupPath) {
-					g.addGroup(group.FullPath, group)
-					break
+		wg.Add(1)
+		// Acquire a token from the semaphore
+		sem <- struct{}{}
+
+		go func(group *gitlab.Group) {
+			defer wg.Done()
+			// Release the token back to the semaphore
+			defer func() { <-sem }()
+
+			// Check if the group matches the filters:
+			//   - either is in the groups map
+			//   - or path starts with any of the groups in the groups map
+			//   - or is a subgroup of any of the groups in the groups map
+			if _, ok := (*groupFilters)[group.FullPath]; ok {
+				g.addGroup(group.FullPath, group)
+			} else {
+				for groupPath := range *groupFilters {
+					if strings.HasPrefix(group.FullPath, groupPath) {
+						// Add the group to the gitlab instance groups cache
+						g.addGroup(group.FullPath, group)
+
+						if isSource {
+							// Retrieve the corresponding group creation options from the mirror mapping
+							groupCreationOptions, ok := mirrorMapping.Groups[groupPath]
+							if !ok {
+								log.Fatalf("Group %s not found in mirror mapping (internal error, please review script)", groupPath)
+							}
+
+							// Calculate the relative path between the group and the groupPath
+							relativePath, err := filepath.Rel(groupPath, group.FullPath)
+							if err != nil {
+								log.Fatalf("Failed to calculate relative path for group %s: %s", group.FullPath, err)
+							}
+
+							// Add the group to the mirror mapping
+							mirrorMapping.AddGroup(group.FullPath, &utils.GroupMirroringOptions{
+								CI_CD_Catalog:  groupCreationOptions.CI_CD_Catalog,
+								Issues:         groupCreationOptions.Issues,
+								DestinationURL: filepath.Join(groupCreationOptions.DestinationURL, relativePath),
+							})
+						}
+						break
+					}
 				}
 			}
-		}
+		}(group)
 	}
 
-	utils.LogVerbosef("Found %d groups to mirror in the source GitLab instance", len(g.Groups))
+	wg.Wait()
+
+	utils.LogVerbosef("Found %d matching groups in %s GitLab instance", len(g.Groups), sourceString)
 
 	return nil
 }
 
-func (g *GitlabInstance) getParentGroupID(projectOrGroupPath string) (int, error) {
+func fetchAll(gitlabInstance *GitlabInstance, projectFilters map[string]bool, groupFilters map[string]bool, mirrorMapping *utils.MirrorMapping, isSource bool) error {
+	wg := sync.WaitGroup{}
+	errCh := make(chan error, 2)
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		if err := gitlabInstance.fetchGroups(&groupFilters, mirrorMapping, isSource); err != nil {
+			errCh <- err
+		}
+	}()
+	go func() {
+		defer wg.Done()
+		if err := gitlabInstance.fetchProjects(&projectFilters, &groupFilters, mirrorMapping, isSource); err != nil {
+			errCh <- err
+		}
+	}()
+	wg.Wait()
+	close(errCh)
+
+	return utils.MergeErrors(errCh, 2)
+}
+
+func (g *GitlabInstance) getParentNamespaceID(projectOrGroupPath string) (int, error) {
 	parentGroupID := -1
-	parentPath := path.Dir(projectOrGroupPath)
+	parentPath := filepath.Dir(projectOrGroupPath)
 	var err error = nil
-	if parentPath != "." {
+	if parentPath != "." && parentPath != "/" {
 		// Check if parent path is already in the instance groups cache
 		if parentGroup, ok := g.Groups[parentPath]; ok {
 			parentGroupID = parentGroup.ID

mirroring/instance.go

@@ -0,0 +1,67 @@
+package mirroring
+
+import (
+	"sync"
+
+	"github.com/boxboxjason/gitlab-sync/utils"
+	gitlab "gitlab.com/gitlab-org/api/client-go"
+)
+
+type GitlabInstance struct {
+	Gitlab        *gitlab.Client
+	Projects      map[string]*gitlab.Project
+	muProjects    sync.RWMutex
+	Groups        map[string]*gitlab.Group
+	muGroups      sync.RWMutex
+	GraphQLClient *utils.GraphQLClient
+}
+
+func newGitlabInstance(gitlabURL string, gitlabToken string) (*GitlabInstance, error) {
+	gitlabClient, err := gitlab.NewClient(gitlabToken, gitlab.WithBaseURL(gitlabURL))
+	if err != nil {
+		return nil, err
+	}
+
+	gitlabInstance := &GitlabInstance{
+		Gitlab:        gitlabClient,
+		Projects:      make(map[string]*gitlab.Project),
+		Groups:        make(map[string]*gitlab.Group),
+		GraphQLClient: utils.NewGitlabGraphQLClient(gitlabToken, gitlabURL),
+	}
+
+	return gitlabInstance, nil
+}
+
+func (g *GitlabInstance) addProject(projectPath string, project *gitlab.Project) {
+	g.muProjects.Lock()
+	defer g.muProjects.Unlock()
+	g.Projects[projectPath] = project
+}
+
+func (g *GitlabInstance) getProject(projectPath string) *gitlab.Project {
+	g.muProjects.RLock()
+	defer g.muProjects.RUnlock()
+	var project *gitlab.Project
+	project, exists := g.Projects[projectPath]
+	if !exists {
+		project = nil
+	}
+	return project
+}
+
+func (g *GitlabInstance) addGroup(groupPath string, group *gitlab.Group) {
+	g.muGroups.Lock()
+	defer g.muGroups.Unlock()
+	g.Groups[groupPath] = group
+}
+
+func (g *GitlabInstance) getGroup(groupPath string) *gitlab.Group {
+	g.muGroups.RLock()
+	defer g.muGroups.RUnlock()
+	var group *gitlab.Group
+	group, exists := g.Groups[groupPath]
+	if !exists {
+		group = nil
+	}
+	return group
+}