refactor: use existing modules

get rid of custom logic modules for logger and concurrency management. Instead use zap for logging and let go manage concurrency limit on its own.
Move files in "internal" because project is expanding
Update go version

Author: BoxBoxJason

.github/workflows/go.yml

@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.23.7'
+          go-version: '1.24.2'
       - run: go mod tidy
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v7
@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.23.7'
+          go-version: '1.24.2'
       - run: go mod tidy
       - name: Run Gosec Security Scanner
         uses: securego/gosec@master
@@ -47,7 +47,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.23.7'
+          go-version: '1.24.2'
       - run: go mod tidy
       - name: go-test
         run: |

cmd/main.go

@@ -8,10 +8,12 @@ import (
 	"strings"
 	"time"
 
-	"gitlab-sync/mirroring"
-	"gitlab-sync/utils"
+	"gitlab-sync/internal/mirroring"
+	"gitlab-sync/internal/utils"
 
 	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 )
 
 var (
@@ -29,34 +31,27 @@ func main() {
 		Short:   "Copy and enable mirroring of gitlab projects and groups",
 		Long:    "Fully customizable gitlab repositories and groups mirroring between two (or one) gitlab instances.",
 		Run: func(cmd *cobra.Command, cmdArgs []string) {
-			// Set the concurrency limit
-			if args.Concurrency == -1 {
-				args.Concurrency = 10000
-			} else if args.Concurrency == 0 {
-				log.Fatal("concurrency limit must be -1 (no limit) or strictly greater than 0")
-			}
-			utils.ConcurrencyManager.SetLimit(args.Concurrency)
+			// Set up the logger
+			setupZapLogger(args.Verbose)
+			zap.L().Debug("Verbose mode enabled")
+			zap.L().Debug("Parsing command line arguments")
 
 			// Obtain the retry count
 			if args.Retry == -1 {
 				args.Retry = 10000
 			} else if args.Retry == 0 {
-				log.Fatal("retry count must be -1 (no limit) or strictly greater than 0")
+				zap.L().Fatal("retry count must be -1 (no limit) or strictly greater than 0")
 			}
 
 			// Set the timeout for GitLab API requests
 			if timeout == -1 {
 				args.Timeout = time.Duration(10000 * time.Second)
 			} else if timeout == 0 {
-				log.Fatal("timeout must be -1 (no limit) or strictly greater than 0")
+				zap.L().Fatal("timeout must be -1 (no limit) or strictly greater than 0")
 			} else {
 				args.Timeout = time.Duration(timeout) * time.Second
 			}
 
-			utils.SetVerbose(args.Verbose)
-			utils.LogVerbose("Verbose mode enabled")
-			utils.LogVerbose("Parsing command line arguments")
-
 			// Check if the source GitLab URL is provided
 			args.SourceGitlabURL = promptForMandatoryInput(args.SourceGitlabURL, "Input Source GitLab URL (MANDATORY)", "Source GitLab URL is mandatory", "Source GitLab URL", args.NoPrompt, false)
 
@@ -68,14 +63,14 @@ func main() {
 
 			// Check if the Mirror Mapping file path is provided
 			mirrorMappingPath = promptForMandatoryInput(mirrorMappingPath, "Input Mirror Mapping file path (MANDATORY)", "Mirror Mapping file path is mandatory", "Mirror Mapping file path set", args.NoPrompt, false)
-			utils.LogVerbose("Mirror Mapping file resolved path: " + filepath.Clean(mirrorMappingPath))
+			zap.L().Debug("Mirror Mapping file resolved path: " + filepath.Clean(mirrorMappingPath))
 
-			utils.LogVerbose("Parsing mirror mapping file")
+			zap.L().Debug("Parsing mirror mapping file")
 			mapping, err := utils.OpenMirrorMapping(mirrorMappingPath)
 			if err != nil {
-				log.Fatalf("Error opening mirror mapping file: %s", err)
+				zap.L().Sugar().Fatalf("Error opening mirror mapping file: %s", err)
 			}
-			utils.LogVerbose("Mirror mapping file parsed successfully")
+			zap.L().Debug("Mirror mapping file parsed successfully")
 			args.MirrorMapping = mapping
 
 			err = mirroring.MirrorGitlabs(&args)
@@ -96,7 +91,6 @@ func main() {
 	rootCmd.Flags().StringVar(&mirrorMappingPath, "mirror-mapping", os.Getenv("MIRROR_MAPPING"), "Path to the mirror mapping file")
 	rootCmd.Flags().BoolVar(&args.DryRun, "dry-run", false, "Perform a dry run without making any changes")
 	rootCmd.Flags().IntVarP(&timeout, "timeout", "t", 30, "Timeout in seconds for GitLab API requests")
-	rootCmd.Flags().IntVarP(&args.Concurrency, "concurrency", "c", 10, "Max number of concurrent requests")
 	rootCmd.Flags().IntVarP(&args.Retry, "retry", "r", 3, "Number of retries for failed requests")
 
 	if err := rootCmd.Execute(); err != nil {
@@ -118,16 +112,37 @@ func promptForMandatoryInput(defaultValue string, prompt string, errorMsg string
 		if !promptsDisabled {
 			input = strings.TrimSpace(promptForInput(prompt))
 			if input == "" {
-				log.Fatal(errorMsg)
+				zap.L().Fatal(errorMsg)
 			}
 			if !hideOutput {
-				utils.LogVerbose(loggerMsg + ": " + input)
+				zap.L().Debug(loggerMsg + ": " + input)
 			} else {
-				utils.LogVerbose(loggerMsg)
+				zap.L().Debug(loggerMsg)
 			}
 		} else {
-			log.Fatalf("Prompting is disabled, %s", errorMsg)
+			zap.L().Sugar().Fatal("Prompting is disabled, %s", errorMsg)
 		}
 	}
 	return input
 }
+
+func setupZapLogger(verbose bool) {
+	// Set up the logger configuration
+	config := zap.NewProductionConfig()
+	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+	config.EncoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder
+	if verbose {
+		config.Level.SetLevel(zapcore.DebugLevel)
+	} else {
+		config.Level.SetLevel(zapcore.InfoLevel)
+	}
+
+	// Create the logger
+	logger, err := config.Build()
+	if err != nil {
+		zap.L().Fatal("Failed to create logger: " + err.Error())
+	}
+
+	// Set the global logger
+	zap.ReplaceGlobals(logger)
+}

go.mod

@@ -1,6 +1,6 @@
 module gitlab-sync
 
-go 1.23.7
+go 1.24.2
 
 require (
 	github.com/spf13/cobra v1.9.1
@@ -13,6 +13,8 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/oauth2 v0.25.0 // indirect
 	golang.org/x/time v0.10.0 // indirect
 )

mirroring/get.go internal/mirroring/get.gomirroring/get.go renamed to internal/mirroring/get.go

@@ -2,45 +2,38 @@ package mirroring
 
 import (
 	"fmt"
-	"log"
 	"path/filepath"
 	"strings"
 	"sync"
 
-	"gitlab-sync/utils"
+	"gitlab-sync/internal/utils"
 
 	gitlab "gitlab.com/gitlab-org/api/client-go"
+	"go.uber.org/zap"
 )
 
 func (g *GitlabInstance) fetchProjects(projectFilters *map[string]bool, groupFilters *map[string]bool, mirrorMapping *utils.MirrorMapping, isSource bool) error {
 	sourceString := "source"
 	if !isSource {
 		sourceString = "destination"
 	}
-	utils.LogVerbosef("Fetching all projects from %s GitLab instance", sourceString)
+	zap.L().Sugar().Debugf("Fetching all projects from %s GitLab instance", sourceString)
 	projects, _, err := g.Gitlab.Projects.ListProjects(nil)
 	if err != nil {
 		return err
 	}
 
-	utils.LogVerbosef("Processing %d projects from %s GitLab instance", len(projects), sourceString)
+	zap.L().Sugar().Debugf("Processing %d projects from %s GitLab instance", len(projects), sourceString)
 
 	// Create a wait group to wait for all goroutines to finish
 	var wg sync.WaitGroup
 
-	// Create a channel to limit the number of concurrent goroutines
-	concurrencyLimit := 10
-	sem := make(chan struct{}, concurrencyLimit)
-
 	for _, project := range projects {
 		wg.Add(1)
 		// Acquire a token from the semaphore
-		sem <- struct{}{}
 
 		go func(project *gitlab.Project) {
 			defer wg.Done()
-			// Release the token back to the semaphore
-			defer func() { <-sem }()
 
 			// Check if the project matches the filters:
 			//   - either is in the projects map
@@ -57,13 +50,13 @@ func (g *GitlabInstance) fetchProjects(projectFilters *map[string]bool, groupFil
 							// Retrieve the corresponding group creation options from the mirror mapping
 							groupCreationOptions, ok := mirrorMapping.Groups[group]
 							if !ok {
-								log.Fatalf("Group %s not found in mirror mapping (internal error, please review script)", group)
+								zap.L().Sugar().Errorf("Group %s not found in mirror mapping (internal error, please review script)", group)
 							}
 
 							// Calculate the relative path between the project and the group
 							relativePath, err := filepath.Rel(group, project.PathWithNamespace)
 							if err != nil {
-								log.Fatalf("Failed to calculate relative path for project %s: %s", project.PathWithNamespace, err)
+								zap.L().Sugar().Errorf("Failed to calculate relative path for project %s: %s", project.PathWithNamespace, err)
 							}
 
 							// Add the project to the mirror mapping
@@ -85,7 +78,7 @@ func (g *GitlabInstance) fetchProjects(projectFilters *map[string]bool, groupFil
 
 	wg.Wait()
 
-	utils.LogVerbosef("Found %d projects to mirror in the %s GitLab instance", len(g.Projects), sourceString)
+	zap.L().Sugar().Debugf("Found %d projects to mirror in the %s GitLab instance", len(g.Projects), sourceString)
 
 	return nil
 }
@@ -95,30 +88,22 @@ func (g *GitlabInstance) fetchGroups(groupFilters *map[string]bool, mirrorMappin
 	if !isSource {
 		sourceString = "destination"
 	}
-	utils.LogVerbosef("Fetching all groups from %s GitLab instance", sourceString)
+	zap.L().Sugar().Debugf("Fetching all groups from %s GitLab instance", sourceString)
 	groups, _, err := g.Gitlab.Groups.ListGroups(nil)
 	if err != nil {
 		return err
 	}
 
-	utils.LogVerbosef("Processing %d groups from %s GitLab instance", len(groups), sourceString)
+	zap.L().Sugar().Debugf("Processing %d groups from %s GitLab instance", len(groups), sourceString)
 
 	// Create a wait group to wait for all goroutines to finish
 	var wg sync.WaitGroup
 
-	// Create a channel to limit the number of concurrent goroutines
-	concurrencyLimit := 10
-	sem := make(chan struct{}, concurrencyLimit)
-
 	for _, group := range groups {
 		wg.Add(1)
-		// Acquire a token from the semaphore
-		sem <- struct{}{}
 
 		go func(group *gitlab.Group) {
 			defer wg.Done()
-			// Release the token back to the semaphore
-			defer func() { <-sem }()
 
 			// Check if the group matches the filters:
 			//   - either is in the groups map
@@ -136,13 +121,13 @@ func (g *GitlabInstance) fetchGroups(groupFilters *map[string]bool, mirrorMappin
 							// Retrieve the corresponding group creation options from the mirror mapping
 							groupCreationOptions, ok := mirrorMapping.Groups[groupPath]
 							if !ok {
-								log.Fatalf("Group %s not found in mirror mapping (internal error, please review script)", groupPath)
+								zap.L().Sugar().Fatalf("Group %s not found in mirror mapping (internal error, please review script)", groupPath)
 							}
 
 							// Calculate the relative path between the group and the groupPath
 							relativePath, err := filepath.Rel(groupPath, group.FullPath)
 							if err != nil {
-								log.Fatalf("Failed to calculate relative path for group %s: %s", group.FullPath, err)
+								zap.L().Sugar().Fatalf("Failed to calculate relative path for group %s: %s", group.FullPath, err)
 							}
 
 							// Add the group to the mirror mapping
@@ -164,7 +149,7 @@ func (g *GitlabInstance) fetchGroups(groupFilters *map[string]bool, mirrorMappin
 
 	wg.Wait()
 
-	utils.LogVerbosef("Found %d matching groups in %s GitLab instance", len(g.Groups), sourceString)
+	zap.L().Sugar().Debugf("Found %d matching groups in %s GitLab instance", len(g.Groups), sourceString)
 
 	return nil
 }

mirroring/instance.go internal/mirroring/instance.gomirroring/instance.go renamed to internal/mirroring/instance.go

@@ -5,7 +5,7 @@ import (
 	"sync"
 	"time"
 
-	"gitlab-sync/utils"
+	"gitlab-sync/internal/utils"
 
 	gitlab "gitlab.com/gitlab-org/api/client-go"
 )

mirroring/instance_test.go internal/mirroring/instance_test.gomirroring/instance_test.go renamed to internal/mirroring/instance_test.go

@@ -6,7 +6,7 @@ import (
 	"path/filepath"
 	"sync"
 
-	"gitlab-sync/utils"
+	"gitlab-sync/internal/utils"
 )
 
 func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) error {

mirroring/main.go internal/mirroring/main.gomirroring/main.go renamed to internal/mirroring/main.go

@@ -4,7 +4,7 @@ import (
 	"reflect"
 	"testing"
 
-	"gitlab-sync/utils"
+	"gitlab-sync/internal/utils"
 )
 
 func TestProcessFilters(t *testing.T) {