fix: logging upgrade

logging output is now done in a proper list instead of an indented string
fixed MergeErrors function to support new errors aggregation required

Author: BoxBoxJason

cmd/main.go

@@ -64,16 +64,16 @@ func main() {
 			zap.L().Debug("Mirror Mapping file resolved path: " + filepath.Clean(mirrorMappingPath))
 
 			zap.L().Debug("Parsing mirror mapping file")
-			mapping, err := utils.OpenMirrorMapping(mirrorMappingPath)
-			if err != nil {
-				zap.L().Fatal("Error opening mirror mapping file", zap.Error(err))
+			mapping, mappingErrors := utils.OpenMirrorMapping(mirrorMappingPath)
+			if mappingErrors != nil {
+				zap.L().Fatal("Error opening mirror mapping file", zap.Array("errors", utils.ErrorArray(mappingErrors)))
 			}
 			zap.L().Debug("Mirror mapping file parsed successfully")
 			args.MirrorMapping = mapping
 
-			err = mirroring.MirrorGitlabs(&args)
-			if err != nil {
-				zap.L().Error("Error during mirroring process: " + err.Error())
+			mirroringErrors := mirroring.MirrorGitlabs(&args)
+			if mirroringErrors != nil {
+				zap.L().Error("Error during mirroring process", zap.Array("errors", utils.ErrorArray(mirroringErrors)))
 			}
 			zap.L().Info("Mirroring completed")
 		},

internal/mirroring/get.go

@@ -12,10 +12,10 @@ import (
 
 // fetchAll retrieves all projects and groups from the GitLab instance
 // that match the filters and stores them in the instance cache.
-func (g *GitlabInstance) fetchAll(projectFilters map[string]struct{}, groupFilters map[string]struct{}, mirrorMapping *utils.MirrorMapping) error {
-	zap.L().Info("Fetching all projects and groups from GitLab instance", zap.String("role", g.Role), zap.String(INSTANCE_SIZE, g.InstanceSize), zap.Int("projects", len(projectFilters)), zap.Int("groups", len(groupFilters)))
+func (g *GitlabInstance) fetchAll(projectFilters map[string]struct{}, groupFilters map[string]struct{}, mirrorMapping *utils.MirrorMapping) []error {
+	zap.L().Info("Fetching all projects and groups from GitLab instance", zap.String(ROLE, g.Role), zap.String(INSTANCE_SIZE, g.InstanceSize), zap.Int("projects", len(projectFilters)), zap.Int("groups", len(groupFilters)))
 	wg := sync.WaitGroup{}
-	errCh := make(chan error, 2)
+	errCh := make(chan []error, 2)
 	wg.Add(2)
 	go func() {
 		defer wg.Done()
@@ -32,7 +32,7 @@ func (g *GitlabInstance) fetchAll(projectFilters map[string]struct{}, groupFilte
 	wg.Wait()
 	close(errCh)
 
-	return utils.MergeErrors(errCh, 2)
+	return utils.MergeErrors(errCh)
 }
 
 // getParentNamespaceID retrieves the parent namespace ID for a given project or group path.

internal/mirroring/get_group.go

@@ -14,10 +14,10 @@ import (
 // It also updates the mirror mapping with the corresponding group creation options.
 //
 // The function is run in a goroutine for each group, and a wait group is used to wait for all goroutines to finish.
-func (g *GitlabInstance) fetchAndProcessGroups(groupFilters *map[string]struct{}, mirrorMapping *utils.MirrorMapping) error {
+func (g *GitlabInstance) fetchAndProcessGroups(groupFilters *map[string]struct{}, mirrorMapping *utils.MirrorMapping) []error {
 	zap.L().Debug("Fetching and processing groups from GitLab instance", zap.String(ROLE, g.Role), zap.Int("groups", len(*groupFilters)))
 	if !g.isBig() {
-		return g.fetchAndProcessGroupsSmallInstance(groupFilters, mirrorMapping)
+		return []error{g.fetchAndProcessGroupsSmallInstance(groupFilters, mirrorMapping)}
 	}
 	return g.fetchAndProcessGroupsLargeInstance(groupFilters, mirrorMapping)
 }
@@ -137,7 +137,7 @@ func (g *GitlabInstance) processGroupsSmallInstance(allGroups []*gitlab.Group, g
 // fetchAndProcessGroupsLargeInstance retrieves all groups that match the filters from the GitLab instance and stores them in the instance cache.
 // It also updates the mirror mapping with the corresponding group creation options.
 // It uses goroutines to fetch groups and their projects concurrently.
-func (g *GitlabInstance) fetchAndProcessGroupsLargeInstance(groupFilters *map[string]struct{}, mirrorMapping *utils.MirrorMapping) error {
+func (g *GitlabInstance) fetchAndProcessGroupsLargeInstance(groupFilters *map[string]struct{}, mirrorMapping *utils.MirrorMapping) []error {
 	errChan := make(chan error)
 	var wg sync.WaitGroup
 	wg.Add(len(*groupFilters))
@@ -166,7 +166,7 @@ func (g *GitlabInstance) fetchAndProcessGroupsLargeInstance(groupFilters *map[st
 	// Wait for all goroutines to finish
 	wg.Wait()
 	close(errChan)
-	return utils.MergeErrors(errs, 2)
+	return utils.MergeErrors(errs)
 }
 
 // fetchAndProcessGroupRecursive fetches a group and its projects recursively

internal/mirroring/get_project.go

@@ -15,7 +15,7 @@ import (
 // It also updates the mirror mapping with the corresponding group creation options.
 //
 // The function is run in a goroutine for each project, and a wait group is used to wait for all goroutines to finish.
-func (g *GitlabInstance) fetchAndProcessProjects(projectFilters *map[string]struct{}, groupFilters *map[string]struct{}, mirrorMapping *utils.MirrorMapping) error {
+func (g *GitlabInstance) fetchAndProcessProjects(projectFilters *map[string]struct{}, groupFilters *map[string]struct{}, mirrorMapping *utils.MirrorMapping) []error {
 	zap.L().Debug("Fetching and processing projects from GitLab instance", zap.String(ROLE, g.Role), zap.String(INSTANCE_SIZE, g.InstanceSize), zap.Int("projects", len(*projectFilters)), zap.Int("groups", len(*groupFilters)))
 	if !g.isBig() {
 		return g.fetchAndProcessProjectsSmallInstance(projectFilters, groupFilters, mirrorMapping)
@@ -63,11 +63,11 @@ func (g *GitlabInstance) storeProject(project *gitlab.Project, parentGroupPath s
 
 // fetchAndProcessProjectsSmallInstance retrieves all projects from the small GitLab instance
 // and processes them to store in the instance cache.
-func (g *GitlabInstance) fetchAndProcessProjectsSmallInstance(projectFilters *map[string]struct{}, groupFilters *map[string]struct{}, mirrorMapping *utils.MirrorMapping) error {
+func (g *GitlabInstance) fetchAndProcessProjectsSmallInstance(projectFilters *map[string]struct{}, groupFilters *map[string]struct{}, mirrorMapping *utils.MirrorMapping) []error {
 	allProjects, err := g.fetchAllProjectsSmallInstance()
 	if err != nil {
 		if len(allProjects) == 0 {
-			return err
+			return []error{err}
 		} else {
 			zap.L().Warn("Failed to fetch all projects from GitLab instance", zap.String(ROLE, g.Role), zap.Error(err))
 		}
@@ -146,7 +146,7 @@ func (g *GitlabInstance) processProjectsSmallInstance(allProjects []*gitlab.Proj
 //
 // It uses goroutines to fetch each project in parallel and a wait group to wait for all goroutines to finish.
 // It returns an error if any of the goroutines fail.
-func (g *GitlabInstance) fetchAndProcessProjectsBigInstance(projectFilters *map[string]struct{}, mirrorMapping *utils.MirrorMapping) error {
+func (g *GitlabInstance) fetchAndProcessProjectsBigInstance(projectFilters *map[string]struct{}, mirrorMapping *utils.MirrorMapping) []error {
 	// Fetch each project in parallel
 	var wg sync.WaitGroup
 	projectsChan := make(chan *gitlab.Project, len(*projectFilters))
@@ -170,7 +170,7 @@ func (g *GitlabInstance) fetchAndProcessProjectsBigInstance(projectFilters *map[
 	for project := range projectsChan {
 		g.storeProject(project, project.PathWithNamespace, mirrorMapping)
 	}
-	return utils.MergeErrors(errCh, 2)
+	return utils.MergeErrors(errCh)
 }
 
 // fetchAndProcessGroupProjects retrieves all projects from the group and processes them to store in the instance cache.

internal/mirroring/main.go

@@ -15,7 +15,7 @@ import (
 // It creates two GitLab instances (source and destination) and fetches the groups and projects from both instances.
 // It then processes the filters for groups and projects, and finally creates the groups and projects in the destination GitLab instance.
 // If the dry run flag is set, it will only print the groups and projects that would be created or updated.
-func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) error {
+func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) []error {
 	zap.L().Info("Starting GitLab mirroring process", zap.String(ROLE_SOURCE, gitlabMirrorArgs.SourceGitlabURL), zap.String(ROLE_DESTINATION, gitlabMirrorArgs.DestinationGitlabURL))
 	sourceGitlabSize := INSTANCE_SIZE_SMALL
 	if gitlabMirrorArgs.SourceGitlabIsBig {
@@ -29,7 +29,7 @@ func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) error {
 		InstanceSize: sourceGitlabSize,
 	})
 	if err != nil {
-		return err
+		return []error{err}
 	}
 
 	destinationGitlabSize := INSTANCE_SIZE_SMALL
@@ -44,26 +44,22 @@ func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) error {
 		InstanceSize: destinationGitlabSize,
 	})
 	if err != nil {
-		return err
+		return []error{err}
 	}
 
 	sourceProjectFilters, sourceGroupFilters, destinationProjectFilters, destinationGroupFilters := processFilters(gitlabMirrorArgs.MirrorMapping)
 
 	wg := sync.WaitGroup{}
-	errCh := make(chan error, 4)
+	errCh := make(chan []error, 4)
 	wg.Add(2)
 
 	go func() {
 		defer wg.Done()
-		if err := sourceGitlabInstance.fetchAll(sourceProjectFilters, sourceGroupFilters, gitlabMirrorArgs.MirrorMapping); err != nil {
-			errCh <- err
-		}
+		errCh <- sourceGitlabInstance.fetchAll(sourceProjectFilters, sourceGroupFilters, gitlabMirrorArgs.MirrorMapping)
 	}()
 	go func() {
 		defer wg.Done()
-		if err := destinationGitlabInstance.fetchAll(destinationProjectFilters, destinationGroupFilters, gitlabMirrorArgs.MirrorMapping); err != nil {
-			errCh <- err
-		}
+		errCh <- destinationGitlabInstance.fetchAll(destinationProjectFilters, destinationGroupFilters, gitlabMirrorArgs.MirrorMapping)
 	}()
 
 	wg.Wait()
@@ -74,17 +70,12 @@ func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) error {
 		return nil
 	}
 
-	// Create groups and projects in the destination GitLab instance
-	err = destinationGitlabInstance.createGroups(sourceGitlabInstance, gitlabMirrorArgs.MirrorMapping)
-	if err != nil {
-		errCh <- err
-	}
-	err = destinationGitlabInstance.createProjects(sourceGitlabInstance, gitlabMirrorArgs.MirrorMapping)
-	if err != nil {
-		errCh <- err
-	}
+	// Create groups and projects in the destination GitLab instance (Groups must be created before projects)
+	errCh <- destinationGitlabInstance.createGroups(sourceGitlabInstance, gitlabMirrorArgs.MirrorMapping)
+	errCh <- destinationGitlabInstance.createProjects(sourceGitlabInstance, gitlabMirrorArgs.MirrorMapping)
+
 	close(errCh)
-	return utils.MergeErrors(errCh, 2)
+	return utils.MergeErrors(errCh)
 }
 
 // processFilters processes the filters for groups and projects.
@@ -137,14 +128,14 @@ func DryRun(sourceGitlabInstance *GitlabInstance, gitlabMirrorArgs *utils.Parser
 	zap.L().Info("Dry run mode enabled, will not create groups or projects")
 	zap.L().Info("Groups that will be created (or updated if they already exist):")
 	for sourceGroupPath, copyOptions := range gitlabMirrorArgs.MirrorMapping.Groups {
-		if sourceGitlabInstance.Groups[sourceGroupPath] != nil {
-			fmt.Printf("  - %s (source gitlab) -> %s (destination gitlab)\n", sourceGroupPath, copyOptions.DestinationPath)
+		if sourceGroup, ok := sourceGitlabInstance.Groups[sourceGroupPath]; ok {
+			fmt.Printf("  - %s (source gitlab) -> %s (destination gitlab)\n", sourceGroup.WebURL, copyOptions.DestinationPath)
 		}
 	}
 	zap.L().Info("Projects that will be created (or updated if they already exist):")
 	for sourceProjectPath, copyOptions := range gitlabMirrorArgs.MirrorMapping.Projects {
-		if sourceGitlabInstance.Projects[sourceProjectPath] != nil {
-			fmt.Printf("  - %s (source gitlab) -> %s (destination gitlab)\n", sourceProjectPath, copyOptions.DestinationPath)
+		if sourceProject, ok := sourceGitlabInstance.Projects[sourceProjectPath]; ok {
+			fmt.Printf("  - %s (source gitlab) -> %s (destination gitlab)\n", sourceProject.WebURL, copyOptions.DestinationPath)
 		}
 	}
 }

internal/mirroring/post.go

@@ -18,7 +18,7 @@ import (
 // createGroups creates GitLab groups in the destination GitLab instance based on the mirror mapping.
 // It retrieves the source group path for each destination group and creates the group in the destination instance.
 // The function also handles the copying of group avatars from the source to the destination instance.
-func (destinationGitlab *GitlabInstance) createGroups(sourceGitlab *GitlabInstance, mirrorMapping *utils.MirrorMapping) error {
+func (destinationGitlab *GitlabInstance) createGroups(sourceGitlab *GitlabInstance, mirrorMapping *utils.MirrorMapping) []error {
 	zap.L().Info("Creating groups in GitLab Instance", zap.String(ROLE, ROLE_DESTINATION))
 
 	// Reverse the mirror mapping to get the source group path for each destination group
@@ -33,7 +33,7 @@ func (destinationGitlab *GitlabInstance) createGroups(sourceGitlab *GitlabInstan
 		}
 	}
 	close(errorChan)
-	return utils.MergeErrors(errorChan, 2)
+	return utils.MergeErrors(errorChan)
 }
 
 // createGroup creates a GitLab group in the destination GitLab instance based on the source group and mirror mapping.
@@ -115,7 +115,7 @@ func (g *GitlabInstance) createGroupFromSource(sourceGroup *gitlab.Group, copyOp
 
 // createProjects creates GitLab projects in the destination GitLab instance based on the mirror mapping.
 // It retrieves the source project path for each destination project and creates the project in the destination instance.
-func (destinationGitlab *GitlabInstance) createProjects(sourceGitlab *GitlabInstance, mirrorMapping *utils.MirrorMapping) error {
+func (destinationGitlab *GitlabInstance) createProjects(sourceGitlab *GitlabInstance, mirrorMapping *utils.MirrorMapping) []error {
 	zap.L().Info("Creating projects in GitLab Instance", zap.String(ROLE, ROLE_DESTINATION))
 
 	// Create a wait group to wait for all goroutines to finish
@@ -147,39 +147,36 @@ func (destinationGitlab *GitlabInstance) createProjects(sourceGitlab *GitlabInst
 	wg.Wait()
 	close(errorChan)
 
-	return utils.MergeErrors(errorChan, 2)
+	return utils.MergeErrors(errorChan)
 }
 
 // createProject creates a GitLab project in the destination GitLab instance based on the source project and mirror mapping.
 // It checks if the project already exists in the destination instance and creates it if not.
 // The function also handles the copying of project avatars from the source to the destination instance.
-func (destinationGitlab *GitlabInstance) createProject(sourceProjectPath string, projectCreationOptions *utils.MirroringOptions, sourceGitlab *GitlabInstance) (*gitlab.Project, error) {
+func (destinationGitlab *GitlabInstance) createProject(sourceProjectPath string, projectCreationOptions *utils.MirroringOptions, sourceGitlab *GitlabInstance) (*gitlab.Project, []error) {
 	destinationProjectPath := projectCreationOptions.DestinationPath
 	// Check if the project already exists
 	destinationProject := destinationGitlab.getProject(destinationProjectPath)
 	var err error
 	sourceProject := sourceGitlab.Projects[sourceProjectPath]
 	if sourceProject == nil {
-		return nil, fmt.Errorf("project %s not found in source GitLab instance (internal error, please review script)", sourceProjectPath)
+		return nil, []error{fmt.Errorf("project %s not found in source GitLab instance (internal error, please review script)", sourceProjectPath)}
 	}
 
 	// Check if the project already exists in the destination GitLab instance
 	// If it does not exist, create it
 	if destinationProject == nil {
 		destinationProject, err = destinationGitlab.createProjectFromSource(sourceProject, projectCreationOptions)
 		if err != nil {
-			return nil, fmt.Errorf("failed to create project %s in destination GitLab instance: %s", destinationProjectPath, err)
+			return nil, []error{fmt.Errorf("failed to create project %s in destination GitLab instance: %s", destinationProjectPath, err)}
 		}
 	}
 
 	// If the project already exists, update it with the source project details
-	err = destinationGitlab.updateProjectFromSource(sourceGitlab, sourceProject, destinationProject, projectCreationOptions)
-	if err != nil {
-		return destinationProject, fmt.Errorf("failed to update project %s in destination GitLab instance: %s", destinationProjectPath, err)
-	}
+	mergedError := destinationGitlab.updateProjectFromSource(sourceGitlab, sourceProject, destinationProject, projectCreationOptions)
 
 	zap.L().Info("Completed project mirroring", zap.String(ROLE_SOURCE, sourceProjectPath), zap.String(ROLE_DESTINATION, destinationProjectPath))
-	return destinationProject, nil
+	return destinationProject, mergedError
 }
 
 // createProjectFromSource creates a GitLab project in the destination GitLab instance based on the source project.
@@ -228,12 +225,12 @@ func (g *GitlabInstance) createProjectFromSource(sourceProject *gitlab.Project,
 // It fetches existing releases from the destination project and creates new releases for those that do not exist.
 // The function handles the API calls concurrently using goroutines and a wait group.
 // It returns an error if any of the API calls fail.
-func (destinationGitlab *GitlabInstance) mirrorReleases(sourceGitlab *GitlabInstance, sourceProject *gitlab.Project, destinationProject *gitlab.Project) error {
+func (destinationGitlab *GitlabInstance) mirrorReleases(sourceGitlab *GitlabInstance, sourceProject *gitlab.Project, destinationProject *gitlab.Project) []error {
 	zap.L().Debug("Starting releases mirroring", zap.String(ROLE_SOURCE, sourceProject.HTTPURLToRepo), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo))
 	// Fetch existing releases from the destination project
 	existingReleases, _, err := destinationGitlab.Gitlab.Releases.ListReleases(destinationProject.ID, &gitlab.ListReleasesOptions{})
 	if err != nil {
-		return fmt.Errorf("failed to fetch existing releases for destination project %s: %s", destinationProject.HTTPURLToRepo, err)
+		return []error{fmt.Errorf("failed to fetch existing releases for destination project %s: %s", destinationProject.HTTPURLToRepo, err)}
 	}
 
 	// Create a map of existing release tags for quick lookup
@@ -247,7 +244,7 @@ func (destinationGitlab *GitlabInstance) mirrorReleases(sourceGitlab *GitlabInst
 	// Fetch releases from the source project
 	sourceReleases, _, err := sourceGitlab.Gitlab.Releases.ListReleases(sourceProject.ID, &gitlab.ListReleasesOptions{})
 	if err != nil {
-		return fmt.Errorf("failed to fetch releases for source project %s: %s", sourceProject.HTTPURLToRepo, err)
+		return []error{fmt.Errorf("failed to fetch releases for source project %s: %s", sourceProject.HTTPURLToRepo, err)}
 	}
 
 	// Create a wait group and an error channel for handling API calls concurrently
@@ -288,7 +285,7 @@ func (destinationGitlab *GitlabInstance) mirrorReleases(sourceGitlab *GitlabInst
 	close(errorChan)
 
 	zap.L().Info("Releases mirroring completed", zap.String(ROLE_SOURCE, sourceProject.HTTPURLToRepo), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo))
-	return utils.MergeErrors(errorChan, 2)
+	return utils.MergeErrors(errorChan)
 }
 
 // ============================================================ //

internal/mirroring/put.go

@@ -92,7 +92,7 @@ func (sourceGitlabInstance *GitlabInstance) copyGroupAvatar(destinationGitlabIns
 // It enables the project mirror pull, copies the project avatar, and optionally adds the project to the CI/CD catalog.
 // It also mirrors releases if the option is set.
 // The function uses goroutines to perform these tasks concurrently and waits for all of them to finish.
-func (destinationGitlabInstance *GitlabInstance) updateProjectFromSource(sourceGitlabInstance *GitlabInstance, sourceProject *gitlab.Project, destinationProject *gitlab.Project, copyOptions *utils.MirroringOptions) error {
+func (destinationGitlabInstance *GitlabInstance) updateProjectFromSource(sourceGitlabInstance *GitlabInstance, sourceProject *gitlab.Project, destinationProject *gitlab.Project, copyOptions *utils.MirroringOptions) []error {
 	wg := sync.WaitGroup{}
 	maxErrors := 2
 	if copyOptions.CI_CD_Catalog {
@@ -145,5 +145,5 @@ func (destinationGitlabInstance *GitlabInstance) updateProjectFromSource(sourceG
 
 	wg.Wait()
 	close(errorChan)
-	return utils.MergeErrors(errorChan, 4)
+	return utils.MergeErrors(errorChan)
 }