feat: add more operationnal logging

add more meaningful data at info level
add more debug logs

Author: BoxBoxJason

internal/mirroring/get.go

@@ -13,6 +13,7 @@ import (
 // fetchAll retrieves all projects and groups from the GitLab instance
 // that match the filters and stores them in the instance cache.
 func (g *GitlabInstance) fetchAll(projectFilters map[string]struct{}, groupFilters map[string]struct{}, mirrorMapping *utils.MirrorMapping) error {
+	zap.L().Info("Fetching all projects and groups from GitLab instance", zap.String("role", g.Role), zap.String(INSTANCE_SIZE, g.InstanceSize), zap.Int("projects", len(projectFilters)), zap.Int("groups", len(groupFilters)))
 	wg := sync.WaitGroup{}
 	errCh := make(chan error, 2)
 	wg.Add(2)

internal/mirroring/get_group.go

@@ -15,6 +15,7 @@ import (
 //
 // The function is run in a goroutine for each group, and a wait group is used to wait for all goroutines to finish.
 func (g *GitlabInstance) fetchAndProcessGroups(groupFilters *map[string]struct{}, mirrorMapping *utils.MirrorMapping) error {
+	zap.L().Debug("Fetching and processing groups from GitLab instance", zap.String(ROLE, g.Role), zap.Int("groups", len(*groupFilters)))
 	if !g.isBig() {
 		return g.fetchAndProcessGroupsSmallInstance(groupFilters, mirrorMapping)
 	}

internal/mirroring/get_project.go

@@ -16,6 +16,7 @@ import (
 //
 // The function is run in a goroutine for each project, and a wait group is used to wait for all goroutines to finish.
 func (g *GitlabInstance) fetchAndProcessProjects(projectFilters *map[string]struct{}, groupFilters *map[string]struct{}, mirrorMapping *utils.MirrorMapping) error {
+	zap.L().Debug("Fetching and processing projects from GitLab instance", zap.String(ROLE, g.Role), zap.String(INSTANCE_SIZE, g.InstanceSize), zap.Int("projects", len(*projectFilters)), zap.Int("groups", len(*groupFilters)))
 	if !g.isBig() {
 		return g.fetchAndProcessProjectsSmallInstance(projectFilters, groupFilters, mirrorMapping)
 	}

internal/mirroring/main.go

@@ -16,6 +16,7 @@ import (
 // It then processes the filters for groups and projects, and finally creates the groups and projects in the destination GitLab instance.
 // If the dry run flag is set, it will only print the groups and projects that would be created or updated.
 func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) error {
+	zap.L().Info("Starting GitLab mirroring process", zap.String(ROLE_SOURCE, gitlabMirrorArgs.SourceGitlabURL), zap.String(ROLE_DESTINATION, gitlabMirrorArgs.DestinationGitlabURL))
 	sourceGitlabSize := INSTANCE_SIZE_SMALL
 	if gitlabMirrorArgs.SourceGitlabIsBig {
 		sourceGitlabSize = INSTANCE_SIZE_BIG
@@ -89,6 +90,7 @@ func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) error {
 // processFilters processes the filters for groups and projects.
 // It returns four maps: sourceProjectFilters, sourceGroupFilters, destinationProjectFilters, and destinationGroupFilters.
 func processFilters(filters *utils.MirrorMapping) (map[string]struct{}, map[string]struct{}, map[string]struct{}, map[string]struct{}) {
+	zap.L().Info("Checking mirror mapping filters")
 	sourceProjectFilters := make(map[string]struct{})
 	sourceGroupFilters := make(map[string]struct{})
 	destinationProjectFilters := make(map[string]struct{})

internal/mirroring/post.go

@@ -11,11 +11,15 @@ import (
 	"go.uber.org/zap"
 )
 
+// ============================================================ //
+//                 GROUP CREATION FUNCTIONS                     //
+// ============================================================ //
+
 // createGroups creates GitLab groups in the destination GitLab instance based on the mirror mapping.
 // It retrieves the source group path for each destination group and creates the group in the destination instance.
 // The function also handles the copying of group avatars from the source to the destination instance.
 func (destinationGitlab *GitlabInstance) createGroups(sourceGitlab *GitlabInstance, mirrorMapping *utils.MirrorMapping) error {
-	zap.L().Debug("Creating groups in GitLab Instance", zap.String(ROLE, ROLE_DESTINATION))
+	zap.L().Info("Creating groups in GitLab Instance", zap.String(ROLE, ROLE_DESTINATION))
 
 	// Reverse the mirror mapping to get the source group path for each destination group
 	reversedMirrorMap, destinationGroupPaths := sourceGitlab.reverseGroupMirrorMap(mirrorMapping)
@@ -71,10 +75,48 @@ func (destinationGitlab *GitlabInstance) createGroup(destinationGroupPath string
 	return destinationGroup, nil
 }
 
+// createGroupFromSource creates a GitLab group in the destination GitLab instance based on the source group.
+// It sets the group name, path, description, visibility, and default branch based on the source group.
+// The function also handles the setting of the parent ID for the group.
+// It returns the created group or an error if the creation fails.
+func (g *GitlabInstance) createGroupFromSource(sourceGroup *gitlab.Group, copyOptions *utils.MirroringOptions) (*gitlab.Group, error) {
+	groupCreationArgs := &gitlab.CreateGroupOptions{
+		Name:          &sourceGroup.Name,
+		Path:          &sourceGroup.Path,
+		Description:   &sourceGroup.Description,
+		Visibility:    &sourceGroup.Visibility,
+		DefaultBranch: &sourceGroup.DefaultBranch,
+	}
+
+	// Retrieve the parent namespace ID for the group
+	// This is used to set the parent ID for the group
+	zap.L().Debug("Retrieving group namespace ID", zap.String(ROLE, ROLE_DESTINATION), zap.String(ROLE_DESTINATION, copyOptions.DestinationPath))
+	parentGroupID, err := g.getParentNamespaceID(copyOptions.DestinationPath)
+	if err != nil {
+		return nil, err
+	} else if parentGroupID >= 0 {
+		groupCreationArgs.ParentID = &parentGroupID
+	}
+
+	// Create the group in the destination GitLab instance
+	zap.L().Debug("Creating group in GitLab Instance", zap.String(ROLE, ROLE_DESTINATION), zap.String(ROLE_DESTINATION, copyOptions.DestinationPath))
+	destinationGroup, _, err := g.Gitlab.Groups.CreateGroup(groupCreationArgs)
+	if err == nil {
+		zap.L().Info("Group created", zap.String("group", destinationGroup.WebURL))
+		g.addGroup(destinationGroup)
+	}
+
+	return destinationGroup, err
+}
+
+// ============================================================ //
+//                 PROJECT CREATION FUNCTIONS                    //
+// ============================================================ //
+
 // createProjects creates GitLab projects in the destination GitLab instance based on the mirror mapping.
 // It retrieves the source project path for each destination project and creates the project in the destination instance.
 func (destinationGitlab *GitlabInstance) createProjects(sourceGitlab *GitlabInstance, mirrorMapping *utils.MirrorMapping) error {
-	zap.L().Debug("Creating projects in GitLab Instance", zap.String(ROLE, ROLE_DESTINATION))
+	zap.L().Info("Creating projects in GitLab Instance", zap.String(ROLE, ROLE_DESTINATION))
 
 	// Create a wait group to wait for all goroutines to finish
 	var wg sync.WaitGroup
@@ -171,46 +213,16 @@ func (g *GitlabInstance) createProjectFromSource(sourceProject *gitlab.Project,
 	zap.L().Debug("Creating project in GitLab Instance", zap.String(ROLE, ROLE_DESTINATION), zap.String(ROLE_DESTINATION, copyOptions.DestinationPath))
 	destinationProject, _, err := g.Gitlab.Projects.CreateProject(projectCreationArgs)
 	if err == nil {
-		zap.L().Debug("Project created", zap.String("project", destinationProject.HTTPURLToRepo))
+		zap.L().Info("Project created", zap.String("project", destinationProject.HTTPURLToRepo))
 		g.addProject(destinationProject)
 	}
 
 	return destinationProject, nil
 }
 
-// createGroupFromSource creates a GitLab group in the destination GitLab instance based on the source group.
-// It sets the group name, path, description, visibility, and default branch based on the source group.
-// The function also handles the setting of the parent ID for the group.
-// It returns the created group or an error if the creation fails.
-func (g *GitlabInstance) createGroupFromSource(sourceGroup *gitlab.Group, copyOptions *utils.MirroringOptions) (*gitlab.Group, error) {
-	groupCreationArgs := &gitlab.CreateGroupOptions{
-		Name:          &sourceGroup.Name,
-		Path:          &sourceGroup.Path,
-		Description:   &sourceGroup.Description,
-		Visibility:    &sourceGroup.Visibility,
-		DefaultBranch: &sourceGroup.DefaultBranch,
-	}
-
-	// Retrieve the parent namespace ID for the group
-	// This is used to set the parent ID for the group
-	zap.L().Debug("Retrieving group namespace ID", zap.String(ROLE, ROLE_DESTINATION), zap.String(ROLE_DESTINATION, copyOptions.DestinationPath))
-	parentGroupID, err := g.getParentNamespaceID(copyOptions.DestinationPath)
-	if err != nil {
-		return nil, err
-	} else if parentGroupID >= 0 {
-		groupCreationArgs.ParentID = &parentGroupID
-	}
-
-	// Create the group in the destination GitLab instance
-	zap.L().Debug("Creating group in GitLab Instance", zap.String(ROLE, ROLE_DESTINATION), zap.String(ROLE_DESTINATION, copyOptions.DestinationPath))
-	destinationGroup, _, err := g.Gitlab.Groups.CreateGroup(groupCreationArgs)
-	if err == nil {
-		zap.L().Debug("Group created", zap.String("group", destinationGroup.WebURL))
-		g.addGroup(destinationGroup)
-	}
-
-	return destinationGroup, err
-}
+// ============================================================ //
+//               RELEASES CREATION FUNCTIONS                    //
+// ============================================================ //
 
 // mirrorReleases mirrors releases from the source project to the destination project.
 // It fetches existing releases from the destination project and creates new releases for those that do not exist.
@@ -279,6 +291,10 @@ func (destinationGitlab *GitlabInstance) mirrorReleases(sourceGitlab *GitlabInst
 	return utils.MergeErrors(errorChan, 2)
 }
 
+// ============================================================ //
+//                 CI/CD CATALOG FUNCTIONS                      //
+// ============================================================ //
+
 // addProjectToCICDCatalog adds a project to the CI/CD catalog in the destination GitLab instance.
 // It uses a GraphQL mutation to create the catalog resource for the project.
 func (g *GitlabInstance) addProjectToCICDCatalog(project *gitlab.Project) error {