diff --git a/README.md b/README.md index 915c6a0..0395ade 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,9 @@ [![Coverage](https://codecov.io/gh/boxboxjason/gitlab-sync/branch/main/graph/badge.svg)](https://codecov.io/gh/boxboxjason/gitlab-sync) ![Go Version](https://img.shields.io/github/go-mod/go-version/boxboxjason/gitlab-sync) [![Latest Release](https://img.shields.io/github/v/release/boxboxjason/gitlab-sync)](https://github.com/boxboxjason/gitlab-sync/releases) -[![Release Workflow](https://github.com/boxboxjason/gitlab-sync/actions/workflows/release.yml/badge.svg)](https://github.com/boxboxjason/gitlab-sync/actions/workflows/release.yml) ![Last Commit](https://img.shields.io/github/last-commit/boxboxjason/gitlab-sync) [![Tests](https://github.com/boxboxjason/gitlab-sync/actions/workflows/go.yml/badge.svg)](https://github.com/boxboxjason/gitlab-sync/actions/workflows/go.yml) -![Stars](https://img.shields.io/github/stars/boxboxjason/gitlab-sync) ![Contributors](https://img.shields.io/github/contributors/boxboxjason/gitlab-sync?style=social) -![Issues](https://img.shields.io/github/issues/boxboxjason/gitlab-sync) -![Pull Requests](https://img.shields.io/github/issues-pr/boxboxjason/gitlab-sync) (golang) CLI tool to synchronize GitLab projects and groups between two GitLab instances. It is designed to be used in a CI/CD pipeline to automate the process of keeping two GitLab instances in sync. @@ -74,6 +70,8 @@ If mandatory arguments are not provided, the program will prompt for them. | `--source-token` | `SOURCE_GITLAB_TOKEN` | No | Access token for the source GitLab instance | | `--source-big` | `SOURCE_GITLAB_BIG` | No | Specify if the source GitLab instance is a big instance (default: false) | | `--destination-url` | `DESTINATION_GITLAB_URL` | Yes | URL of the destination GitLab instance | +| `--destination-force-freemium` or `-f` | N/A | No | Force the destination GitLab to be treated as a non-premium instance (default: false) | +| `--destination-force-premium` or `-p` | N/A | No | Force the destination GitLab to be treated as a premium instance (default: false) | | `--destination-token` | `DESTINATION_GITLAB_TOKEN` | Yes | Access token for the destination GitLab instance | | `--destination-big` | `DESTINATION_GITLAB_BIG` | No | Specify if the destination GitLab instance is a big instance (default: false) | | `--mirror-mapping` | `MIRROR_MAPPING` | Yes | Path to a JSON file containing the mirror mapping | @@ -101,7 +99,7 @@ Allowed options are: |--------|-------------| | `destination_path` | The path to the project / group on the destination GitLab instance. | | `ci_cd_catalog` | Whether to add the project to the CI/CD catalog. | -| `issues` | Whether to copy issues from the source project to the destination project. | +| `mirror_issues` | Whether to copy issues from the source project to the destination project. | | `visibility` | The visibility level of the project on the destination GitLab instance. Can be `public`, `internal`, or `private`. | | `mirror_trigger_builds` | Whether to trigger builds on the destination project when a push is made to the source project. | | `mirror_releases` | Whether to mirror releases from the source project to the destination project. | @@ -116,7 +114,7 @@ Also, the destination namespace must exist on the destination GitLab instance. I "existingGroup1/project1" : { "destination_path": "existingGroup64/project1", "ci_cd_catalog": true, - "issues": false, + "mirror_issues": false, "visibility": "public", "mirror_trigger_builds": false, "mirror_releases": false @@ -126,7 +124,7 @@ Also, the destination namespace must exist on the destination GitLab instance. I "existingGroup152" : { "destination_path": "existingGroup64/existingGroup152", "ci_cd_catalog": true, - "issues": false, + "mirror_issues": false, "visibility": "public", "mirror_trigger_builds": false, "mirror_releases": false diff --git a/internal/mirroring/get.go b/internal/mirroring/get.go index 0ed4189..eb66091 100644 --- a/internal/mirroring/get.go +++ b/internal/mirroring/get.go @@ -5,7 +5,6 @@ import ( "gitlab-sync/internal/utils" "gitlab-sync/pkg/helpers" "path/filepath" - "strings" "sync" "github.com/Masterminds/semver/v3" @@ -62,30 +61,6 @@ func (g *GitlabInstance) getParentNamespaceID(projectOrGroupPath string) (int, e return parentGroupID, err } -// checkPathMatchesFilters checks if the resources matches the filters -// - either is in the projects map -// - or path starts with any of the groups in the groups map -// -// In the case of a match with a group, it returns the group path -func checkPathMatchesFilters(resourcePath string, projectFilters *map[string]struct{}, groupFilters *map[string]struct{}) (string, bool) { - zap.L().Debug("Checking if path matches filters", zap.String("path", resourcePath)) - if projectFilters != nil { - if _, ok := (*projectFilters)[resourcePath]; ok { - zap.L().Debug("Resource path matches project filter", zap.String("project", resourcePath)) - return "", true - } - } - if groupFilters != nil { - for groupPath := range *groupFilters { - if strings.HasPrefix(resourcePath, groupPath) { - zap.L().Debug("Resource path matches group filter", zap.String("resource", resourcePath), zap.String("group", groupPath)) - return groupPath, true - } - } - } - return "", false -} - // IsVersionGreaterThanThreshold checks if the GitLab instance version is below the defined threshold. // It retrieves the metadata from the GitLab instance and compares the version // with the INSTANCE_SEMVER_THRESHOLD. diff --git a/internal/mirroring/get_group.go b/internal/mirroring/get_group.go index 24aed46..d6092cd 100644 --- a/internal/mirroring/get_group.go +++ b/internal/mirroring/get_group.go @@ -50,7 +50,7 @@ func (g *GitlabInstance) storeGroup(group *gitlab.Group, parentGroupPath string, mirrorMapping.AddGroup(group.FullPath, &utils.MirroringOptions{ DestinationPath: filepath.Join(groupCreationOptions.DestinationPath, relativePath), CI_CD_Catalog: groupCreationOptions.CI_CD_Catalog, - Issues: groupCreationOptions.Issues, + MirrorIssues: groupCreationOptions.MirrorIssues, MirrorTriggerBuilds: groupCreationOptions.MirrorTriggerBuilds, Visibility: groupCreationOptions.Visibility, MirrorReleases: groupCreationOptions.MirrorReleases, @@ -121,7 +121,7 @@ func (g *GitlabInstance) processGroupsSmallInstance(allGroups []*gitlab.Group, g go func(group *gitlab.Group) { defer wg.Done() - groupPath, matches := checkPathMatchesFilters(group.FullPath, nil, groupFilters) + groupPath, matches := helpers.MatchPathAgainstFilters(group.FullPath, nil, groupFilters) if matches { g.storeGroup(group, groupPath, mirrorMapping) } diff --git a/internal/mirroring/get_project.go b/internal/mirroring/get_project.go index 58d9dca..d2526b5 100644 --- a/internal/mirroring/get_project.go +++ b/internal/mirroring/get_project.go @@ -50,7 +50,7 @@ func (g *GitlabInstance) storeProject(project *gitlab.Project, parentGroupPath s mirrorMapping.AddProject(project.PathWithNamespace, &utils.MirroringOptions{ DestinationPath: filepath.Join(groupCreationOptions.DestinationPath, relativePath), CI_CD_Catalog: groupCreationOptions.CI_CD_Catalog, - Issues: groupCreationOptions.Issues, + MirrorIssues: groupCreationOptions.MirrorIssues, MirrorTriggerBuilds: groupCreationOptions.MirrorTriggerBuilds, Visibility: groupCreationOptions.Visibility, MirrorReleases: groupCreationOptions.MirrorReleases, @@ -127,7 +127,7 @@ func (g *GitlabInstance) processProjectsSmallInstance(allProjects []*gitlab.Proj go func(project *gitlab.Project) { defer wg.Done() - group, matches := checkPathMatchesFilters(project.PathWithNamespace, projectFilters, groupFilters) + group, matches := helpers.MatchPathAgainstFilters(project.PathWithNamespace, projectFilters, groupFilters) if matches { g.storeProject(project, group, mirrorMapping) } diff --git a/internal/mirroring/get_test.go b/internal/mirroring/get_test.go index 4d3a63b..d2ff301 100644 --- a/internal/mirroring/get_test.go +++ b/internal/mirroring/get_test.go @@ -2,6 +2,7 @@ package mirroring import ( "gitlab-sync/internal/utils" + "gitlab-sync/pkg/helpers" "net/http" "testing" ) @@ -61,7 +62,7 @@ func TestCheckPathMatchesFilters(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() // Call the function with the test case parameters - _, got := checkPathMatchesFilters(test.path, &test.projectFilters, &test.groupFilters) + _, got := helpers.MatchPathAgainstFilters(test.path, &test.projectFilters, &test.groupFilters) // Check if the result matches the expected value if got != test.expected { diff --git a/internal/mirroring/helper_test.go b/internal/mirroring/helper_test.go index 0c0d5c8..ef1d5a5 100644 --- a/internal/mirroring/helper_test.go +++ b/internal/mirroring/helper_test.go @@ -263,6 +263,112 @@ var ( "links": [] } }`} + + issue_string = "issue" + + TEST_ISSUE = &gitlab.Issue{ + IID: 1, + Title: "Test Issue", + Description: "This is a test issue", + Labels: []string{"bug", "urgent"}, + Confidential: false, + Weight: 3, + IssueType: &issue_string, + } + + // TEST_ISSUE_STRING is the string representation of TEST_ISSUE. + TEST_ISSUE_STRING = `{ + "project_id" : 1, + "milestone" : { + "due_date" : null, + "project_id" : 4, + "state" : "closed", + "description" : "Rerum est voluptatem provident consequuntur molestias similique ipsum dolor.", + "iid" : 3, + "id" : 11, + "title" : "v3.0", + "created_at" : "2016-01-04T15:31:39.788Z", + "updated_at" : "2016-01-04T15:31:39.788Z" + }, + "author" : { + "state" : "active", + "web_url" : "https://gitlab.example.com/root", + "avatar_url" : null, + "username" : "root", + "id" : 1, + "name" : "Administrator" + }, + "description" : "Omnis vero earum sunt corporis dolor et placeat.", + "state" : "closed", + "iid" : 1, + "assignees" : [{ + "avatar_url" : null, + "web_url" : "https://gitlab.example.com/lennie", + "state" : "active", + "username" : "lennie", + "id" : 9, + "name" : "Dr. Luella Kovacek" + }], + "assignee" : { + "avatar_url" : null, + "web_url" : "https://gitlab.example.com/lennie", + "state" : "active", + "username" : "lennie", + "id" : 9, + "name" : "Dr. Luella Kovacek" + }, + "type" : "ISSUE", + "labels" : ["foo", "bar"], + "upvotes": 4, + "downvotes": 0, + "merge_requests_count": 0, + "id" : 41, + "title" : "Ut commodi ullam eos dolores perferendis nihil sunt.", + "updated_at" : "2016-01-04T15:31:46.176Z", + "created_at" : "2016-01-04T15:31:46.176Z", + "closed_at" : "2016-01-05T15:31:46.176Z", + "closed_by" : { + "state" : "active", + "web_url" : "https://gitlab.example.com/root", + "avatar_url" : null, + "username" : "root", + "id" : 1, + "name" : "Administrator" + }, + "user_notes_count": 1, + "due_date": "2016-07-22", + "imported": false, + "imported_from": "none", + "web_url": "http://gitlab.example.com/my-group/my-project/issues/1", + "references": { + "short": "#1", + "relative": "#1", + "full": "my-group/my-project#1" + }, + "time_stats": { + "time_estimate": 0, + "total_time_spent": 0, + "human_time_estimate": null, + "human_total_time_spent": null + }, + "has_tasks": true, + "task_status": "10 of 15 tasks completed", + "confidential": false, + "discussion_locked": false, + "issue_type": "issue", + "severity": "UNKNOWN", + "_links":{ + "self":"http://gitlab.example.com/api/v4/projects/4/issues/41", + "notes":"http://gitlab.example.com/api/v4/projects/4/issues/41/notes", + "award_emoji":"http://gitlab.example.com/api/v4/projects/4/issues/41/award_emoji", + "project":"http://gitlab.example.com/api/v4/projects/4", + "closed_as_duplicate_of": "http://gitlab.example.com/api/v4/projects/1/issues/75" + }, + "task_completion_status":{ + "count":0, + "completed_count":0 + } + }` ) func setupEmptyTestServer(t *testing.T, role string, instanceSize string) (*http.ServeMux, *GitlabInstance) { @@ -552,6 +658,39 @@ func setupTestProject(mux *http.ServeMux, project *gitlab.Project, stringRespons w.WriteHeader(http.StatusOK) fmt.Fprint(w, "{}") }) + // Setup the get project issues response from the project ID + mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/issues", project.ID), func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + w.Header().Set(HEADER_CONTENT_TYPE, HEADER_ACCEPT) + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "[%s]", TEST_ISSUE_STRING) + case http.MethodPost: + w.Header().Set(HEADER_CONTENT_TYPE, HEADER_ACCEPT) + w.WriteHeader(http.StatusCreated) + fmt.Fprint(w, TEST_ISSUE_STRING) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + }) + // Setup the get project issue response from the project ID and issue IID + mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/issues/%d", project.ID, TEST_ISSUE.IID), func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + w.Header().Set(HEADER_CONTENT_TYPE, HEADER_ACCEPT) + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, TEST_ISSUE_STRING) + case http.MethodPut: + w.Header().Set(HEADER_CONTENT_TYPE, HEADER_ACCEPT) + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, TEST_ISSUE_STRING) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + }) + // Setup the get project issue notes response from the project ID and issue IID } func TestReverseGroupMirrorMap(t *testing.T) { diff --git a/internal/mirroring/issues.go b/internal/mirroring/issues.go new file mode 100644 index 0000000..191ae27 --- /dev/null +++ b/internal/mirroring/issues.go @@ -0,0 +1,158 @@ +package mirroring + +import ( + "fmt" + "gitlab-sync/pkg/helpers" + "sync" + + gitlab "gitlab.com/gitlab-org/api/client-go" + "go.uber.org/zap" +) + +var ( + CLOSE_STATE_EVENT = "close" +) + +// =========================================================================== +// ISSUES MIRRORING FUNCTIONS // +// =========================================================================== + +// ================ +// GET +// ================ + +// FetchProjectIssues retrieves all issues for a project and processes them +func (g *GitlabInstance) FetchProjectIssues(project *gitlab.Project) ([]*gitlab.Issue, error) { + zap.L().Debug("Fetching issues for project", zap.String("project", project.PathWithNamespace)) + fetchOpts := &gitlab.ListProjectIssuesOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: 100, + Page: 1, + }, + } + + var issues = make([]*gitlab.Issue, 0) + + for { + fetchedIssues, resp, err := g.Gitlab.Issues.ListProjectIssues(project.ID, fetchOpts) + if err != nil { + return nil, err + } + issues = append(issues, fetchedIssues...) + + if resp.CurrentPage >= resp.TotalPages { + break + } + fetchOpts.Page = resp.NextPage + } + + return issues, nil +} + +// FetchProjectIssuesTitles retrieves all issue titles for a project and returns them as a map +func (g *GitlabInstance) FetchProjectIssuesTitles(project *gitlab.Project) (map[string]struct{}, error) { + // Fetch existing issues from the destination project + issues, err := g.FetchProjectIssues(project) + if err != nil { + return nil, err + } + + // Create a map of existing issue titles for quick lookup + issueTitles := make(map[string]struct{}) + for _, issue := range issues { + if issue != nil { + issueTitles[issue.Title] = struct{}{} + } + } + return issueTitles, nil +} + +// ================ +// POST +// ================ + +// MirrorIssue creates an issue in the destination project +func (g *GitlabInstance) MirrorIssue(project *gitlab.Project, issue *gitlab.Issue) error { + zap.L().Debug("Creating issue in destination project", zap.String("issue", issue.Title), zap.String(ROLE_DESTINATION, project.HTTPURLToRepo)) + + // Create the issue in the destination project + _, _, err := g.Gitlab.Issues.CreateIssue(project.ID, &gitlab.CreateIssueOptions{ + Title: &issue.Title, + Description: &issue.Description, + Labels: (*gitlab.LabelOptions)(&issue.Labels), + CreatedAt: issue.CreatedAt, + Confidential: &issue.Confidential, + DueDate: issue.DueDate, + Weight: &issue.Weight, + IssueType: issue.IssueType, + }) + + if err == nil && issue.State == string(gitlab.ClosedEventType) { + // If the issue is closed, close it in the destination project + err = g.CloseIssue(project, issue) + } + + return err +} + +// CloseIssue closes an issue in the destination project +func (g *GitlabInstance) CloseIssue(project *gitlab.Project, issue *gitlab.Issue) error { + zap.L().Debug("Closing issue in destination project", zap.String("issue", issue.Title), zap.String(ROLE_DESTINATION, project.HTTPURLToRepo)) + _, _, err := g.Gitlab.Issues.UpdateIssue(project.ID, issue.IID, &gitlab.UpdateIssueOptions{ + StateEvent: &CLOSE_STATE_EVENT, + }) + return err +} + +// ================ +// CONTROLLER +// ================ + +// MirrorIssues mirrors issues from the source project to the destination project. +// It fetches existing issues from the destination project and creates new issues for those that do not +func (destinationGitlab *GitlabInstance) MirrorIssues(sourceGitlab *GitlabInstance, sourceProject *gitlab.Project, destinationProject *gitlab.Project) []error { + zap.L().Info("Starting issues mirroring", zap.String(ROLE_SOURCE, sourceProject.HTTPURLToRepo), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo)) + + // Fetch existing issues from the destination project + existingIssuesTitles, err := destinationGitlab.FetchProjectIssuesTitles(destinationProject) + if err != nil { + return []error{fmt.Errorf("failed to fetch existing issues for destination project %s: %v", destinationProject.HTTPURLToRepo, err)} + } + + // Fetch issues from the source project + sourceIssues, err := sourceGitlab.FetchProjectIssues(sourceProject) + if err != nil { + return []error{fmt.Errorf("failed to fetch issues for source project %s: %v", sourceProject.HTTPURLToRepo, err)} + } + + // Create a wait group and an error channel for handling API calls concurrently + var wg sync.WaitGroup + errorChan := make(chan error, len(sourceIssues)) + + // Iterate over each source issue + for _, issue := range sourceIssues { + // Check if the issue already exists in the destination project + if _, exists := existingIssuesTitles[issue.Title]; exists { + zap.L().Debug("Issue already exists", zap.String("issue", issue.Title), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo)) + continue + } + + // Increment the wait group counter + wg.Add(1) + + // Define the API call logic for creating an issue + go func(project *gitlab.Project, issueToMirror *gitlab.Issue) { + defer wg.Done() + err := destinationGitlab.MirrorIssue(destinationProject, issueToMirror) + if err != nil { + errorChan <- fmt.Errorf("failed to create issue %s in project %s: %s", issueToMirror.Title, destinationProject.HTTPURLToRepo, err) + } + }(destinationProject, issue) + } + + // Wait for all goroutines to finish + wg.Wait() + close(errorChan) + zap.L().Info("Issues mirroring completed", zap.String(ROLE_SOURCE, sourceProject.HTTPURLToRepo), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo)) + return helpers.MergeErrors(errorChan) +} diff --git a/internal/mirroring/issues_test.go b/internal/mirroring/issues_test.go new file mode 100644 index 0000000..f62ea9b --- /dev/null +++ b/internal/mirroring/issues_test.go @@ -0,0 +1,62 @@ +package mirroring + +import ( + "testing" +) + +func TestFetchProjectIssues(t *testing.T) { + _, gitlabInstance := setupTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL) + t.Run("Fetch Project Issues", func(t *testing.T) { + issues, err := gitlabInstance.FetchProjectIssues(TEST_PROJECT) + if err != nil { + t.Errorf("Unexpected error when fetching project issues: %v", err) + } + if len(issues) == 0 { + t.Error("Expected to fetch at least one issue") + } + }) +} + +func TestFetchProjectIssuesTitles(t *testing.T) { + _, gitlabInstance := setupTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL) + t.Run("Fetch Project Issues Titles", func(t *testing.T) { + issueTitles, err := gitlabInstance.FetchProjectIssuesTitles(TEST_PROJECT) + if err != nil { + t.Errorf("Unexpected error when fetching project issues titles: %v", err) + } + if len(issueTitles) == 0 { + t.Error("Expected to fetch at least one issue title") + } + }) +} + +func TestMirrorIssue(t *testing.T) { + _, gitlabInstance := setupTestServer(t, ROLE_SOURCE, INSTANCE_SIZE_SMALL) + t.Run("Mirror Issue", func(t *testing.T) { + err := gitlabInstance.MirrorIssue(TEST_PROJECT, TEST_ISSUE) + if err != nil { + t.Errorf("Unexpected error when mirroring issue: %v", err) + } + }) +} + +func TestCloseIssue(t *testing.T) { + _, gitlabInstance := setupTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL) + t.Run("Close Issue", func(t *testing.T) { + err := gitlabInstance.CloseIssue(TEST_PROJECT, TEST_ISSUE) + if err != nil { + t.Errorf("Unexpected error when closing issue: %v", err) + } + }) +} + +func TestMirrorIssues(t *testing.T) { + _, sourceGitlabInstance := setupTestServer(t, ROLE_SOURCE, INSTANCE_SIZE_SMALL) + _, destinationGitlabInstance := setupTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL) + t.Run("Mirror Issues", func(t *testing.T) { + errors := destinationGitlabInstance.MirrorIssues(sourceGitlabInstance, TEST_PROJECT, TEST_PROJECT_2) + if len(errors) > 0 { + t.Errorf("Unexpected errors when mirroring issues: %v", errors) + } + }) +} diff --git a/internal/mirroring/main.go b/internal/mirroring/main.go index e4bb3a9..f1ceb60 100644 --- a/internal/mirroring/main.go +++ b/internal/mirroring/main.go @@ -173,7 +173,6 @@ func (destinationGitlabInstance *GitlabInstance) DryRun(sourceGitlabInstance *Gi // DryRunReleases prints the releases that would be created in dry run mode. // It fetches the releases from the source project and prints them. -// It does not create any releases in the destination project. func (destinationGitlabInstance *GitlabInstance) DryRunReleases(sourceGitlabInstance *GitlabInstance, sourceProject *gitlab.Project, copyOptions *utils.MirroringOptions) error { // Fetch releases from the source project sourceReleases, _, err := sourceGitlabInstance.Gitlab.Releases.ListReleases(sourceProject.ID, &gitlab.ListReleasesOptions{}) @@ -187,6 +186,10 @@ func (destinationGitlabInstance *GitlabInstance) DryRunReleases(sourceGitlabInst return nil } +// =========================================================================== +// INSTANCE HEALTH MANAGEMENT // +// =========================================================================== + // IsPullMirrorAvailable checks the destination GitLab instance for version and license compatibility. func (g *GitlabInstance) IsPullMirrorAvailable(forcePremium bool, forceNonPremium bool) (bool, error) { zap.L().Info("Checking destination GitLab instance") diff --git a/internal/mirroring/main_test.go b/internal/mirroring/main_test.go index d89a982..a83c4ec 100644 --- a/internal/mirroring/main_test.go +++ b/internal/mirroring/main_test.go @@ -35,14 +35,14 @@ func TestProcessFilters(t *testing.T) { "sourceProject": { DestinationPath: "destinationGroupPath/destinationProjectPath", CI_CD_Catalog: true, - Issues: true, + MirrorIssues: true, }, }, Groups: map[string]*utils.MirroringOptions{ "sourceGroup": { DestinationPath: "destinationGroupPath", CI_CD_Catalog: true, - Issues: true, + MirrorIssues: true, }, }, }, @@ -66,24 +66,24 @@ func TestProcessFilters(t *testing.T) { "sourceProject1": { DestinationPath: "destinationGroupPath1/destinationProjectPath1", CI_CD_Catalog: true, - Issues: true, + MirrorIssues: true, }, "sourceProject2": { DestinationPath: "destinationGroupPath2/destinationProjectPath2", CI_CD_Catalog: false, - Issues: false, + MirrorIssues: false, }, }, Groups: map[string]*utils.MirroringOptions{ "sourceGroup1": { DestinationPath: "destinationGroupPath3", CI_CD_Catalog: true, - Issues: true, + MirrorIssues: true, }, "sourceGroup2": { DestinationPath: "destinationGroupPath4", CI_CD_Catalog: false, - Issues: false, + MirrorIssues: false, }, }, }, diff --git a/internal/mirroring/post.go b/internal/mirroring/post.go index 7c438a8..bb24ea6 100644 --- a/internal/mirroring/post.go +++ b/internal/mirroring/post.go @@ -217,77 +217,6 @@ func (g *GitlabInstance) createProjectFromSource(sourceProject *gitlab.Project, return destinationProject, err } -// ============================================================ // -// RELEASES CREATION FUNCTIONS // -// ============================================================ // - -// mirrorReleases mirrors releases from the source project to the destination project. -// It fetches existing releases from the destination project and creates new releases for those that do not exist. -// The function handles the API calls concurrently using goroutines and a wait group. -// It returns an error if any of the API calls fail. -func (destinationGitlab *GitlabInstance) mirrorReleases(sourceGitlab *GitlabInstance, sourceProject *gitlab.Project, destinationProject *gitlab.Project) []error { - zap.L().Info("Starting releases mirroring", zap.String(ROLE_SOURCE, sourceProject.HTTPURLToRepo), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo)) - // Fetch existing releases from the destination project - existingReleases, _, err := destinationGitlab.Gitlab.Releases.ListReleases(destinationProject.ID, &gitlab.ListReleasesOptions{}) - if err != nil { - return []error{fmt.Errorf("failed to fetch existing releases for destination project %s: %s", destinationProject.HTTPURLToRepo, err)} - } - - // Create a map of existing release tags for quick lookup - existingReleaseTags := make(map[string]struct{}) - for _, release := range existingReleases { - if release != nil { - existingReleaseTags[release.TagName] = struct{}{} - } - } - - // Fetch releases from the source project - sourceReleases, _, err := sourceGitlab.Gitlab.Releases.ListReleases(sourceProject.ID, &gitlab.ListReleasesOptions{}) - if err != nil { - return []error{fmt.Errorf("failed to fetch releases for source project %s: %s", sourceProject.HTTPURLToRepo, err)} - } - - // Create a wait group and an error channel for handling API calls concurrently - var wg sync.WaitGroup - errorChan := make(chan error, len(sourceReleases)) - - // Iterate over each source release - for _, release := range sourceReleases { - // Check if the release already exists in the destination project - if _, exists := existingReleaseTags[release.TagName]; exists { - zap.L().Debug("Release already exists", zap.String("release", release.TagName), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo)) - continue - } - - // Increment the wait group counter - wg.Add(1) - - // Define the API call logic for creating a release - go func(releaseToMirror *gitlab.Release) { - defer wg.Done() - zap.L().Debug("Creating release in destination project", zap.String("release", releaseToMirror.TagName), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo)) - - // Create the release in the destination project - _, _, err := destinationGitlab.Gitlab.Releases.CreateRelease(destinationProject.ID, &gitlab.CreateReleaseOptions{ - Name: &releaseToMirror.Name, - TagName: &releaseToMirror.TagName, - Description: &releaseToMirror.Description, - ReleasedAt: releaseToMirror.ReleasedAt, - }) - if err != nil { - errorChan <- fmt.Errorf("failed to create release %s in project %s: %s", releaseToMirror.TagName, destinationProject.HTTPURLToRepo, err) - } - }(release) - } - - // Wait for all goroutines to finish - wg.Wait() - close(errorChan) - - zap.L().Info("Releases mirroring completed", zap.String(ROLE_SOURCE, sourceProject.HTTPURLToRepo), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo)) - return helpers.MergeErrors(errorChan) -} - // ============================================================ // // CI/CD CATALOG FUNCTIONS // // ============================================================ // diff --git a/internal/mirroring/post_test.go b/internal/mirroring/post_test.go index 4aaf38f..4fac400 100644 --- a/internal/mirroring/post_test.go +++ b/internal/mirroring/post_test.go @@ -87,7 +87,7 @@ func TestCreateProjectFromSource(t *testing.T) { gitlabInstance.addGroup(TEST_GROUP) createdProject, err := gitlabInstance.createProjectFromSource(TEST_PROJECT, &utils.MirroringOptions{ DestinationPath: TEST_PROJECT.PathWithNamespace, - Issues: true, + MirrorIssues: true, MirrorReleases: true, MirrorTriggerBuilds: true, Visibility: "public", @@ -180,17 +180,6 @@ func TestCopyProjectAvatar(t *testing.T) { }) } -func TestMirrorReleases(t *testing.T) { - _, sourceGitlabInstance := setupTestServer(t, ROLE_SOURCE, INSTANCE_SIZE_SMALL) - _, destinationGitlabInstance := setupTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL) - t.Run("Mirror Releases", func(t *testing.T) { - err := destinationGitlabInstance.mirrorReleases(sourceGitlabInstance, TEST_PROJECT, TEST_PROJECT_2) - if err != nil { - t.Errorf("Unexpected error when mirroring releases: %v", err) - } - }) -} - func TestCreateProjects(t *testing.T) { t.Run("Test Create Projects", func(t *testing.T) { _, sourceGitlabInstance := setupTestServer(t, ROLE_SOURCE, INSTANCE_SIZE_SMALL) @@ -204,7 +193,7 @@ func TestCreateProjects(t *testing.T) { TEST_PROJECT.PathWithNamespace: { DestinationPath: TEST_PROJECT.PathWithNamespace, CI_CD_Catalog: false, - Issues: true, + MirrorIssues: true, MirrorTriggerBuilds: false, Visibility: "public", MirrorReleases: true, diff --git a/internal/mirroring/put.go b/internal/mirroring/put.go index 6a56b89..1c877ff 100644 --- a/internal/mirroring/put.go +++ b/internal/mirroring/put.go @@ -30,7 +30,7 @@ func (destinationGitlabInstance *GitlabInstance) updateProjectFromSource(sourceG wg := sync.WaitGroup{} wg.Add(3) - errorChan := make(chan error, 4) + errorChan := make(chan error, 5) go func(sp *gitlab.Project, dp *gitlab.Project) { defer wg.Done() @@ -55,6 +55,19 @@ func (destinationGitlabInstance *GitlabInstance) updateProjectFromSource(sourceG }(dstProj) } + if copyOptions.MirrorIssues { + wg.Add(1) + go func(sp *gitlab.Project, dp *gitlab.Project) { + defer wg.Done() + allErrors := destinationGitlabInstance.MirrorIssues(sourceGitlabInstance, sp, dp) + for _, err := range allErrors { + if err != nil { + errorChan <- fmt.Errorf("failed to mirror issues from %s to %s: %v", sp.HTTPURLToRepo, dp.HTTPURLToRepo, err) + } + } + }(srcProj, dstProj) + } + // Wait for git duplication to finish wg.Wait() @@ -63,7 +76,7 @@ func (destinationGitlabInstance *GitlabInstance) updateProjectFromSource(sourceG wg.Add(1) go func(sp *gitlab.Project, dp *gitlab.Project) { defer wg.Done() - allErrors = destinationGitlabInstance.mirrorReleases(sourceGitlabInstance, sp, dp) + allErrors = destinationGitlabInstance.MirrorReleases(sourceGitlabInstance, sp, dp) }(srcProj, dstProj) } diff --git a/internal/mirroring/releases.go b/internal/mirroring/releases.go new file mode 100644 index 0000000..492afbf --- /dev/null +++ b/internal/mirroring/releases.go @@ -0,0 +1,136 @@ +package mirroring + +import ( + "fmt" + "gitlab-sync/pkg/helpers" + "sync" + + gitlab "gitlab.com/gitlab-org/api/client-go" + "go.uber.org/zap" +) + +// =========================================================================== +// RELEASES MIRRORING FUNCTIONS // +// =========================================================================== + +// ================ +// GET +// ================ + +// FetchProjectReleases retrieves all releases for a project and returns them +func (g *GitlabInstance) FetchProjectReleases(project *gitlab.Project) ([]*gitlab.Release, error) { + zap.L().Debug("Fetching releases for project", zap.String("project", project.PathWithNamespace)) + fetchOpts := &gitlab.ListReleasesOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: 100, + Page: 1, + }, + } + + var releases = make([]*gitlab.Release, 0) + + for { + fetchedReleases, resp, err := g.Gitlab.Releases.ListReleases(project.ID, fetchOpts) + if err != nil { + return nil, err + } + releases = append(releases, fetchedReleases...) + + if resp.CurrentPage >= resp.TotalPages { + break + } + fetchOpts.Page = resp.NextPage + } + + return releases, nil +} + +// FetchProjectReleasesTags retrieves all release tags for a project and returns them as a map +func (g *GitlabInstance) FetchProjectReleasesTags(project *gitlab.Project) (map[string]struct{}, error) { + // Fetch existing releases from the destination project + releases, err := g.FetchProjectReleases(project) + if err != nil { + return nil, err + } + + // Create a map of existing release tags for quick lookup + releasesTags := make(map[string]struct{}) + for _, release := range releases { + if release != nil { + releasesTags[release.TagName] = struct{}{} + } + } + return releasesTags, nil +} + +// ================ +// POST +// ================ + +// MirrorRelease creates a release in the destination project +func (g *GitlabInstance) MirrorRelease(project *gitlab.Project, release *gitlab.Release) error { + zap.L().Debug("Creating release in destination project", zap.String("release", release.TagName), zap.String(ROLE_DESTINATION, project.HTTPURLToRepo)) + + // Create the release in the destination project + _, _, err := g.Gitlab.Releases.CreateRelease(project.ID, &gitlab.CreateReleaseOptions{ + Name: &release.Name, + TagName: &release.TagName, + Description: &release.Description, + ReleasedAt: release.ReleasedAt, + }) + return err +} + +// ================ +// CONTROLLER +// ================ + +// MirrorReleases mirrors releases from the source project to the destination project. +// It fetches existing releases from the destination project and creates new releases for those that do not exist. +// The function handles the API calls concurrently using goroutines +func (destinationGitlab *GitlabInstance) MirrorReleases(sourceGitlab *GitlabInstance, sourceProject *gitlab.Project, destinationProject *gitlab.Project) []error { + zap.L().Info("Starting releases mirroring", zap.String(ROLE_SOURCE, sourceProject.HTTPURLToRepo), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo)) + // Fetch existing releases from the destination project + existingReleasesTags, err := destinationGitlab.FetchProjectReleasesTags(destinationProject) + if err != nil { + return []error{fmt.Errorf("failed to fetch existing releases for destination project %s: %s", destinationProject.HTTPURLToRepo, err)} + } + + // Fetch releases from the source project + sourceReleases, err := sourceGitlab.FetchProjectReleases(sourceProject) + if err != nil { + return []error{fmt.Errorf("failed to fetch releases for source project %s: %s", sourceProject.HTTPURLToRepo, err)} + } + + // Create a wait group and an error channel for handling API calls concurrently + var wg sync.WaitGroup + errorChan := make(chan error, len(sourceReleases)) + + // Iterate over each source release + for _, release := range sourceReleases { + // Check if the release already exists in the destination project + if _, exists := existingReleasesTags[release.TagName]; exists { + zap.L().Debug("Release already exists", zap.String("release", release.TagName), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo)) + continue + } + + // Increment the wait group counter + wg.Add(1) + + // Define the API call logic for creating a release + go func(releaseToMirror *gitlab.Release) { + defer wg.Done() + err := destinationGitlab.MirrorRelease(destinationProject, releaseToMirror) + if err != nil { + errorChan <- fmt.Errorf("failed to create release %s in project %s: %s", releaseToMirror.TagName, destinationProject.HTTPURLToRepo, err) + } + }(release) + } + + // Wait for all goroutines to finish + wg.Wait() + close(errorChan) + + zap.L().Info("Releases mirroring completed", zap.String(ROLE_SOURCE, sourceProject.HTTPURLToRepo), zap.String(ROLE_DESTINATION, destinationProject.HTTPURLToRepo)) + return helpers.MergeErrors(errorChan) +} diff --git a/internal/mirroring/releases_test.go b/internal/mirroring/releases_test.go new file mode 100644 index 0000000..b8b3567 --- /dev/null +++ b/internal/mirroring/releases_test.go @@ -0,0 +1,59 @@ +package mirroring + +import ( + "testing" + + gitlab "gitlab.com/gitlab-org/api/client-go" +) + +func TestMirrorReleases(t *testing.T) { + _, sourceGitlabInstance := setupTestServer(t, ROLE_SOURCE, INSTANCE_SIZE_SMALL) + _, destinationGitlabInstance := setupTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL) + t.Run("Mirror Releases", func(t *testing.T) { + err := destinationGitlabInstance.MirrorReleases(sourceGitlabInstance, TEST_PROJECT, TEST_PROJECT_2) + if err != nil { + t.Errorf("Unexpected error when mirroring releases: %v", err) + } + }) +} + +func TestFetchProjectReleases(t *testing.T) { + _, gitlabInstance := setupTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL) + t.Run("Fetch Project Releases", func(t *testing.T) { + releases, err := gitlabInstance.FetchProjectReleases(TEST_PROJECT) + if err != nil { + t.Errorf("Unexpected error when fetching project releases: %v", err) + } + if len(releases) == 0 { + t.Error("Expected to fetch at least one release") + } + }) +} + +func TestFetchProjectReleasesTags(t *testing.T) { + _, gitlabInstance := setupTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL) + t.Run("Fetch Project Releases Tags", func(t *testing.T) { + releasesTags, err := gitlabInstance.FetchProjectReleasesTags(TEST_PROJECT) + if err != nil { + t.Errorf("Unexpected error when fetching project releases tags: %v", err) + } + if len(releasesTags) == 0 { + t.Error("Expected to fetch at least one release tag") + } + }) +} + +func TestMirrorRelease(t *testing.T) { + _, gitlabInstance := setupTestServer(t, ROLE_SOURCE, INSTANCE_SIZE_SMALL) + t.Run("Mirror Release", func(t *testing.T) { + release := &gitlab.Release{ + Name: "Test Release", + TagName: "v1.0.0", + Description: "This is a test release", + } + err := gitlabInstance.MirrorRelease(TEST_PROJECT, release) + if err != nil { + t.Errorf("Unexpected error when mirroring release: %v", err) + } + }) +} diff --git a/internal/utils/types.go b/internal/utils/types.go index fd1e248..6e063c6 100644 --- a/internal/utils/types.go +++ b/internal/utils/types.go @@ -56,7 +56,7 @@ type ParserArgs struct { type MirroringOptions struct { DestinationPath string `json:"destination_path"` CI_CD_Catalog bool `json:"ci_cd_catalog"` - Issues bool `json:"issues"` + MirrorIssues bool `json:"mirror_issues"` MirrorTriggerBuilds bool `json:"mirror_trigger_builds"` Visibility string `json:"visibility"` MirrorReleases bool `json:"mirror_releases"` diff --git a/internal/utils/types_test.go b/internal/utils/types_test.go index 8a0be21..e3a276e 100644 --- a/internal/utils/types_test.go +++ b/internal/utils/types_test.go @@ -21,7 +21,7 @@ var ( FAKE_VALID_PROJECT: { DestinationPath: FAKE_VALID_PROJECT, CI_CD_Catalog: true, - Issues: true, + MirrorIssues: true, MirrorTriggerBuilds: false, Visibility: "private", }, @@ -30,7 +30,7 @@ var ( FAKE_VALID_GROUP: { DestinationPath: FAKE_VALID_GROUP, CI_CD_Catalog: true, - Issues: true, + MirrorIssues: true, MirrorTriggerBuilds: false, Visibility: "private", }, @@ -42,7 +42,7 @@ var ( "%s": { "destination_path": "%s", "ci_cd_catalog": true, - "issues": true, + "mirror_issues": true, "mirror_trigger_builds": false, "visibility": "private" } @@ -51,7 +51,7 @@ var ( "%s": { "destination_path": "%s", "ci_cd_catalog": true, - "issues": true, + "mirror_issues": true, "mirror_trigger_builds": false, "visibility": "private" } @@ -64,7 +64,7 @@ func testMirroringOptions() *MirroringOptions { return &MirroringOptions{ DestinationPath: "project", CI_CD_Catalog: true, - Issues: true, + MirrorIssues: true, } } @@ -170,14 +170,14 @@ func TestCheck(t *testing.T) { FAKE_VALID_PROJECT: { DestinationPath: FAKE_VALID_PROJECT, CI_CD_Catalog: true, - Issues: true, + MirrorIssues: true, }, }, Groups: map[string]*MirroringOptions{ FAKE_VALID_GROUP: { DestinationPath: FAKE_VALID_GROUP, CI_CD_Catalog: true, - Issues: true, + MirrorIssues: true, }, }, }, @@ -401,7 +401,7 @@ func TestMirrorMappingGetProject(t *testing.T) { opts1 := &MirroringOptions{ DestinationPath: "dest1", CI_CD_Catalog: true, - Issues: false, + MirrorIssues: false, MirrorTriggerBuilds: true, Visibility: "public", MirrorReleases: false, @@ -409,7 +409,7 @@ func TestMirrorMappingGetProject(t *testing.T) { opts2 := &MirroringOptions{ DestinationPath: "dest2", CI_CD_Catalog: false, - Issues: true, + MirrorIssues: true, MirrorTriggerBuilds: false, Visibility: "private", MirrorReleases: true, @@ -468,7 +468,7 @@ func TestMirrorMappingGetGroup(t *testing.T) { optsA := &MirroringOptions{ DestinationPath: "groupDestA", CI_CD_Catalog: true, - Issues: true, + MirrorIssues: true, MirrorTriggerBuilds: false, Visibility: "internal", MirrorReleases: true, @@ -476,7 +476,7 @@ func TestMirrorMappingGetGroup(t *testing.T) { optsB := &MirroringOptions{ DestinationPath: "groupDestB", CI_CD_Catalog: false, - Issues: false, + MirrorIssues: false, MirrorTriggerBuilds: true, Visibility: "private", MirrorReleases: false, diff --git a/pkg/helpers/formats.go b/pkg/helpers/formats.go index 63a31be..d014e1e 100644 --- a/pkg/helpers/formats.go +++ b/pkg/helpers/formats.go @@ -1,5 +1,9 @@ package helpers +import ( + "strings" +) + // toStrings converts a []error into a []string for easy comparison func ToStrings(errs []error) []string { if errs == nil { @@ -11,3 +15,24 @@ func ToStrings(errs []error) []string { } return out } + +// MatchPathAgainstFilters determines if a given path matches any of the specified filters. +// - Returns true if the path is an exact match in the allowList. +// - Returns true if the path has a prefix matching any entry in the prefixList, and returns the matching prefix. +// +// If a prefix match is found, the matching prefix is returned. Otherwise, an empty string is returned. +func MatchPathAgainstFilters(path string, allowList *map[string]struct{}, prefixList *map[string]struct{}) (string, bool) { + if allowList != nil { + if _, ok := (*allowList)[path]; ok { + return "", true + } + } + if prefixList != nil { + for prefix := range *prefixList { + if strings.HasPrefix(path, prefix) { + return prefix, true + } + } + } + return "", false +} diff --git a/pkg/helpers/formats_test.go b/pkg/helpers/formats_test.go new file mode 100644 index 0000000..33cb81e --- /dev/null +++ b/pkg/helpers/formats_test.go @@ -0,0 +1,29 @@ +package helpers + +import "testing" + +func TestMatchPathAgainstFilters(t *testing.T) { + allowList := map[string]struct{}{ + "/allowed/path": {}, + } + prefixList := map[string]struct{}{ + "/prefix/": {}, + } + + tests := []struct { + path string + expected string + matched bool + }{ + {"/allowed/path", "", true}, + {"/prefix/something", "/prefix/", true}, + {"/not/matching", "", false}, + } + + for _, test := range tests { + prefix, matched := MatchPathAgainstFilters(test.path, &allowList, &prefixList) + if prefix != test.expected || matched != test.matched { + t.Errorf("For path %s: expected (%s, %t), got (%s, %t)", test.path, test.expected, test.matched, prefix, matched) + } + } +}