diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index e9f5bb4..5c2b5f5 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,7 +2,6 @@ name: lint-test on: workflow_dispatch: - push: pull_request: permissions: @@ -17,12 +16,13 @@ jobs: fetch-depth: 0 - uses: actions/setup-go@v5 with: - go-version: "1.24.2" + go-version: "1.24.4" - run: go mod tidy - name: golangci-lint - uses: golangci/golangci-lint-action@v7 + uses: golangci/golangci-lint-action@v8 with: - version: v2.0 + version: latest + args: --tests=false continue-on-error: true gosec: @@ -35,7 +35,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-go@v5 with: - go-version: "1.24.2" + go-version: "1.24.4" - run: go mod tidy - name: Run Gosec Security Scanner uses: securego/gosec@master @@ -47,9 +47,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-go@v5 with: - go-version: "1.24.2" + go-version: "1.24.4" - run: go mod tidy - run: go install gotest.tools/gotestsum@latest - run: gotestsum --junitfile unit-tests.xml -- -coverprofile=coverage.out -covermode=atomic ./... diff --git a/.gitignore b/.gitignore index f8e12f3..f1f4cd5 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,5 @@ sonar-project.properties unit-tests.xml coverage.out .scannerwork/ +*-report* +.sonarlint/ \ No newline at end of file diff --git a/README.md b/README.md index 41eabcc..915c6a0 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,9 @@ It is designed to be used in a CI/CD pipeline to automate the process of keeping ## Features - Synchronize projects / groups between two GitLab instances -- Enable Pull Mirroring for projects (requires GitLab Premium) +- Recreates your git repository content in another location: + - Enable Pull Mirroring for projects (requires GitLab Premium) + - Clone the repository content from the source GitLab instance to the destination GitLab instance (on GitLab Free) - Can add projects to CI/CD catalog - Full copy of the project (description, icon, topics,...). Can also copy issues diff --git a/cmd/main.go b/cmd/main.go index 1205e7a..f09eb7e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -77,6 +77,8 @@ func main() { rootCmd.Flags().StringVar(&args.DestinationGitlabURL, "destination-url", os.Getenv("DESTINATION_GITLAB_URL"), "Destination GitLab URL") rootCmd.Flags().StringVar(&args.DestinationGitlabToken, "destination-token", os.Getenv("DESTINATION_GITLAB_TOKEN"), "Destination GitLab Token") rootCmd.Flags().BoolVar(&args.DestinationGitlabIsBig, "destination-big", strings.TrimSpace(os.Getenv("DESTINATION_GITLAB_BIG")) != "", "Destination GitLab is a big instance") + rootCmd.Flags().BoolVarP(&args.ForcePremium, "destination-force-premium", "p", false, "Force the destination GitLab to be treated as a premium instance") + rootCmd.Flags().BoolVarP(&args.ForceNonPremium, "destination-force-freemium", "f", false, "Force the destination GitLab to be treated as a non premium instance") rootCmd.Flags().BoolVarP(&args.Verbose, "verbose", "v", false, "Enable verbose output") rootCmd.Flags().BoolVarP(&args.NoPrompt, "no-prompt", "n", strings.TrimSpace(os.Getenv("NO_PROMPT")) != "", "Disable prompting for missing values") rootCmd.Flags().StringVar(&mirrorMappingPath, "mirror-mapping", os.Getenv("MIRROR_MAPPING"), "Path to the mirror mapping file") diff --git a/go.mod b/go.mod index 8de817b..1e63a6f 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,41 @@ module gitlab-sync -go 1.24.3 +go 1.24.4 require ( - github.com/Masterminds/semver/v3 v3.3.1 - github.com/hashicorp/go-retryablehttp v0.7.7 + github.com/Masterminds/semver/v3 v3.4.0 + github.com/go-git/go-git/v5 v5.16.2 + github.com/hashicorp/go-retryablehttp v0.7.8 github.com/spf13/cobra v1.9.1 - gitlab.com/gitlab-org/api/client-go v0.128.0 + gitlab.com/gitlab-org/api/client-go v0.134.0 go.uber.org/zap v1.27.0 ) require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.3.0 // indirect + github.com/cloudflare/circl v1.6.1 // indirect + github.com/cyphar/filepath-securejoin v0.4.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.6.2 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/pjbgf/sha1cd v0.4.0 // indirect + github.com/sergi/go-diff v1.4.0 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect github.com/spf13/pflag v1.0.6 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.42.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/time v0.11.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/time v0.12.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/internal/mirroring/get.go b/internal/mirroring/get.go index ab31539..0ed4189 100644 --- a/internal/mirroring/get.go +++ b/internal/mirroring/get.go @@ -86,39 +86,38 @@ func checkPathMatchesFilters(resourcePath string, projectFilters *map[string]str return "", false } -func (g *GitlabInstance) CheckVersion() error { +// IsVersionGreaterThanThreshold checks if the GitLab instance version is below the defined threshold. +// It retrieves the metadata from the GitLab instance and compares the version +// with the INSTANCE_SEMVER_THRESHOLD. +func (g *GitlabInstance) IsVersionGreaterThanThreshold() (bool, error) { metadata, _, err := g.Gitlab.Metadata.GetMetadata() if err != nil { - return fmt.Errorf("failed to get GitLab version: %w", err) + return false, fmt.Errorf("failed to get GitLab version: %w", err) } zap.L().Debug("GitLab Instance version", zap.String(ROLE, g.Role), zap.String("version", metadata.Version)) currentVer, err := semver.NewVersion(metadata.Version) if err != nil { - return fmt.Errorf("failed to parse GitLab version: %w", err) + return false, fmt.Errorf("failed to parse GitLab version: %w", err) } thresholdVer, err := semver.NewVersion(INSTANCE_SEMVER_THRESHOLD) if err != nil { - return fmt.Errorf("failed to parse version threshold: %w", err) + return false, fmt.Errorf("failed to parse version threshold: %w", err) } - if currentVer.LessThan(thresholdVer) { - return fmt.Errorf("GitLab version %s is below required threshold %s", currentVer, thresholdVer) - } - return nil + return currentVer.GreaterThanEqual(thresholdVer), nil } -func (g *GitlabInstance) CheckLicense() error { +// IsLicensePremium checks if the GitLab instance has a premium license. +// It retrieves the license information and checks the plan type. +func (g *GitlabInstance) IsLicensePremium() (bool, error) { license, _, err := g.Gitlab.License.GetLicense() if err != nil { - return fmt.Errorf("failed to get GitLab license: %w", err) + return false, fmt.Errorf("failed to get GitLab license: %w", err) } - if license.Plan != ULTIMATE_PLAN && license.Plan != PREMIUM_PLAN { - return fmt.Errorf("GitLab license plan %s is not supported, only %s and %s are supported", license.Plan, ULTIMATE_PLAN, PREMIUM_PLAN) - } else if license.Expired { - return fmt.Errorf("GitLab license is expired") + zap.L().Info("GitLab Instance license", zap.String(ROLE, g.Role), zap.String("plan", license.Plan)) + if license.Plan != ULTIMATE_PLAN && license.Plan != PREMIUM_PLAN || license.Expired { + return false, nil } - - zap.L().Debug("GitLab Instance license", zap.String(ROLE, g.Role), zap.String("plan", license.Plan)) - return nil + return true, nil } diff --git a/internal/mirroring/get_group.go b/internal/mirroring/get_group.go index 3c48ad1..24aed46 100644 --- a/internal/mirroring/get_group.go +++ b/internal/mirroring/get_group.go @@ -195,7 +195,7 @@ func (g *GitlabInstance) fetchAndProcessGroupRecursive(gid any, fetchOriginPath } if group != nil { g.storeGroup(group, fetchOriginPath, mirrorMapping) - if g.isSource() { + if g.isSource() || g.isBig() { wg.Add(2) // Fetch the projects of the group go g.fetchAndProcessGroupProjects(group, fetchOriginPath, mirrorMapping, errChan, wg) @@ -225,7 +225,7 @@ func (g *GitlabInstance) fetchAndProcessGroupSubgroups(group *gitlab.Group, fetc } for _, subgroup := range subgroups { g.storeGroup(subgroup, fetchOriginPath, mirrorMapping) - if g.isSource() { + if g.isSource() || g.isBig() { wg.Add(1) go g.fetchAndProcessGroupRecursive(subgroup, fetchOriginPath, mirrorMapping, errChan, wg) } diff --git a/internal/mirroring/get_test.go b/internal/mirroring/get_test.go index 4a5595c..4d3a63b 100644 --- a/internal/mirroring/get_test.go +++ b/internal/mirroring/get_test.go @@ -6,6 +6,10 @@ import ( "testing" ) +const ( + EXPECTED_ERROR_MESSAGE = "expected error: %v, got: %v" +) + func TestCheckPathMatchesFilters(t *testing.T) { tests := []struct { name string @@ -112,7 +116,7 @@ func TestGetParentNamespaceID(t *testing.T) { // Check if an error was expected if (err != nil) != test.expectedError { - t.Errorf("expected error: %v, got: %v", test.expectedError, err) + t.Errorf(EXPECTED_ERROR_MESSAGE, test.expectedError, err) } }) } @@ -180,7 +184,7 @@ func TestFetchAll(t *testing.T) { // Check if an error was expected if (err != nil) != test.expectedError { - t.Errorf("expected error: %v, got: %v", test.expectedError, err) + t.Errorf(EXPECTED_ERROR_MESSAGE, test.expectedError, err) } //Check if the instance cache contains the expected projects and groups @@ -195,36 +199,50 @@ func TestFetchAll(t *testing.T) { } -func TestCheckVersion(t *testing.T) { +func TestIsVersionGreaterThanThreshold(t *testing.T) { tests := []struct { - name string - version string - expectedError bool + name string + version string + expectedError bool + expectedResponse bool + noApiResponse bool }{ { - name: "Valid version under threshold", - version: "15.0.0", - expectedError: true, + name: "Valid version under threshold", + version: "15.0.0", + expectedError: false, + expectedResponse: false, }, { - name: "Valid version above threshold", - version: "17.9.3-ce.0", - expectedError: false, + name: "Valid version above threshold", + version: "17.9.3-ce.0", + expectedError: false, + expectedResponse: true, }, { - name: "Invalid version format with 1 dot", - version: "invalid.version", - expectedError: true, + name: "Invalid version format with 1 dot", + version: "invalid.version", + expectedError: true, + expectedResponse: false, }, { - name: "Invalid version format with 2 dots", - version: "invalid.version.1", - expectedError: true, + name: "Invalid version format with 2 dots", + version: "invalid.version.1", + expectedError: true, + expectedResponse: false, }, { - name: "Invalid empty version", - version: "", - expectedError: true, + name: "Invalid empty version", + version: "", + expectedError: true, + expectedResponse: false, + }, + { + name: "No API response", + version: "", + expectedError: true, + expectedResponse: false, + noApiResponse: true, }, } @@ -234,53 +252,64 @@ func TestCheckVersion(t *testing.T) { t.Parallel() mux, gitlabInstance := setupEmptyTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL) - mux.HandleFunc("/api/v4/metadata", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _, err := w.Write([]byte(`{"version": "` + test.version + `"}`)) - if err != nil { - t.Errorf("failed to write response: %v", err) - } - }) + if !test.noApiResponse { + mux.HandleFunc("/api/v4/metadata", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{"version": "` + test.version + `"}`)) + if err != nil { + t.Errorf("failed to write response: %v", err) + } + }) + } - err := gitlabInstance.CheckVersion() + thresholdOk, err := gitlabInstance.IsVersionGreaterThanThreshold() if (err != nil) != test.expectedError { - t.Errorf("expected error: %v, got: %v", test.expectedError, err) + t.Fatalf(EXPECTED_ERROR_MESSAGE, test.expectedError, err) + } + if thresholdOk != test.expectedResponse { + t.Errorf("expected thresholdOk: %v, got: %v", test.expectedResponse, thresholdOk) } }) } } -func TestCheckLicense(t *testing.T) { +func TestIsLicensePremium(t *testing.T) { tests := []struct { - name string - license string - expectedError bool + name string + license string + expectedError bool + expectedResponse bool }{ { - name: "Ultimate tier license", - license: ULTIMATE_PLAN, - expectedError: false, + name: "Ultimate tier license", + license: ULTIMATE_PLAN, + expectedError: false, + expectedResponse: true, }, { - name: "Premium tier license", - license: PREMIUM_PLAN, - expectedError: false, + name: "Premium tier license", + license: PREMIUM_PLAN, + expectedError: false, + expectedResponse: true, }, { - name: "Free tier license", - license: "free", - expectedError: true, + name: "Free tier license", + license: "free", + expectedError: false, + expectedResponse: false, }, { - name: "Invalid license", - license: "invalid", - expectedError: true, + name: "Invalid license", + license: "invalid", + expectedError: false, + expectedResponse: false, }, { - name: "Empty license", - license: "", - expectedError: true, + name: "Error API response", + license: "", + expectedError: true, + expectedResponse: false, }, } // Iterate over the test cases @@ -289,18 +318,23 @@ func TestCheckLicense(t *testing.T) { t.Parallel() mux, gitlabInstance := setupEmptyTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL) - mux.HandleFunc("/api/v4/license", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _, err := w.Write([]byte(`{"plan": "` + test.license + `"}`)) - if err != nil { - t.Errorf("failed to write response: %v", err) - } - }) + if !test.expectedError { + mux.HandleFunc("/api/v4/license", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{"plan": "` + test.license + `"}`)) + if err != nil { + t.Errorf("failed to write response: %v", err) + } + }) + } - err := gitlabInstance.CheckLicense() + isPremium, err := gitlabInstance.IsLicensePremium() if (err != nil) != test.expectedError { - t.Errorf("expected error: %v, got: %v", test.expectedError, err) + t.Fatalf(EXPECTED_ERROR_MESSAGE, test.expectedError, err) + } + if isPremium != test.expectedResponse { + t.Errorf("expected isPremium: %v, got: %v", test.expectedResponse, isPremium) } }) } diff --git a/internal/mirroring/instance.go b/internal/mirroring/instance.go index 0f87e49..c90b5ad 100644 --- a/internal/mirroring/instance.go +++ b/internal/mirroring/instance.go @@ -1,8 +1,10 @@ package mirroring import ( + "gitlab-sync/pkg/helpers" "sync" + "github.com/go-git/go-git/v5/plumbing/transport" "github.com/hashicorp/go-retryablehttp" gitlab "gitlab.com/gitlab-org/api/client-go" ) @@ -28,6 +30,10 @@ type GitlabInstance struct { // InstanceSize is the size of the GitLab instance, it can be either "small" or "big" // It is used to determine the behavior of the fetching process InstanceSize string + // PullMirrorAvailable is a boolean indicating whether the GitLab instance supports pull mirroring + PullMirrorAvailable bool + // GitAuth is the HTTP authentication used for GitLab git over HTTP operations (only for non premium instances) + GitAuth transport.AuthMethod } type GitlabInstanceOpts struct { @@ -59,6 +65,7 @@ func newGitlabInstance(initArgs *GitlabInstanceOpts) (*GitlabInstance, error) { Groups: make(map[string]*gitlab.Group), Role: initArgs.Role, InstanceSize: initArgs.InstanceSize, + GitAuth: helpers.BuildHTTPAuth("", initArgs.GitlabToken), } return gitlabInstance, nil diff --git a/internal/mirroring/main.go b/internal/mirroring/main.go index 194c7fd..e4bb3a9 100644 --- a/internal/mirroring/main.go +++ b/internal/mirroring/main.go @@ -19,10 +19,13 @@ import ( // If the dry run flag is set, it will only print the groups and projects that would be created or updated. func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) []error { zap.L().Info("Starting GitLab mirroring process", zap.String(ROLE_SOURCE, gitlabMirrorArgs.SourceGitlabURL), zap.String(ROLE_DESTINATION, gitlabMirrorArgs.DestinationGitlabURL)) + + // Create source GitLab instance sourceGitlabSize := INSTANCE_SIZE_SMALL if gitlabMirrorArgs.SourceGitlabIsBig { sourceGitlabSize = INSTANCE_SIZE_BIG } + sourceGitlabInstance, err := newGitlabInstance(&GitlabInstanceOpts{ GitlabURL: gitlabMirrorArgs.SourceGitlabURL, GitlabToken: gitlabMirrorArgs.SourceGitlabToken, @@ -34,6 +37,7 @@ func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) []error { return []error{err} } + // Create destination GitLab instance destinationGitlabSize := INSTANCE_SIZE_SMALL if gitlabMirrorArgs.DestinationGitlabIsBig { destinationGitlabSize = INSTANCE_SIZE_BIG @@ -48,10 +52,20 @@ func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) []error { if err != nil { return []error{err} } - err = destinationGitlabInstance.CheckDestinationInstance() + pullMirrorAvailable, err := destinationGitlabInstance.IsPullMirrorAvailable(gitlabMirrorArgs.ForcePremium, gitlabMirrorArgs.ForceNonPremium) + if err != nil { + // Could not obtain a result from the destination GitLab instance, so we cannot proceed with the mirroring process. return []error{err} + } else if pullMirrorAvailable { + // Proceed with the pull mirroring process + zap.L().Info("GitLab instance is compatible with the pull mirroring process", zap.String(ROLE, destinationGitlabInstance.Role), zap.String(INSTANCE_SIZE, destinationGitlabInstance.InstanceSize)) + } else { + // Use local pull/push mirroring instead + zap.L().Warn("Destination GitLab instance is not compatible with the pull mirroring process (requires a >= 17.6 ; >= Premium destination GitLab instance)", zap.String(ROLE, destinationGitlabInstance.Role), zap.String(INSTANCE_SIZE, destinationGitlabInstance.InstanceSize)) + zap.L().Warn("Will use local pull / push mirroring instead (takes a lot longer)", zap.String(ROLE, destinationGitlabInstance.Role), zap.String(INSTANCE_SIZE, destinationGitlabInstance.InstanceSize)) } + destinationGitlabInstance.PullMirrorAvailable = pullMirrorAvailable sourceProjectFilters, sourceGroupFilters, destinationProjectFilters, destinationGroupFilters := processFilters(gitlabMirrorArgs.MirrorMapping) @@ -70,6 +84,8 @@ func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) []error { wg.Wait() + zap.L().Debug("Fully Computed Mirror Mapping", zap.Any("MirrorMapping", gitlabMirrorArgs.MirrorMapping)) + // In case of dry run, simply print the groups and projects that would be created or updated if gitlabMirrorArgs.DryRun { destinationGitlabInstance.DryRun(sourceGitlabInstance, gitlabMirrorArgs.MirrorMapping) @@ -98,7 +114,7 @@ func processFilters(filters *utils.MirrorMapping) (map[string]struct{}, map[stri var wg sync.WaitGroup wg.Add(2) - // Process group filters concurrently. + // Process group filters concurrently go func() { defer wg.Done() for group, copyOptions := range filters.Groups { @@ -109,7 +125,7 @@ func processFilters(filters *utils.MirrorMapping) (map[string]struct{}, map[stri } }() - // Process project filters concurrently. + // Process project filters concurrently go func() { defer wg.Done() for project, copyOptions := range filters.Projects { @@ -121,7 +137,6 @@ func processFilters(filters *utils.MirrorMapping) (map[string]struct{}, map[stri destinationGroupFilters[destinationGroupPath] = struct{}{} mu.Unlock() } - } }() @@ -172,14 +187,20 @@ func (destinationGitlabInstance *GitlabInstance) DryRunReleases(sourceGitlabInst return nil } -// CheckDestinationInstance checks the destination GitLab instance for version and license compatibility. -func (g *GitlabInstance) CheckDestinationInstance() error { +// IsPullMirrorAvailable checks the destination GitLab instance for version and license compatibility. +func (g *GitlabInstance) IsPullMirrorAvailable(forcePremium bool, forceNonPremium bool) (bool, error) { zap.L().Info("Checking destination GitLab instance") - if err := g.CheckVersion(); err != nil { - return fmt.Errorf("destination GitLab instance version check failed: %w", err) + thresholdOk, err := g.IsVersionGreaterThanThreshold() + if err != nil { + return false, fmt.Errorf("destination GitLab instance version check failed: %w", err) } - if err := g.CheckLicense(); err != nil { - return fmt.Errorf("destination GitLab instance version check failed: %w", err) + + isPremium, err := g.IsLicensePremium() + if err != nil { + if !forcePremium && !forceNonPremium { + return false, fmt.Errorf("failed to check if destination GitLab instance is premium: %w", err) + } } - return nil + + return !forceNonPremium && (thresholdOk && (isPremium || forcePremium)), nil } diff --git a/internal/mirroring/main_test.go b/internal/mirroring/main_test.go index bf65ff3..d89a982 100644 --- a/internal/mirroring/main_test.go +++ b/internal/mirroring/main_test.go @@ -174,72 +174,95 @@ func TestDryRun(t *testing.T) { } } -func TestCheckDestinationInstance(t *testing.T) { +func TestIsPullMirrorAvailable(t *testing.T) { + const supportedVersion = "18.0.0" + const unsupportedVersion = "17.0.0" tests := []struct { - name string - licensePlan string - version string - expectedError bool + name string + licensePlan string + version string + expectedError bool + expectedResult bool + forcePremium bool }{ { - name: "Premium license, good version", - licensePlan: PREMIUM_PLAN, - version: "18.0.0", - expectedError: false, + name: "Premium license, good version", + licensePlan: PREMIUM_PLAN, + version: supportedVersion, + expectedResult: true, }, { - name: "Ultimate license, good version", - licensePlan: ULTIMATE_PLAN, - version: "18.0.0", - expectedError: false, + name: "Ultimate license, good version", + licensePlan: ULTIMATE_PLAN, + version: supportedVersion, + expectedResult: true, }, { - name: "Free license, good version", - licensePlan: "free", - version: "18.0.0", - expectedError: true, + name: "Free license, good version", + licensePlan: "free", + version: supportedVersion, + expectedResult: false, }, { - name: "Premium license, bad version", - licensePlan: PREMIUM_PLAN, - version: "17.0.0", - expectedError: true, + name: "Free license, good version, force premium", + licensePlan: "free", + version: supportedVersion, + expectedResult: true, + forcePremium: true, }, { - name: "Ultimate license, bad version", - licensePlan: ULTIMATE_PLAN, - version: "17.0.0", - expectedError: true, + name: "Premium license, bad version", + licensePlan: PREMIUM_PLAN, + version: unsupportedVersion, + expectedResult: false, }, { - name: "Bad license, good version", - licensePlan: "bad_license", - version: "18.0.0", - expectedError: true, + name: "Ultimate license, bad version", + licensePlan: ULTIMATE_PLAN, + version: unsupportedVersion, + expectedResult: false, }, { - name: "Bad license, bad version", - licensePlan: "bad_license", - version: "17.0.0", - expectedError: true, + name: "Bad license, good version", + licensePlan: "bad_license", + version: supportedVersion, + expectedResult: false, + }, + { + name: "Bad license, bad version", + licensePlan: "bad_license", + version: unsupportedVersion, + expectedResult: false, + }, + { + name: "Error API response", + licensePlan: "", + version: "", + expectedError: true, + expectedResult: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() mux, gitlabInstance := setupEmptyTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL) - mux.HandleFunc("/api/v4/license", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Write([]byte(`{"plan": "` + tt.licensePlan + `", "expired": false}`)) - }) - mux.HandleFunc("/api/v4/metadata", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Write([]byte(`{"version": "` + tt.version + `"}`)) - }) + if !tt.expectedError { + mux.HandleFunc("/api/v4/license", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte(`{"plan": "` + tt.licensePlan + `", "expired": false}`)) + }) + mux.HandleFunc("/api/v4/metadata", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte(`{"version": "` + tt.version + `"}`)) + }) + } - err := gitlabInstance.CheckDestinationInstance() + pullMirrorAvailable, err := gitlabInstance.IsPullMirrorAvailable(tt.forcePremium, false) if (err != nil) != tt.expectedError { - t.Errorf("CheckDestinationInstance() error = %v, expectedError %v", err, tt.expectedError) + t.Fatalf("CheckDestinationInstance() error = %v, expectedError %v", err, tt.expectedError) + } + if pullMirrorAvailable != tt.expectedResult { + t.Errorf("CheckDestinationInstance() = %v, expectedResult %v", pullMirrorAvailable, tt.expectedResult) } }) } diff --git a/internal/mirroring/post.go b/internal/mirroring/post.go index 7163d65..7c438a8 100644 --- a/internal/mirroring/post.go +++ b/internal/mirroring/post.go @@ -1,7 +1,6 @@ package mirroring import ( - "context" "fmt" "sync" @@ -312,6 +311,6 @@ func (g *GitlabInstance) addProjectToCICDCatalog(project *gitlab.Project) error } `json:"data"` } - _, err := g.Gitlab.GraphQL.Do(context.Background(), gitlab.GraphQLQuery{Query: query}, &response) + _, err := g.Gitlab.GraphQL.Do(gitlab.GraphQLQuery{Query: query}, &response) return err } diff --git a/internal/mirroring/post_test.go b/internal/mirroring/post_test.go index 903abca..4aaf38f 100644 --- a/internal/mirroring/post_test.go +++ b/internal/mirroring/post_test.go @@ -198,6 +198,7 @@ func TestCreateProjects(t *testing.T) { sourceGitlabInstance.addProject(TEST_PROJECT) _, destinationGitlabInstance := setupTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL) destinationGitlabInstance.addGroup(TEST_GROUP) + destinationGitlabInstance.PullMirrorAvailable = true mirrorMapping := &utils.MirrorMapping{ Projects: map[string]*utils.MirroringOptions{ TEST_PROJECT.PathWithNamespace: { @@ -211,7 +212,7 @@ func TestCreateProjects(t *testing.T) { }, } err := destinationGitlabInstance.createProjects(sourceGitlabInstance, mirrorMapping) - if err != nil && len(err) > 0 { + if len(err) > 0 { t.Errorf("Unexpected error when creating projects: %v", err) } if len(destinationGitlabInstance.Projects) == 0 { diff --git a/internal/mirroring/put.go b/internal/mirroring/put.go index dde65ce..6a56b89 100644 --- a/internal/mirroring/put.go +++ b/internal/mirroring/put.go @@ -29,16 +29,8 @@ func (destinationGitlabInstance *GitlabInstance) updateProjectFromSource(sourceG } wg := sync.WaitGroup{} - maxErrors := 3 - if copyOptions.CI_CD_Catalog { - maxErrors++ - } - if copyOptions.MirrorReleases { - maxErrors++ - } - - wg.Add(maxErrors) - errorChan := make(chan error, maxErrors) + wg.Add(3) + errorChan := make(chan error, 4) go func(sp *gitlab.Project, dp *gitlab.Project) { defer wg.Done() @@ -47,7 +39,7 @@ func (destinationGitlabInstance *GitlabInstance) updateProjectFromSource(sourceG go func(sp *gitlab.Project, dp *gitlab.Project) { defer wg.Done() - errorChan <- destinationGitlabInstance.enableProjectMirrorPull(sp, dp, copyOptions) + errorChan <- destinationGitlabInstance.mirrorProjectGit(sourceGitlabInstance, sp, dp, copyOptions) }(srcProj, dstProj) go func(sp *gitlab.Project, dp *gitlab.Project) { @@ -56,14 +48,19 @@ func (destinationGitlabInstance *GitlabInstance) updateProjectFromSource(sourceG }(srcProj, dstProj) if copyOptions.CI_CD_Catalog { + wg.Add(1) go func(dp *gitlab.Project) { defer wg.Done() errorChan <- destinationGitlabInstance.addProjectToCICDCatalog(dp) }(dstProj) } + // Wait for git duplication to finish + wg.Wait() + allErrors := []error{} if copyOptions.MirrorReleases { + wg.Add(1) go func(sp *gitlab.Project, dp *gitlab.Project) { defer wg.Done() allErrors = destinationGitlabInstance.mirrorReleases(sourceGitlabInstance, sp, dp) @@ -133,6 +130,13 @@ func (destinationGitlabInstance *GitlabInstance) syncProjectAttributes(sourcePro return nil } +func (destinationGitlabInstance *GitlabInstance) mirrorProjectGit(sourceGitlabInstance *GitlabInstance, sourceProject *gitlab.Project, destinationProject *gitlab.Project, mirrorOptions *utils.MirroringOptions) error { + if destinationGitlabInstance.PullMirrorAvailable { + return destinationGitlabInstance.enableProjectMirrorPull(sourceProject, destinationProject, mirrorOptions) + } + return helpers.MirrorRepo(sourceProject.HTTPURLToRepo, destinationProject.HTTPURLToRepo, sourceGitlabInstance.GitAuth, destinationGitlabInstance.GitAuth) +} + // enableProjectMirrorPull enables the pull mirror for a project in the destination GitLab instance. // It sets the source project URL, enables mirroring, and configures other options like triggering builds and overwriting diverged branches. func (g *GitlabInstance) enableProjectMirrorPull(sourceProject *gitlab.Project, destinationProject *gitlab.Project, mirrorOptions *utils.MirroringOptions) error { diff --git a/internal/utils/types.go b/internal/utils/types.go index 4b3d781..fd1e248 100644 --- a/internal/utils/types.go +++ b/internal/utils/types.go @@ -39,6 +39,8 @@ type ParserArgs struct { DestinationGitlabURL string DestinationGitlabToken string DestinationGitlabIsBig bool + ForcePremium bool + ForceNonPremium bool MirrorMapping *MirrorMapping Verbose bool NoPrompt bool @@ -72,18 +74,25 @@ type MirrorMapping struct { muGroups sync.RWMutex } +// AddProject adds a project to the mapping +// It takes the project name and the mirroring options as parameters +// It locks the projects mutex to ensure thread safety func (m *MirrorMapping) AddProject(project string, options *MirroringOptions) { m.muProjects.Lock() defer m.muProjects.Unlock() m.Projects[project] = options } +// AddGroup adds a group to the mapping +// It takes the group name and the mirroring options as parameters +// It locks the groups mutex to ensure thread safety func (m *MirrorMapping) AddGroup(group string, options *MirroringOptions) { m.muGroups.Lock() defer m.muGroups.Unlock() m.Groups[group] = options } +// GetProject retrieves the mirroring options for a project func (m *MirrorMapping) GetProject(project string) (*MirroringOptions, bool) { m.muProjects.RLock() defer m.muProjects.RUnlock() @@ -91,6 +100,7 @@ func (m *MirrorMapping) GetProject(project string) (*MirroringOptions, bool) { return options, ok } +// GetGroup retrieves the mirroring options for a group func (m *MirrorMapping) GetGroup(group string) (*MirroringOptions, bool) { m.muGroups.RLock() defer m.muGroups.RUnlock() @@ -167,6 +177,9 @@ func (m *MirrorMapping) checkProjects(errChan chan error) { } } +// checkCopyPaths checks if the source and destination paths are valid +// It checks if the paths are not empty, do not start or end with a slash, +// and if the destination path is in a namespace for projects func checkCopyPaths(sourcePath string, destinationPath string, pathType string, errChan chan error) { // Ensure the source project path and destination path are not empty if sourcePath == "" || destinationPath == "" { @@ -191,6 +204,8 @@ func checkCopyPaths(sourcePath string, destinationPath string, pathType string, } } +// checkGroups checks if the groups are valid +// It checks if the group names and destination paths are valid func (m *MirrorMapping) checkGroups(errChan chan error) { duplicateDestinationFinder := make(map[string]struct{}, len(m.Groups)) for group, options := range m.Groups { @@ -212,6 +227,8 @@ func (m *MirrorMapping) checkGroups(errChan chan error) { } } +// checkVisibility checks if the visibility string is valid +// It checks if the visibility string is one of the valid GitLab visibility values func checkVisibility(visibility string) bool { var valid bool switch visibility { @@ -227,6 +244,8 @@ func checkVisibility(visibility string) bool { return valid } +// ConvertVisibility converts a visibility string to a gitlab.VisibilityValue +// It returns the corresponding gitlab.VisibilityValue or gitlab.PublicVisibility if the string is invalid func ConvertVisibility(visibility string) gitlab.VisibilityValue { switch visibility { case string(gitlab.PublicVisibility): @@ -240,6 +259,8 @@ func ConvertVisibility(visibility string) gitlab.VisibilityValue { } } +// StringArraysMatchValues checks if two string arrays match in values +// It returns true if both arrays have the same values, regardless of order func StringArraysMatchValues(array1 []string, array2 []string) bool { if len(array1) != len(array2) { return false diff --git a/internal/utils/types_test.go b/internal/utils/types_test.go index 75823bf..8a0be21 100644 --- a/internal/utils/types_test.go +++ b/internal/utils/types_test.go @@ -396,7 +396,7 @@ func TestCheckVisibility(t *testing.T) { } } -func TestMirrorMapping_GetProject(t *testing.T) { +func TestMirrorMappingGetProject(t *testing.T) { // Prepare a mirror mapping with some project entries opts1 := &MirroringOptions{ DestinationPath: "dest1", @@ -463,7 +463,7 @@ func TestMirrorMapping_GetProject(t *testing.T) { } } -func TestMirrorMapping_GetGroup(t *testing.T) { +func TestMirrorMappingGetGroup(t *testing.T) { // Prepare a mirror mapping with some group entries optsA := &MirroringOptions{ DestinationPath: "groupDestA", diff --git a/pkg/helpers/git.go b/pkg/helpers/git.go new file mode 100644 index 0000000..ed63bcd --- /dev/null +++ b/pkg/helpers/git.go @@ -0,0 +1,123 @@ +package helpers + +import ( + "fmt" + "net/url" + "os" + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "go.uber.org/zap" +) + +const ( + DEFAULT_GIT_USER = "git" +) + +// MirrorRepo clones the source remote as a bare repo and pushes all refs +// (branches, tags, and then fixes the bare-repo HEAD) to the destination. +func MirrorRepo(sourceURL, destinationURL string, pullAuth, pushAuth transport.AuthMethod) error { + // Clone source into a temp bare repo: + tmpDir, err := os.MkdirTemp("", "bare-mirror-*") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + pullOpts := &git.CloneOptions{ + URL: sourceURL, + Mirror: true, + } + if pullAuth != nil { + pullOpts.Auth = pullAuth + } + + zap.L().Debug("Cloning source repository", zap.String("sourceURL", sourceURL), zap.String("destinationURL", destinationURL)) + srcRepo, err := git.PlainClone(tmpDir, true, pullOpts) + if err != nil { + return fmt.Errorf("failed to clone source repository locally: %w", err) + } + + // Add destination as a remote + zap.L().Debug("Adding destination remote", zap.String("destinationURL", destinationURL)) + _, err = srcRepo.CreateRemote(&config.RemoteConfig{ + Name: "destination", + URLs: []string{destinationURL}, + }) + if err != nil { + return fmt.Errorf("failed to create remote for destination: %w", err) + } + + // Push *all* refs up to it + zap.L().Debug("Pushing to destination repository", zap.String("destinationURL", destinationURL)) + pushOpts := &git.PushOptions{ + RemoteName: "destination", + Force: true, + RefSpecs: []config.RefSpec{ + // force-update everything (branches, tags, etc) + config.RefSpec("+refs/*:refs/*"), + }, + } + if pushAuth != nil { + pushOpts.Auth = pushAuth + } + + if err := srcRepo.Push(pushOpts); err != nil { + return fmt.Errorf("failed to push to destination repository: %w", err) + } + + // Finally, repair the bare-repo HEAD to point at the right branch + if err := fixBareRepoHEAD(destinationURL, srcRepo); err != nil { + return fmt.Errorf("failed to set destination HEAD: %w", err) + } + + return nil +} + +// fixBareRepoHEAD will open the bare repo on disk (via file:// URL), +// read the srcRepo’s HEAD symbolic name (e.g. refs/heads/main), and then +// rewrite the bare repo’s HEAD to point there. +// (This is necessary because bare repos do not have a working tree, +// so they cannot automatically determine the HEAD branch.) +func fixBareRepoHEAD(destinationURL string, srcRepo *git.Repository) error { + u, err := url.Parse(destinationURL) + if err != nil { + return err + } + path := u.Path + + destRepo, err := git.PlainOpen(path) + if err != nil { + return err + } + + // figure out what branch the source HEAD was on + srcHead, err := srcRepo.Head() + if err != nil { + return err + } + + // write a new symbolic HEAD in the bare repo + zap.L().Debug("Setting HEAD in destination repository", zap.String("destinationURL", destinationURL), zap.String("branch", srcHead.Name().String())) + sym := plumbing.NewSymbolicReference(plumbing.HEAD, srcHead.Name()) + return destRepo.Storer.SetReference(sym) +} + +// BuildHTTPAuth creates an HTTP BasicAuth object using a username and token. +func BuildHTTPAuth(username string, token string) transport.AuthMethod { + if token == "" && username == "" { + return nil + } + + if strings.TrimSpace(username) == "" { + username = DEFAULT_GIT_USER + } + return &http.BasicAuth{ + Username: username, + Password: token, + } +} diff --git a/pkg/helpers/git_test.go b/pkg/helpers/git_test.go new file mode 100644 index 0000000..c6cd557 --- /dev/null +++ b/pkg/helpers/git_test.go @@ -0,0 +1,105 @@ +package helpers + +import ( + "os" + "testing" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/storer" + "github.com/go-git/go-git/v5/plumbing/transport/http" +) + +const ( + FILE_SCHEME = "file://" + githubHTTPURL = "https://github.com/BoxBoxJason/gitlab-sync.git" +) + +func TestBuildHTTPAuth(t *testing.T) { + tests := []struct { + name string + username, token string + wantUser, wantPW string + }{ + {"both provided", "alice", "secr3t", "alice", "secr3t"}, + {"empty username", "", "tk", DEFAULT_GIT_USER, "tk"}, + {"spaces username", " ", "x", DEFAULT_GIT_USER, "x"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + auth := BuildHTTPAuth(tt.username, tt.token) + basic, ok := auth.(*http.BasicAuth) + if !ok { + t.Fatalf("BuildHTTPAuth returned non-BasicAuth: %T", auth) + } + if basic.Username != tt.wantUser { + t.Errorf("Username = %q; want %q", basic.Username, tt.wantUser) + } + if basic.Password != tt.wantPW { + t.Errorf("Password = %q; want %q", basic.Password, tt.wantPW) + } + }) + } +} + +func TestMirrorRepo(t *testing.T) { + t.Run("mirror via HTTPS public repo", func(t *testing.T) { + t.Parallel() + destDir, err := os.MkdirTemp("/tmp", "destrepo-*.git") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(destDir) + + // Initialize a bare git repository at the destination + _, err = git.PlainInit(destDir, true) // true indicates a bare repository + if err != nil { + t.Fatalf("failed to initialize bare repository at destination: %v", err) + } + + if err := MirrorRepo(githubHTTPURL, FILE_SCHEME+destDir, nil, nil); err != nil { + t.Fatalf("MirrorRepo(HTTPS) failed: %v", err) + } + + destRepo, err := git.PlainOpen(destDir) + if err != nil { + t.Fatal(err) + } + headRef, err := destRepo.Head() + if err != nil { + t.Fatalf("dest HEAD error: %v", err) + } + if headRef.Hash().IsZero() { + t.Error("dest HEAD hash is zero") + } + branches, _ := destRepo.Branches() + found := false + _ = branches.ForEach(func(r *plumbing.Reference) error { + found = true + return storer.ErrStop + }) + if !found { + t.Error("no branches found in mirrored repo") + } + }) + + t.Run("error on invalid source", func(t *testing.T) { + t.Parallel() + destDir, err := os.MkdirTemp("", "destrepo-bad") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(destDir) + + // Initialize a bare git repository at the destination + _, err = git.PlainInit(destDir, true) + if err != nil { + t.Fatalf("failed to initialize bare repository at destination: %v", err) + } + + err = MirrorRepo("file:///no/such/path", FILE_SCHEME+destDir, nil, nil) + if err == nil { + t.Error("expected error for invalid source URL, got nil") + } + }) +}