feat: enable mirroring on non premium instances

add a git operations helper to clone the git content with minimal resources
update compatibility prechecks
add new means of auth for the git over https part
update tests
update docs

Closes #13

Author: BoxBoxJason

README.md

@@ -19,7 +19,9 @@ It is designed to be used in a CI/CD pipeline to automate the process of keeping
 ## Features
 
 - Synchronize projects / groups between two GitLab instances
-- Enable Pull Mirroring for projects (requires GitLab Premium)
+- Recreates your git repository content in another location:
+  - Enable Pull Mirroring for projects (requires GitLab Premium)
+  - Clone the repository content from the source GitLab instance to the destination GitLab instance (on GitLab Free)
 - Can add projects to CI/CD catalog
 - Full copy of the project (description, icon, topics,...). Can also copy issues
 

go.mod

@@ -1,21 +1,41 @@
 module gitlab-sync
 
-go 1.24.3
+go 1.24.4
 
 require (
 	github.com/Masterminds/semver/v3 v3.3.1
+	github.com/go-git/go-git/v5 v5.16.0
 	github.com/hashicorp/go-retryablehttp v0.7.7
 	github.com/spf13/cobra v1.9.1
 	gitlab.com/gitlab-org/api/client-go v0.128.0
 	go.uber.org/zap v1.27.0
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/ProtonMail/go-crypto v1.1.6 // indirect
+	github.com/cloudflare/circl v1.6.1 // indirect
+	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.6.2 // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/pjbgf/sha1cd v0.3.2 // indirect
+	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
+	github.com/skeema/knownhosts v1.3.1 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.37.0 // indirect
+	golang.org/x/net v0.39.0 // indirect
 	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/sys v0.32.0 // indirect
 	golang.org/x/time v0.11.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 )

internal/mirroring/get.go

@@ -86,39 +86,38 @@ func checkPathMatchesFilters(resourcePath string, projectFilters *map[string]str
 	return "", false
 }
 
-func (g *GitlabInstance) CheckVersion() error {
+// IsVersionGreaterThanThreshold checks if the GitLab instance version is below the defined threshold.
+// It retrieves the metadata from the GitLab instance and compares the version
+// with the INSTANCE_SEMVER_THRESHOLD.
+func (g *GitlabInstance) IsVersionGreaterThanThreshold() (bool, error) {
 	metadata, _, err := g.Gitlab.Metadata.GetMetadata()
 	if err != nil {
-		return fmt.Errorf("failed to get GitLab version: %w", err)
+		return false, fmt.Errorf("failed to get GitLab version: %w", err)
 	}
 	zap.L().Debug("GitLab Instance version", zap.String(ROLE, g.Role), zap.String("version", metadata.Version))
 
 	currentVer, err := semver.NewVersion(metadata.Version)
 	if err != nil {
-		return fmt.Errorf("failed to parse GitLab version: %w", err)
+		return false, fmt.Errorf("failed to parse GitLab version: %w", err)
 	}
 	thresholdVer, err := semver.NewVersion(INSTANCE_SEMVER_THRESHOLD)
 	if err != nil {
-		return fmt.Errorf("failed to parse version threshold: %w", err)
+		return false, fmt.Errorf("failed to parse version threshold: %w", err)
 	}
 
-	if currentVer.LessThan(thresholdVer) {
-		return fmt.Errorf("GitLab version %s is below required threshold %s", currentVer, thresholdVer)
-	}
-	return nil
+	return currentVer.GreaterThanEqual(thresholdVer), nil
 }
 
-func (g *GitlabInstance) CheckLicense() error {
+// IsLicensePremium checks if the GitLab instance has a premium license.
+// It retrieves the license information and checks the plan type.
+func (g *GitlabInstance) IsLicensePremium() (bool, error) {
 	license, _, err := g.Gitlab.License.GetLicense()
 	if err != nil {
-		return fmt.Errorf("failed to get GitLab license: %w", err)
+		return false, fmt.Errorf("failed to get GitLab license: %w", err)
 	}
-	if license.Plan != ULTIMATE_PLAN && license.Plan != PREMIUM_PLAN {
-		return fmt.Errorf("GitLab license plan %s is not supported, only %s and %s are supported", license.Plan, ULTIMATE_PLAN, PREMIUM_PLAN)
-	} else if license.Expired {
-		return fmt.Errorf("GitLab license is expired")
+	zap.L().Info("GitLab Instance license", zap.String(ROLE, g.Role), zap.String("plan", license.Plan))
+	if license.Plan != ULTIMATE_PLAN && license.Plan != PREMIUM_PLAN || license.Expired {
+		return false, nil
 	}
-
-	zap.L().Debug("GitLab Instance license", zap.String(ROLE, g.Role), zap.String("plan", license.Plan))
-	return nil
+	return true, nil
 }

internal/mirroring/get_test.go

@@ -195,36 +195,50 @@ func TestFetchAll(t *testing.T) {
 
 }
 
-func TestCheckVersion(t *testing.T) {
+func TestIsVersionGreaterThanThreshold(t *testing.T) {
 	tests := []struct {
-		name          string
-		version       string
-		expectedError bool
+		name             string
+		version          string
+		expectedError    bool
+		expectedResponse bool
+		noApiResponse    bool
 	}{
 		{
-			name:          "Valid version under threshold",
-			version:       "15.0.0",
-			expectedError: true,
+			name:             "Valid version under threshold",
+			version:          "15.0.0",
+			expectedError:    false,
+			expectedResponse: false,
 		},
 		{
-			name:          "Valid version above threshold",
-			version:       "17.9.3-ce.0",
-			expectedError: false,
+			name:             "Valid version above threshold",
+			version:          "17.9.3-ce.0",
+			expectedError:    false,
+			expectedResponse: true,
 		},
 		{
-			name:          "Invalid version format with 1 dot",
-			version:       "invalid.version",
-			expectedError: true,
+			name:             "Invalid version format with 1 dot",
+			version:          "invalid.version",
+			expectedError:    true,
+			expectedResponse: false,
 		},
 		{
-			name:          "Invalid version format with 2 dots",
-			version:       "invalid.version.1",
-			expectedError: true,
+			name:             "Invalid version format with 2 dots",
+			version:          "invalid.version.1",
+			expectedError:    true,
+			expectedResponse: false,
 		},
 		{
-			name:          "Invalid empty version",
-			version:       "",
-			expectedError: true,
+			name:             "Invalid empty version",
+			version:          "",
+			expectedError:    true,
+			expectedResponse: false,
+		},
+		{
+			name:             "No API response",
+			version:          "",
+			expectedError:    true,
+			expectedResponse: false,
+			noApiResponse:    true,
 		},
 	}
 
@@ -234,53 +248,64 @@ func TestCheckVersion(t *testing.T) {
 			t.Parallel()
 
 			mux, gitlabInstance := setupEmptyTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL)
-			mux.HandleFunc("/api/v4/metadata", func(w http.ResponseWriter, r *http.Request) {
-				w.Header().Set("Content-Type", "application/json")
-				w.WriteHeader(http.StatusOK)
-				_, err := w.Write([]byte(`{"version": "` + test.version + `"}`))
-				if err != nil {
-					t.Errorf("failed to write response: %v", err)
-				}
-			})
+			if !test.noApiResponse {
+				mux.HandleFunc("/api/v4/metadata", func(w http.ResponseWriter, r *http.Request) {
+					w.Header().Set("Content-Type", "application/json")
+					w.WriteHeader(http.StatusOK)
+					_, err := w.Write([]byte(`{"version": "` + test.version + `"}`))
+					if err != nil {
+						t.Errorf("failed to write response: %v", err)
+					}
+				})
+			}
 
-			err := gitlabInstance.CheckVersion()
+			thresholdOk, err := gitlabInstance.IsVersionGreaterThanThreshold()
 			if (err != nil) != test.expectedError {
-				t.Errorf("expected error: %v, got: %v", test.expectedError, err)
+				t.Fatalf("expected error: %v, got: %v", test.expectedError, err)
+			}
+			if thresholdOk != test.expectedResponse {
+				t.Errorf("expected thresholdOk: %v, got: %v", test.expectedResponse, thresholdOk)
 			}
 		})
 	}
 }
 
-func TestCheckLicense(t *testing.T) {
+func TestIsLicensePremium(t *testing.T) {
 	tests := []struct {
-		name          string
-		license       string
-		expectedError bool
+		name             string
+		license          string
+		expectedError    bool
+		expectedResponse bool
 	}{
 		{
-			name:          "Ultimate tier license",
-			license:       ULTIMATE_PLAN,
-			expectedError: false,
+			name:             "Ultimate tier license",
+			license:          ULTIMATE_PLAN,
+			expectedError:    false,
+			expectedResponse: true,
 		},
 		{
-			name:          "Premium tier license",
-			license:       PREMIUM_PLAN,
-			expectedError: false,
+			name:             "Premium tier license",
+			license:          PREMIUM_PLAN,
+			expectedError:    false,
+			expectedResponse: true,
 		},
 		{
-			name:          "Free tier license",
-			license:       "free",
-			expectedError: true,
+			name:             "Free tier license",
+			license:          "free",
+			expectedError:    false,
+			expectedResponse: false,
 		},
 		{
-			name:          "Invalid license",
-			license:       "invalid",
-			expectedError: true,
+			name:             "Invalid license",
+			license:          "invalid",
+			expectedError:    false,
+			expectedResponse: false,
 		},
 		{
-			name:          "Empty license",
-			license:       "",
-			expectedError: true,
+			name:             "Error API response",
+			license:          "",
+			expectedError:    true,
+			expectedResponse: false,
 		},
 	}
 	// Iterate over the test cases
@@ -289,18 +314,23 @@ func TestCheckLicense(t *testing.T) {
 			t.Parallel()
 
 			mux, gitlabInstance := setupEmptyTestServer(t, ROLE_DESTINATION, INSTANCE_SIZE_SMALL)
-			mux.HandleFunc("/api/v4/license", func(w http.ResponseWriter, r *http.Request) {
-				w.Header().Set("Content-Type", "application/json")
-				w.WriteHeader(http.StatusOK)
-				_, err := w.Write([]byte(`{"plan": "` + test.license + `"}`))
-				if err != nil {
-					t.Errorf("failed to write response: %v", err)
-				}
-			})
+			if !test.expectedError {
+				mux.HandleFunc("/api/v4/license", func(w http.ResponseWriter, r *http.Request) {
+					w.Header().Set("Content-Type", "application/json")
+					w.WriteHeader(http.StatusOK)
+					_, err := w.Write([]byte(`{"plan": "` + test.license + `"}`))
+					if err != nil {
+						t.Errorf("failed to write response: %v", err)
+					}
+				})
+			}
 
-			err := gitlabInstance.CheckLicense()
+			isPremium, err := gitlabInstance.IsLicensePremium()
 			if (err != nil) != test.expectedError {
-				t.Errorf("expected error: %v, got: %v", test.expectedError, err)
+				t.Fatalf("expected error: %v, got: %v", test.expectedError, err)
+			}
+			if isPremium != test.expectedResponse {
+				t.Errorf("expected isPremium: %v, got: %v", test.expectedResponse, isPremium)
 			}
 		})
 	}

internal/mirroring/instance.go

@@ -1,8 +1,10 @@
 package mirroring
 
 import (
+	"gitlab-sync/pkg/helpers"
 	"sync"
 
+	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/hashicorp/go-retryablehttp"
 	gitlab "gitlab.com/gitlab-org/api/client-go"
 )
@@ -28,6 +30,10 @@ type GitlabInstance struct {
 	// InstanceSize is the size of the GitLab instance, it can be either "small" or "big"
 	// It is used to determine the behavior of the fetching process
 	InstanceSize string
+	// PullMirrorAvailable is a boolean indicating whether the GitLab instance supports pull mirroring
+	PullMirrorAvailable bool
+	// GitAuth is the HTTP authentication used for GitLab git over HTTP operations (only for non premium instances)
+	GitAuth transport.AuthMethod
 }
 
 type GitlabInstanceOpts struct {
@@ -59,6 +65,7 @@ func newGitlabInstance(initArgs *GitlabInstanceOpts) (*GitlabInstance, error) {
 		Groups:       make(map[string]*gitlab.Group),
 		Role:         initArgs.Role,
 		InstanceSize: initArgs.InstanceSize,
+		GitAuth:      helpers.BuildHTTPAuth("", initArgs.GitlabToken),
 	}
 
 	return gitlabInstance, nil

internal/mirroring/main.go

@@ -48,10 +48,16 @@ func MirrorGitlabs(gitlabMirrorArgs *utils.ParserArgs) []error {
 	if err != nil {
 		return []error{err}
 	}
-	err = destinationGitlabInstance.CheckDestinationInstance()
+	pullMirrorAvailable, err := destinationGitlabInstance.IsPullMirrorAvailable()
 	if err != nil {
 		return []error{err}
+	} else if pullMirrorAvailable {
+		zap.L().Info("GitLab instance is compatible with the pull mirroring process", zap.String(ROLE, destinationGitlabInstance.Role), zap.String(INSTANCE_SIZE, destinationGitlabInstance.InstanceSize))
+	} else {
+		zap.L().Warn("Destination GitLab instance is not compatible with the pull mirroring process (requires a >= 17.6 ; >= Premium destination GitLab instance)", zap.String(ROLE, destinationGitlabInstance.Role), zap.String(INSTANCE_SIZE, destinationGitlabInstance.InstanceSize))
+		zap.L().Warn("Will use local pull / push mirroring instead (takes a lot longer)", zap.String(ROLE, destinationGitlabInstance.Role), zap.String(INSTANCE_SIZE, destinationGitlabInstance.InstanceSize))
 	}
+	destinationGitlabInstance.PullMirrorAvailable = pullMirrorAvailable
 
 	sourceProjectFilters, sourceGroupFilters, destinationProjectFilters, destinationGroupFilters := processFilters(gitlabMirrorArgs.MirrorMapping)
 
@@ -121,7 +127,6 @@ func processFilters(filters *utils.MirrorMapping) (map[string]struct{}, map[stri
 				destinationGroupFilters[destinationGroupPath] = struct{}{}
 				mu.Unlock()
 			}
-
 		}
 	}()
 
@@ -172,14 +177,18 @@ func (destinationGitlabInstance *GitlabInstance) DryRunReleases(sourceGitlabInst
 	return nil
 }
 
-// CheckDestinationInstance checks the destination GitLab instance for version and license compatibility.
-func (g *GitlabInstance) CheckDestinationInstance() error {
+// IsPullMirrorAvailable checks the destination GitLab instance for version and license compatibility.
+func (g *GitlabInstance) IsPullMirrorAvailable() (bool, error) {
 	zap.L().Info("Checking destination GitLab instance")
-	if err := g.CheckVersion(); err != nil {
-		return fmt.Errorf("destination GitLab instance version check failed: %w", err)
+	thresholdOk, err := g.IsVersionGreaterThanThreshold()
+	if err != nil {
+		return false, fmt.Errorf("destination GitLab instance version check failed: %w", err)
 	}
-	if err := g.CheckLicense(); err != nil {
-		return fmt.Errorf("destination GitLab instance version check failed: %w", err)
+
+	isPremium, err := g.IsLicensePremium()
+	if err != nil {
+		return false, fmt.Errorf("failed to check if destination GitLab instance is premium: %w", err)
 	}
-	return nil
+
+	return thresholdOk && isPremium, nil
 }