build: Add first project draft

Author: BoxBoxJason

.gitignore

@@ -0,0 +1,54 @@
+# IDE
+.vscode
+.idea
+.idea/
+*.iml
+.idea_modules
+*.ipr
+*.iws
+*.bak
+*.swp
+*.swo
+*.swn
+*.suo
+*.workspace
+
+# Compile output
+dist/
+build/
+
+# Node.js
+node_modules/
+npm-debug.log
+yarn-error.log
+yarn-debug.log*
+yarn.lock
+
+# Python
+__pycache__/
+*.pyc
+.env/
+.venv/
+venv/
+env/
+
+# Mac
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Windows
+Thumbs.db
+ehthumbs.db
+Desktop.ini
+
+# Keys
+*.key
+*.pem
+*.p12
+*.pfx
+*.crt
+*.csr
+*.cer
+*.jks
+*.pub

cmd/main.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/boxboxjason/gitlab-sync/utils"
+	"github.com/spf13/cobra"
+)
+
+type ParserArgs struct {
+	SourceGitlabURL        string
+	SourceGitlabToken      string
+	DestinationGitlabURL   string
+	DestinationGitlabToken string
+	MirrorMapping          utils.MirrorMapping
+	Verbose                bool
+	NoPrompt               bool
+}
+
+func main() {
+	var args ParserArgs
+	var mirrorMappingPath string
+
+	var rootCmd = &cobra.Command{
+		Use:   "gitlab-sync",
+		Short: "Copy and enable mirroring of gitlab projects and groups",
+		Long:  "Fully customizable gitlab repositories and groups mirroring between two (or one) gitlab instances.",
+		Run: func(cmd *cobra.Command, cmdArgs []string) {
+
+			utils.SetVerbose(args.Verbose)
+			utils.LogVerbose("Verbose mode enabled")
+			utils.LogVerbose("Parsing command line arguments")
+
+			// Check if the source GitLab URL is provided
+			args.SourceGitlabURL = promptForMandatoryInput(args.SourceGitlabURL, "Input Source GitLab URL (MANDATORY)", "Source GitLab URL is mandatory", "Source GitLab URL", args.NoPrompt, false)
+
+			// Check if the destination GitLab URL is provided
+			args.DestinationGitlabURL = promptForMandatoryInput(args.DestinationGitlabURL, "Input Destination GitLab URL (MANDATORY)", "Destination GitLab URL is mandatory", "Destination GitLab URL", args.NoPrompt, false)
+
+			// Check if the Destination GitLab Token is provided
+			args.DestinationGitlabToken = promptForMandatoryInput(args.DestinationGitlabToken, "Input Destination GitLab Token with api permissions (MANDATORY)", "Destination GitLab Token is mandatory", "Destination GitLab Token set", args.NoPrompt, true)
+
+			// Check if the Mirror Mapping file path is provided
+			mirrorMappingPath = promptForMandatoryInput(mirrorMappingPath, "Input Mirror Mapping file path (MANDATORY)", "Mirror Mapping file path is mandatory", "Mirror Mapping file path set", args.NoPrompt, false)
+			utils.LogVerbose("Mirror Mapping file resolved path: " + path.Clean(mirrorMappingPath))
+
+			utils.LogVerbose("Parsing mirror mapping file")
+			mapping, err := utils.OpenMirrorMapping(mirrorMappingPath)
+			if err != nil {
+				log.Fatalf("Error opening mirror mapping file: %v", err)
+			}
+			args.MirrorMapping = *mapping
+			utils.LogVerbose("Mirror mapping file parsed successfully")
+		},
+	}
+
+	rootCmd.Flags().StringVar(&args.SourceGitlabURL, "source-url", os.Getenv("SOURCE_GITLAB_URL"), "Source GitLab URL")
+	rootCmd.Flags().StringVar(&args.SourceGitlabToken, "source-token", os.Getenv("SOURCE_GITLAB_TOKEN"), "Source GitLab Token")
+	rootCmd.Flags().StringVar(&args.DestinationGitlabURL, "destination-url", os.Getenv("DESTINATION_GITLAB_URL"), "Destination GitLab URL")
+	rootCmd.Flags().StringVar(&args.DestinationGitlabToken, "destination-token", os.Getenv("DESTINATION_GITLAB_TOKEN"), "Destination GitLab Token")
+	rootCmd.Flags().BoolVar(&args.Verbose, "verbose", false, "Enable verbose output")
+	rootCmd.Flags().BoolVar(&args.NoPrompt, "no-prompt", false, "Disable prompting for missing values")
+	rootCmd.Flags().StringVar(&mirrorMappingPath, "mirror-mapping", os.Getenv("MIRROR_MAPPING"), "Path to the mirror mapping file")
+
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+}
+
+func promptForInput(prompt string) string {
+	var input string
+	fmt.Printf("%s: ", prompt)
+	fmt.Scanln(&input)
+	return input
+}
+
+func promptForMandatoryInput(defaultValue string, prompt string, errorMsg string, loggerMsg string, promptsEnabled bool, hideOutput bool) string {
+	input := strings.TrimSpace(defaultValue)
+	if input == "" {
+		if promptsEnabled {
+			input = strings.TrimSpace(promptForInput(prompt))
+			if input == "" {
+				log.Fatal(errorMsg)
+			}
+			if !hideOutput {
+				utils.LogVerbose(loggerMsg + ": " + input)
+			} else {
+				utils.LogVerbose(loggerMsg)
+			}
+		} else {
+			log.Fatalf("Prompting is disabled, %s", errorMsg)
+		}
+	}
+	return input
+}

go.mod

@@ -0,0 +1,15 @@
+module github.com/boxboxjason/gitlab-sync
+
+go 1.23.7
+
+require (
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/spf13/cobra v1.9.1 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	gitlab.com/gitlab-org/api/client-go v0.126.0 // indirect
+	golang.org/x/oauth2 v0.25.0 // indirect
+	golang.org/x/time v0.10.0 // indirect
+)

graphql/main.go

@@ -0,0 +1,56 @@
+package graphql
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+)
+
+type GraphQLClient struct {
+	token string
+	URL   string
+}
+
+type GraphQLRequest struct {
+	Query     string `json:"query"`
+	Variables string `json:"variables,omitempty"`
+}
+
+func NewGitlabGraphQLClient(token, gitlabUrl string) *GraphQLClient {
+	return &GraphQLClient{
+		token: token,
+		URL:   strings.TrimSuffix(gitlabUrl, "/") + "/api/graphql",
+	}
+}
+
+func (g *GraphQLClient) SendRequest(request *GraphQLRequest, method string) (string, error) {
+	requestBody, err := json.Marshal(request)
+	if err != nil {
+		return "", err
+	}
+	req, err := http.NewRequestWithContext(context.Background(), method, g.URL, bytes.NewBuffer(requestBody))
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "Bearer "+g.token)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("GraphQL request failed with status: %s", resp.Status)
+	}
+	var responseBody bytes.Buffer
+	if _, err := responseBody.ReadFrom(resp.Body); err != nil {
+		return "", err
+	}
+	return responseBody.String(), nil
+}

mirroring/get.go

@@ -0,0 +1,79 @@
+package mirroring
+
+import (
+	"fmt"
+	"path"
+	"strings"
+
+	"github.com/boxboxjason/gitlab-sync/utils"
+)
+
+func (g *GitlabInstance) fetchProjects(filters *utils.MirrorMapping) error {
+	projects, _, err := g.Gitlab.Projects.ListProjects(nil)
+	if err != nil {
+		return err
+	}
+
+	for _, project := range projects {
+		// Check if the project matches the filters:
+		//   - either is in the projects map
+		//   - or path starts with any of the groups in the groups map
+		if _, ok := filters.Projects[project.PathWithNamespace]; ok {
+			g.addProject(project.PathWithNamespace, project)
+		} else {
+			for group := range filters.Groups {
+				if strings.HasPrefix(project.PathWithNamespace, group) {
+					g.addProject(project.PathWithNamespace, project)
+					break
+				}
+			}
+		}
+	}
+
+	utils.LogVerbosef("Found %d projects to mirror in the source GitLab instance", len(g.Projects))
+
+	return nil
+}
+
+func (g *GitlabInstance) fetchGroups(filters *utils.MirrorMapping) error {
+	groups, _, err := g.Gitlab.Groups.ListGroups(nil)
+	if err != nil {
+		return err
+	}
+
+	for _, group := range groups {
+		// Check if the group matches the filters:
+		//   - either is in the groups map
+		//   - or path starts with any of the groups in the groups map
+		//   - or is a subgroup of any of the groups in the groups map
+		if _, ok := filters.Groups[group.FullPath]; ok {
+			g.addGroup(group.FullPath, group)
+		} else {
+			for groupPath := range filters.Groups {
+				if strings.HasPrefix(group.FullPath, groupPath) {
+					g.addGroup(group.FullPath, group)
+					break
+				}
+			}
+		}
+	}
+
+	utils.LogVerbosef("Found %d groups to mirror in the source GitLab instance", len(g.Groups))
+
+	return nil
+}
+
+func (g *GitlabInstance) getParentGroupID(projectOrGroupPath string) (int, error) {
+	parentGroupID := -1
+	parentPath := path.Dir(projectOrGroupPath)
+	var err error = nil
+	if parentPath != "." {
+		// Check if parent path is already in the instance groups cache
+		if parentGroup, ok := g.Groups[parentPath]; ok {
+			parentGroupID = parentGroup.ID
+		} else {
+			err = fmt.Errorf("parent group not found for path: %s", parentPath)
+		}
+	}
+	return parentGroupID, err
+}

mirroring/main.go

@@ -0,0 +1,69 @@
+package mirroring
+
+import (
+	"sync"
+
+	"github.com/boxboxjason/gitlab-sync/graphql"
+	"github.com/boxboxjason/gitlab-sync/utils"
+	gitlab "gitlab.com/gitlab-org/api/client-go"
+)
+
+type GitlabInstance struct {
+	Gitlab        *gitlab.Client
+	Projects      map[string]*gitlab.Project
+	muProjects    sync.RWMutex
+	Groups        map[string]*gitlab.Group
+	muGroups      sync.RWMutex
+	MirrorMapping *utils.MirrorMapping
+	GraphQLClient *graphql.GraphQLClient
+}
+
+func NewGitlabInstance(gitlabURL, gitlabToken string, mirrorMapping *utils.MirrorMapping) (*GitlabInstance, error) {
+	gitlabClient, err := gitlab.NewClient(gitlabToken, gitlab.WithBaseURL(gitlabURL))
+	if err != nil {
+		return nil, err
+	}
+
+	gitlabInstance := &GitlabInstance{
+		Gitlab:        gitlabClient,
+		Projects:      make(map[string]*gitlab.Project),
+		Groups:        make(map[string]*gitlab.Group),
+		MirrorMapping: mirrorMapping,
+	}
+
+	return gitlabInstance, nil
+}
+
+func (g *GitlabInstance) addProject(projectPath string, project *gitlab.Project) {
+	g.muProjects.Lock()
+	defer g.muProjects.Unlock()
+	g.Projects[projectPath] = project
+}
+
+func (g *GitlabInstance) getProject(projectPath string) *gitlab.Project {
+	g.muProjects.RLock()
+	defer g.muProjects.RUnlock()
+	var project *gitlab.Project
+	project, exists := g.Projects[projectPath]
+	if !exists {
+		project = nil
+	}
+	return project
+}
+
+func (g *GitlabInstance) addGroup(groupPath string, group *gitlab.Group) {
+	g.muGroups.Lock()
+	defer g.muGroups.Unlock()
+	g.Groups[groupPath] = group
+}
+
+func (g *GitlabInstance) getGroup(groupPath string) *gitlab.Group {
+	g.muGroups.RLock()
+	defer g.muGroups.RUnlock()
+	var group *gitlab.Group
+	group, exists := g.Groups[groupPath]
+	if !exists {
+		group = nil
+	}
+	return group
+}