Add Gitlab Support (#2)

* Add gitlab support

* Add gitlab project selectors

* Add docs

* Clean up

Co-authored-by: Thomas Biesaart <[email protected]>

Author: ChappIO

README.md

@@ -35,6 +35,27 @@ github:
     # organisations of which you are a member.
     # (default: true)
     org_member: true
+# The gitlab section contains backup jobs for
+# GitLab.com and GitLab on premise
+gitlab:
+  # (optional) The job name. This is used to
+  # create a subfolder in the backup folder. 
+  # (default: GitLab)
+  - job_name: gitlab.com
+    # (required) The GitLab access token.
+    # Create one with the scopes: "api"
+    # https://gitlab.com/-/profile/personal_access_tokens?scopes=api&name=git-backup
+    access_token: glpat-6t78yuihy789uy8t768
+    # (optional) Back up repos you own.
+    # (default: true)
+    owned: true
+    # (optional) Back up repos you starred.
+    # (default: true)
+    starred: true
+    # (optional) Back up repos owned by 
+    # teams of which you are a member.
+    # (default: true)
+    member: true
 ```
 
 ## Usage
@@ -47,4 +68,4 @@ Options:
         The target path to the backup folder. (default "backup")
   -config.file string
         The path to your config file. (default "git-backup.yml")
-```
+```

cmd/git-backup/main.go

@@ -55,4 +55,4 @@ func loadConfig() gitbackup.Config {
 		os.Exit(1)
 	}
 	return config
-}
+}

config.go

@@ -8,14 +8,19 @@ import (
 
 type Config struct {
 	Github []*GithubConfig `yaml:"github"`
+	GitLab []*GitLabConfig `yaml:"gitlab"`
 }
 
 func (c *Config) GetSources() []RepositorySource {
-	sources := make([]RepositorySource, len(c.Github))
+	sources := make([]RepositorySource, len(c.Github)+len(c.GitLab))
 
 	offset := 0
 	for i := 0; i < len(c.Github); i++ {
-		sources[i+offset] = c.Github[i]
+		sources[offset] = c.Github[i]
+		offset++
+	}
+	for i := 0; i < len(c.GitLab); i++ {
+		sources[offset] = c.GitLab[i]
 		offset++
 	}
 
@@ -28,6 +33,11 @@ func (c *Config) setDefaults() {
 			config.setDefaults()
 		}
 	}
+	if c.GitLab != nil {
+		for _, config := range c.GitLab {
+			config.setDefaults()
+		}
+	}
 }
 
 func LoadFile(path string) (out Config, err error) {

github.go

@@ -10,10 +10,6 @@ import (
 	"strings"
 )
 
-func boolPointer(b bool) *bool {
-	return &b
-}
-
 type GithubConfig struct {
 	JobName      string `yaml:"job_name"`
 	AccessToken  string `yaml:"access_token"`
@@ -167,4 +163,4 @@ func (c *GithubConfig) getStarredRepos(page int) ([]*github.Repository, *github.
 		repos[i] = starred[i].Repository
 	}
 	return repos, response, err
-}
+}

gitlab.go

@@ -0,0 +1,127 @@
+package git_backup
+
+import (
+	"github.com/xanzy/go-gitlab"
+	"log"
+	"net/url"
+)
+
+type GitLabConfig struct {
+	URL         string `yaml:"url,omitempty"`
+	JobName     string `yaml:"job_name"`
+	AccessToken string `yaml:"access_token"`
+	Starred     *bool  `yaml:"starred,omitempty"`
+	Member      *bool  `yaml:"member,omitempty"`
+	Owned       *bool  `yaml:"owned,omitempty"`
+	client      *gitlab.Client
+}
+
+func (g *GitLabConfig) GetName() string {
+	return g.JobName
+}
+
+func (g *GitLabConfig) Test() error {
+	user, _, err := g.client.Users.CurrentUser()
+	if err != nil {
+		return err
+	}
+	log.Printf("Authenticated as with gitlab as: %s", user.Username)
+	return nil
+}
+
+func (g *GitLabConfig) ListRepositories() ([]*Repository, error) {
+	out := make(map[string]*Repository, 0)
+
+	if *g.Starred {
+		if repos, err := g.getAllRepos(&gitlab.ListProjectsOptions{Starred: boolPointer(true)}); err != nil {
+			return nil, err
+		} else {
+			for _, repo := range repos {
+				out[repo.FullName] = repo
+			}
+		}
+	}
+
+	if *g.Owned {
+		if repos, err := g.getAllRepos(&gitlab.ListProjectsOptions{Owned: boolPointer(true)}); err != nil {
+			return nil, err
+		} else {
+			for _, repo := range repos {
+				out[repo.FullName] = repo
+			}
+		}
+	}
+
+	if *g.Member {
+		if repos, err := g.getAllRepos(&gitlab.ListProjectsOptions{Membership: boolPointer(true)}); err != nil {
+			return nil, err
+		} else {
+			for _, repo := range repos {
+				out[repo.FullName] = repo
+			}
+		}
+	}
+
+	outSlice := make([]*Repository, 0, len(out))
+	for _, repository := range out {
+		outSlice = append(outSlice, repository)
+	}
+
+	return outSlice, nil
+}
+
+func (g *GitLabConfig) getAllRepos(opts *gitlab.ListProjectsOptions) ([]*Repository, error) {
+	out := make([]*Repository, 0)
+	for i := 1; true; i++ {
+		opts.ListOptions.Page = i
+		repos, _, err := g.getRepos(opts)
+		if err != nil {
+			return out, err
+		}
+		for _, repo := range repos {
+			gitUrl, err := url.Parse(repo.HTTPURLToRepo)
+			if err != nil {
+				return out, err
+			}
+			gitUrl.User = url.UserPassword("git", g.AccessToken)
+			out = append(out, &Repository{
+				GitURL:   *gitUrl,
+				FullName: repo.PathWithNamespace,
+			})
+		}
+		if len(repos) == 0 {
+			break
+		}
+	}
+	return out, nil
+}
+
+func (g *GitLabConfig) getRepos(opts *gitlab.ListProjectsOptions) ([]*gitlab.Project, *gitlab.Response, error) {
+	opts.ListOptions.PerPage = 100
+	opts.Simple = boolPointer(true)
+	return g.client.Projects.ListProjects(opts)
+}
+
+func (g *GitLabConfig) setDefaults() {
+	if g.Member == nil {
+		g.Member = boolPointer(true)
+	}
+	if g.Owned == nil {
+		g.Owned = boolPointer(true)
+	}
+	if g.Starred == nil {
+		g.Starred = boolPointer(true)
+	}
+	if g.JobName == "" {
+		g.JobName = "GitLab"
+	}
+	if g.URL == "" {
+		g.client, _ = gitlab.NewClient(g.AccessToken)
+	} else {
+		client, err := gitlab.NewClient(g.AccessToken, gitlab.WithBaseURL(g.URL))
+		if err != nil {
+			panic(err)
+		}
+		g.client = client
+	}
+}

go.mod

@@ -3,21 +3,9 @@ module git-backup
 go 1.17
 
 require (
-	github.com/ChappIO/terraform-encrypt v0.0.0-20190529081821-8d96f6eb70d9
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/google/go-github/v43 v43.0.0
-	github.com/gorilla/pat v1.0.1
-	github.com/ian-kent/envconf v0.0.0-20141026121121-c19809918c02
-	github.com/ian-kent/go-log v0.0.0-20160113211217-5731446c36ab
-	github.com/influxdata/influxdb v1.9.6
-	github.com/karalabe/hid v1.0.0
-	github.com/mailhog/MailHog v1.0.1
-	github.com/mailhog/MailHog-Server v1.0.1
-	github.com/mailhog/MailHog-UI v1.0.1
-	github.com/mailhog/http v1.0.1
-	github.com/mailhog/mhsendmail v0.2.0
-	github.com/spf13/cobra v1.4.0
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
+	github.com/xanzy/go-gitlab v0.60.0
 	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -26,38 +14,24 @@ require (
 	github.com/Microsoft/go-winio v0.4.16 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
-	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.3.1 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/gorilla/context v1.1.1 // indirect
-	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
-	github.com/ian-kent/goose v0.0.0-20141221090059-c3541ea826ad // indirect
-	github.com/ian-kent/linkio v0.0.0-20170807205755-97566b872887 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
-	github.com/mailhog/data v1.0.1 // indirect
-	github.com/mailhog/smtp v1.0.1 // indirect
-	github.com/mailhog/storage v1.0.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/ogier/pflag v0.0.1 // indirect
-	github.com/philhofer/fwd v1.0.0 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
-	github.com/smartystreets/goconvey v1.7.2 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/t-k/fluent-logger-golang v1.0.0 // indirect
-	github.com/tinylib/msgp v1.1.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
 	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
 	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect
-	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 )