feat: first commit

Author: ericfourrier

.env.example

@@ -0,0 +1 @@
+GITHUB_TOKEN=------FILL ME-------

.gometalinter.json

@@ -0,0 +1,3 @@
+{
+        "Enable": ["goimports","gosec", "staticcheck"]
+}

README.md

@@ -0,0 +1,72 @@
+# git-users-extractor
+
+## Introduction
+
+The purpose of this package is to extract all git user information of your developers from your hosted source version control system
+
+It supports 3 main on premise version control service:
+
+* GitHub Enterprise
+* Gitlab CE and EE
+* BitBucket
+
+## Use the package
+
+* Build binary
+    ```sh
+    go build ./cmd/dna-collector
+    ```
+* Set env var `GITHUB_TOKEN` or `GITLAB_TOKEN`
+    ```sh
+    export GITHUB_TOKEN="<token>"
+    export GITLAB_TOKEN="<token>"
+    ```
+* Run and read doc
+    ```sh
+    ./dna-collector -help
+    ```
+* Run on a given user/group
+    ```sh
+    ./dna-collector github Uber
+    ./dna-collector -provider-url http://gitlab.example.com gitlab Groupe
+    ```
+
+## Architecture
+
+### Main overview
+All the git information can be found inside commit that are located inside git repositories
+Our tree element step are the following:
+* Collect all repositories URL from the company.
+* Clone them with the appropriate authentication.
+* Clone them and iterate over commits to extract git config information.
+* Store this information in a json file.
+
+### Implementation
+The root package is the abstract implementation of the extractor. It contains a Cloner, that clones a git repository.
+It contains a Pipeline that extracts git information for every commit, of every repository of an organization.
+
+The github package contains the implementation of the Github Provider.
+The gitlab package contains the implementation of the Gitlab Provider.
+
+The cmd/guser-extractor package contains the binary code. It reads from CLI and environment the configuration and run the Pipeline on an organization.
+
+### Library we use
+
+#### Providers
+* GitHub: "github.com/google/go-github/v18/github"
+* Gitlab go wrapper: "github.com/xanzy/go-gitlab"
+* bitbucket not supported yet
+
+#### Cloning
+* go-git: https://github.com/src-d/go-git
+
+
+### Issues
+* Repo size seems not to work on go gitlab wrapper.
+* Channels are cheap. Complex design overloading semantics isn't.
+
+
+### Notes eric 
+
+* We want to add more extractors -> We need to extract file shas
+* We will add more analyzer -> From the commits we want to get commits shas

analyzer.go

@@ -0,0 +1,14 @@
+package dnacollector
+
+import (
+	git "gopkg.in/src-d/go-git.v4/plumbing/object"
+)
+
+// Analyzer analyzer a commit to extract its author
+type Analyzer struct {
+}
+
+// AnalyzeCommit extracts author and commiter from a commit
+func (a *Analyzer) AnalyzeCommit(commit *git.Commit) (author git.Signature, commiter git.Signature) {
+	return commit.Author, commit.Committer
+}

analyzer_test.go

@@ -0,0 +1,37 @@
+package dnacollector_test
+
+import (
+	"dnacollector"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+	"gopkg.in/src-d/go-git.v4/plumbing/object"
+)
+
+type AnalyzerTestSuite struct {
+	suite.Suite
+}
+
+func (suite *AnalyzerTestSuite) TestAnalyzeCommit() {
+	analyzer := dnacollector.Analyzer{}
+	commit := &object.Commit{
+		Author: object.Signature{
+			Name:  "Author",
+			Email: "[email protected]",
+		},
+		Committer: object.Signature{
+			Name:  "Committer",
+			Email: "[email protected]",
+		},
+	}
+
+	author, committer := analyzer.AnalyzeCommit(commit)
+
+	assert.Equal(suite.T(), object.Signature{Name: "Author", Email: "[email protected]"}, author)
+	assert.Equal(suite.T(), object.Signature{Name: "Committer", Email: "[email protected]"}, committer)
+}
+
+func TestAnalyzer(t *testing.T) {
+	suite.Run(t, new(AnalyzerTestSuite))
+}

cloner.go

@@ -0,0 +1,46 @@
+package dnacollector
+
+import (
+	"io/ioutil"
+
+	"gopkg.in/src-d/go-billy.v4/osfs"
+	git "gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
+	"gopkg.in/src-d/go-git.v4/plumbing/transport"
+	"gopkg.in/src-d/go-git.v4/storage/filesystem"
+	"gopkg.in/src-d/go-git.v4/storage/memory"
+)
+
+// Cloner represents a cloner of git repository
+type Cloner interface {
+	CloneRepository(url string, auth transport.AuthMethod) (*git.Repository, error)
+}
+
+// MemoryCloner clones a git repository in memory
+type MemoryCloner struct{}
+
+// CloneRepository clones a git repository given its information
+func (*MemoryCloner) CloneRepository(url string, auth transport.AuthMethod) (*git.Repository, error) {
+	return git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
+		URL:      url,
+		Progress: ioutil.Discard,
+		Auth:     auth,
+	})
+}
+
+// DiskCloner closes a git repository on disk in a temporary file
+type DiskCloner struct{}
+
+// CloneRepository clones a git repository given its information
+func (*DiskCloner) CloneRepository(url string, auth transport.AuthMethod) (*git.Repository, error) {
+	tmpDir, err := ioutil.TempDir("", "fs-")
+	if err != nil {
+		return nil, err
+	}
+	fs := osfs.New(tmpDir)
+	return git.Clone(filesystem.NewStorage(fs, cache.NewObjectLRUDefault()), nil, &git.CloneOptions{
+		URL:      url,
+		Progress: ioutil.Discard,
+		Auth:     auth,
+	})
+}

cmd/dna-collector-test/main.go

@@ -0,0 +1,204 @@
+package main
+
+import (
+	"dnacollector"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/caarlos0/env"
+	log "github.com/sirupsen/logrus"
+	git2 "gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/plumbing/format/diff"
+	git "gopkg.in/src-d/go-git.v4/plumbing/object"
+	"io"
+)
+
+type config struct {
+	GithubToken string `env:"GITHUB_TOKEN"`
+	GitlabToken string `env:"GITLAB_TOKEN"`
+}
+
+var (
+	// ErrGroupNotFound is the error returned when group can not be found
+	ErrFileSimplifiedCreation = errors.New("we could not instantiate GitFileSimplified from GitFile")
+)
+
+type GitFileSimplified struct {
+	Name     string `json:"name"`
+	Sha      string `json:"sha"`
+	IsBinary bool   `json:"is_binary"`
+	Size     int64  `json:"size"`
+}
+
+type CommitSimplified struct {
+	Message   string               `json:"message"`
+	Sha       string               `json:"sha"`
+	Author    git.Signature        `json:"author"`
+	Committer git.Signature        `json:"committer"`
+	Files     []*GitFileSimplified `json:"files"`
+}
+
+func NewFromGitFile(file *git.File) *GitFileSimplified {
+	isBinary, _ := file.IsBinary()
+	return &GitFileSimplified{Name: file.Name, Sha: file.Hash.String(), IsBinary: isBinary, Size: file.Size}
+}
+
+func NewFromCommit(commit *git.Commit, files []*GitFileSimplified) *CommitSimplified {
+	return &CommitSimplified{Message: commit.Message, Sha: commit.Hash.String(), Author: commit.Author, Committer: commit.Committer, Files: files}
+}
+
+func NewFromFilePatch(filePatch diff.FilePatch) (*GitFileSimplified, error) {
+	isBinary := filePatch.IsBinary()
+	from, to := filePatch.Files()
+	// If the patch creates a new file, "from" will be nil.
+	// If the patch deletes a file, "to" will be nil.
+
+	// Rare usecase
+	if to == nil && from == nil {
+		return nil, ErrFileSimplifiedCreation
+	} else if to != nil {
+		// File creation
+		return &GitFileSimplified{Name: to.Path(), Sha: to.Hash().String(), IsBinary: isBinary, Size: 0}, nil
+	} else {
+		// File deletion
+		return &GitFileSimplified{Name: from.Path(), Sha: from.Hash().String(), IsBinary: isBinary, Size: 0}, nil
+	}
+}
+
+func NewAnalyzer() *Analyzer {
+	return &Analyzer{make([]*CommitSimplified, 0)}
+}
+
+type Analyzer struct {
+	CommitsList []*CommitSimplified
+}
+
+func (a *Analyzer) GetFilesFromCommit(commit *git.Commit) ([]*GitFileSimplified, error) {
+	var files []*GitFileSimplified
+
+	parent, err := commit.Parent(0)
+
+	// There is no parent, so we take all the files
+	if err != nil {
+		filesIter, err := commit.Files()
+		if err != nil {
+			return nil, err
+		}
+
+		filesIter.ForEach(func(file *git.File) error {
+			fileSimplified := NewFromGitFile(file)
+			log.Debugf("Appending file %s", fileSimplified.Name)
+			if fileSimplified.Size > 0 {
+				/*		fileSimplifiedJson, _ := json.Marshal(fileSimplified)*/
+				log.Info(fileSimplified)
+			}
+
+			files = append(files, fileSimplified)
+			return nil
+		})
+		// There is a parent, so we consider only the diff
+	} else {
+		patch, _ := commit.Patch(parent)
+		filePatches := patch.FilePatches()
+		//log.Info(patch.Stats())
+		for _, fp := range filePatches {
+			fileSimplified, err := NewFromFilePatch(fp)
+			//for _, chunk := range fp.Chunks() {
+			//	log.Debug(chunk)
+			//}
+			log.Debugf("Appending file %s", fileSimplified.Name)
+			if err != nil {
+				log.Warn(fileSimplified)
+				files = append(files, fileSimplified)
+			} else {
+				continue
+				//log.Error(ErrFileSimplifiedCreation)
+				//log.Warn(commit)
+				//log.Warn(fp)
+
+			}
+		}
+	}
+
+	return files, nil
+}
+
+// AnalyzeCommit extracts author and committer from a commit
+func (a *Analyzer) AnalyzeCommit(commit *git.Commit) string {
+    // Store commmit sha
+	files, _ := a.GetFilesFromCommit(commit)
+	a.CommitsList = append(a.CommitsList, NewFromCommit(commit, files))
+	return commit.Hash.String()
+}
+
+func (a *Analyzer) GetStats() map[string]int {
+	res := make(map[string]int)
+	res["nb_commits"] = len(a.CommitsList)
+	nb_files_shas := 0
+	for _, commit := range a.CommitsList {
+		nb_files_shas += len(commit.Files)
+	}
+	res["nb_files_shas"] = nb_files_shas
+	return res
+}
+
+//func (a *Analyzer) GetCommitShasArr() []string {
+//	var res []string
+//	for k := range a.SetCommitsSha {
+//		res = append(res, k)
+//	}
+//	return res
+//}
+
+func main() {
+	conf := config{}
+
+	// Config log
+	log.SetFormatter(&log.TextFormatter{
+		DisableColors: true,
+		FullTimestamp: true,
+	})
+	log.SetReportCaller(true)
+	log.SetLevel(log.InfoLevel)
+
+	if err := env.Parse(&conf); err != nil {
+		log.Fatalf("Could not parse env: %v\n", err)
+	}
+	log.Debug(conf)
+	//var cloner dnacollector.Cloner = &dnacollector.MemoryCloner{}
+	//auth := &http.BasicAuth{
+	//	Username: "ericfourrier",
+	//	Password: conf.GithubToken,
+	//}
+
+	repository, err := git2.PlainOpen("/Users/ericfourrier/Documents/GGCode/dna-collector/testdata/react-vis")
+	if err != nil {
+		fmt.Print(err)
+	}
+	repository.Config()
+	//log.Infof("Cloned repo %v (size: %v)\n", repository.n, repository.GetStorageSize())
+	extractor, err := dnacollector.NewExtractor(repository)
+	analyzer := NewAnalyzer()
+	for {
+		commit, err := extractor.ExtractNextCommit()
+		if err != nil && err != io.EOF {
+			log.Panic(err)
+		}
+		if commit == nil {
+			break
+		}
+
+		analyzer.AnalyzeCommit(commit)
+	}
+	res2, _ := json.Marshal(analyzer.CommitsList)
+	log.Debug(string(res2))
+	//fmt.Print(analyzer.SetCommitsSha)
+	//for _, files := range analyzer.CommitTable {
+	//	for _, file := range files {
+	//		log.Info(file.Sha)
+	//	}
+	//}
+	log.Info(analyzer.GetStats())
+	log.Infof("Done extracting %v\n", repository)
+
+}