feat: add git files shas collection capabilities

Author: ericfourrier

README.md

@@ -8,7 +8,7 @@ It supports 3 main on premise version control service:
 
 * GitHub Enterprise
 * Gitlab CE and EE
-* BitBucket
+* BitBucket (not supported yet)
 
 ## Use the package
 
@@ -31,6 +31,12 @@ It supports 3 main on premise version control service:
     ./dna-collector -provider-url http://gitlab.example.com gitlab Groupe
     ```
 
+## Some examples 
+
+* Don't forget to build the package:  `go build ./cmd/dna-collector`
+
+1. Export all files sha from a GitHub Org to a file with logs: `./dna-collector -verbose -output file_shas_collected_dna.json github GitGuardian`
+2. Name params should be passed before positional parameters.
 ## Architecture
 
 ### Main overview

cmd/dna-collector/main.go

@@ -41,6 +41,12 @@ type authorInfo struct {
 	LastCommitDate time.Time
 }
 
+type AugmentedGitFile struct {
+
+	RepositoryName string `json:"repository_name"`
+	dnacollector.GitFile
+}
+
 func main() {
 	var (
 		verbose        = flag.Bool("verbose", false, "set to add verbose logging")
@@ -123,9 +129,11 @@ func main() {
 		doneRepo     int
 		authors      map[string]*authorInfo
 		commitsCount int
+		gitFilesCount int
 	)
 
 	authors = make(map[string]*authorInfo)
+	gitFilesArray := make([]*AugmentedGitFile, 0)
 loop:
 	for {
 		select {
@@ -141,7 +149,7 @@ loop:
 				if typedEvent.Finished {
 					doneRepo++
 				}
-			case dnacollector.ResultPipelineEvent:
+			case dnacollector.ResultCommitPipelineEvent:
 				commitsCount++
 
 				identity := typedEvent.Author.Name + typedEvent.Author.Email
@@ -155,6 +163,11 @@ loop:
 				if commit.Author.When.UTC().After(authors[identity].LastCommitDate) {
 					authors[identity].LastCommitDate = commit.Author.When.UTC()
 				}
+			// Collecting gitFiles
+			case dnacollector.ResultGitFilePipelineEvent:
+				gitFilesCount++
+				gitFilesArray = append(gitFilesArray, &AugmentedGitFile{typedEvent.Repository.GetName(), *typedEvent.GitFile})
+
 			}
 		case <-ticker:
 			if totalRepo == 0 {
@@ -163,28 +176,29 @@ loop:
 
 			log.Infof("%v/%v repos: ", doneRepo, totalRepo)
 			log.Infof("%v distinct authors, %v commit analyzed\n", len(authors), commitsCount)
+			log.Infof("%v files analyzed\n", gitFilesCount)
 		}
 	}
 
 	log.Infoln("Final stats:")
 	log.Infof("%v/%v repos: ", doneRepo, totalRepo)
 	log.Infof("%v distinct authors, %v commit analyzed\n", len(authors), commitsCount)
-
+	log.Infof("%v git files analyzed\n", gitFilesCount)
 	log.Infof("Dumping to output %v", *outputFilename)
 
-	authorsList := make([]*authorInfo, 0, len(authors))
-	for _, author := range authors {
-		authorsList = append(authorsList, author)
-	}
+	//authorsList := make([]*authorInfo, 0, len(authors))
+	//for _, author := range authors {
+	//	authorsList = append(authorsList, author)
+	//}
 
 	var (
 		jsonBytes []byte
 		err       error
 	)
 	if *prettyPrint {
-		jsonBytes, err = json.MarshalIndent(authorsList, "", "\t")
+		jsonBytes, err = json.MarshalIndent(gitFilesArray, "", "\t")
 	} else {
-		jsonBytes, err = json.Marshal(authorsList)
+		jsonBytes, err = json.Marshal(gitFilesArray)
 	}
 
 	if err != nil {

cmd/test2/main.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"dnacollector"
+	"github.com/caarlos0/env"
+	log "github.com/sirupsen/logrus"
+	git2 "gopkg.in/src-d/go-git.v4"
+)
+
+type config struct {
+	GithubToken string `env:"GITHUB_TOKEN"`
+	GitlabToken string `env:"GITLAB_TOKEN"`
+}
+
+func main() {
+	conf := config{}
+	if err := env.Parse(&conf); err != nil {
+		log.Fatalf("Could not parse env: %v\n", err)
+	}
+
+	// Config log
+	log.SetFormatter(&log.TextFormatter{
+		DisableColors: true,
+		FullTimestamp: true,
+		TimestampFormat: "Jan _2 15:04:05.000000000",
+	})
+	log.SetReportCaller(true)
+	log.SetLevel(log.InfoLevel)
+	//var cloner dnacollector.Cloner = &dnacollector.DiskCloner{}
+	//auth := &http.BasicAuth{
+	//	Username: "ericfourrier",
+	//	Password: conf.GithubToken,
+	//}
+
+	//repository, err := cloner.CloneRepository("https://github.com/uber/cadence.git", auth)
+	//if err != nil {
+	//	log.Panic(err)
+	//
+	//}
+	repository, err := git2.PlainOpen("/Users/ericfourrier/Documents/GGCode/dna-collector/testdata/cadence")
+	if err != nil {
+		log.Panic(err)
+	}
+	extractor := dnacollector.NewFastExtractor()
+	extractor.Run(repository)
+	arrGitFiles := make([]*dnacollector.GitFile, 0)
+	for gitFile := range extractor.ChanGitFiles {
+		arrGitFiles = append(arrGitFiles, gitFile)
+		log.Debug(gitFile)
+
+	}
+	log.Infof("length of files collected %d", len(arrGitFiles))
+}

extractor.go

@@ -1,8 +1,16 @@
 package dnacollector
 
 import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/src-d/go-billy.v4/helper/chroot"
 	git "gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
+	"gopkg.in/src-d/go-git.v4/storage/filesystem"
+	"os"
+	"os/exec"
 )
 
 // Extractor extracts commits for a given repository
@@ -26,3 +34,115 @@ func NewExtractor(repository *git.Repository) (*Extractor, error) {
 func (e *Extractor) ExtractNextCommit() (*object.Commit, error) {
 	return e.iter.Next()
 }
+
+// FastExtractor will directly extract the information without using an Analyzer
+// There are designed to use raw git commands to get what is needed
+
+func GetBasePathGoGitRepo(r *git.Repository) (string, error) {
+	//
+	// Try to grab the repository Storer
+	s, ok := r.Storer.(*filesystem.Storage)
+	if !ok {
+		return "", errors.New("Repository storage is not filesystem.Storage")
+	}
+
+	// Try to get the underlying billy.Filesystem
+	fs, ok := s.Filesystem().(*chroot.ChrootHelper)
+	if !ok {
+		return "", errors.New("Filesystem is not chroot.ChrootHelper")
+	}
+
+	return fs.Root(), nil
+}
+
+//type BaseGitFileIterator struct {
+//	repository *git.Repository
+//	filesChan  chan *GitFile
+//}
+//
+//func (i *BaseGitFileIterator) Compute(repository *git.Repositor) error {
+//
+//}
+//
+//func (i *BaseGitFileIterator) Close() error {
+//
+//}
+
+type BaseExtractor interface {
+	Next() (interface{}, error)
+}
+
+
+
+type GitFile struct {
+	Sha      string `json:"sha"`
+	Type     string `json:"type"`
+	Filepath string `json:"filepath"`
+	Size     string `json:"size"`
+}
+
+
+func NewFastExtractor() *FastExtractor {
+	return &FastExtractor{make(chan *GitFile)}
+}
+type FastExtractor struct{
+	ChanGitFiles chan *GitFile
+}
+
+func (fe *FastExtractor) Run(repository *git.Repository) chan *GitFile{
+	// https://gist.github.com/ochinchina/9e409a88e77c3cfd94c3
+	path, err := GetBasePathGoGitRepo(repository)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = os.Chdir(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Infof("Extracting commits from path %s", path)
+	cmdBase := "git rev-list --objects --all | git cat-file --batch-check='{\"sha\": \"%(objectname)\", \"type\": \"%(objecttype)\", \"filepath\": \"%(rest)\", \"size\": \"%(objectsize:disk)\"}' | grep '\"type\": \"blob\"'"
+	cmd := exec.Command("bash", "-c", cmdBase)
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = cmd.Start()
+	if err != nil {
+		log.Fatal(err)
+	}
+	buf := bufio.NewReader(stdout) // Notice that this is not in a loop
+	num := 0
+	go func() {
+		for {
+			line, _, _ := buf.ReadLine()
+			if len(line) == 0 {
+				log.Info("finish reading all files from stdout from git")
+				break
+			}
+			num += 1
+			log.Debugf("parsing line %s", line)
+			var gitFile GitFile
+			json.Unmarshal(line, &gitFile)
+			fe.ChanGitFiles <- &gitFile
+
+		}
+
+		close(fe.ChanGitFiles)
+		log.Info("channel is closed")
+		log.Infof("finishing iterating over files, we have collected %d files", num)
+
+	}()
+	return fe.ChanGitFiles
+}
+
+
+
+//targetFile := fe.BaseDirStorage + "/" + "sha_files.jsonl"
+//log.Infof("Saving to file %s", targetFile)
+//dataFile, err := os.Create(targetFile)
+//if err != nil {
+//	log.Error(err)
+//}
+//defer dataFile.Close()
+//_, err = dataFile.WriteString(out.String())
+//return err