feat(extractor): add an option --after to select only a given time window in fileshas

Author: pierrelalanne

extractor.go

@@ -3,6 +3,7 @@ package srcfingerprint
 import (
 	"bufio"
 	"encoding/json"
+	"fmt"
 	"os"
 	"os/exec"
 
@@ -30,9 +31,16 @@ type FastExtractor struct {
 	ChanGitFiles chan *GitFile
 }
 
-func (fe *FastExtractor) Run(path string) chan *GitFile {
+func (fe *FastExtractor) Run(path string, after string) chan *GitFile {
 	log.Infof("Extracting commits from path %s\n", path)
-	cmdBase := "git rev-list --objects --all | git cat-file --batch-check='{\"sha\": \"%(objectname)\", \"type\": \"%(objecttype)\", \"filepath\": \"%(rest)\", \"size\": \"%(objectsize)\"}' | grep '\"type\": \"blob\"'" //nolint
+
+	cmdRevList := "git rev-list --objects --all"
+
+	if after != "" {
+		cmdRevList = fmt.Sprintf("git rev-list --objects --all --after '%s'", after)
+	}
+
+	cmdBase := cmdRevList + "| git cat-file --batch-check='{\"sha\": \"%(objectname)\", \"type\": \"%(objecttype)\", \"filepath\": \"%(rest)\", \"size\": \"%(objectsize)\"}' | grep '\"type\": \"blob\"'" //nolint
 	cmd := exec.Command("bash", "-c", cmdBase)
 	cmd.Dir = path
 

pipeline.go

@@ -90,7 +90,7 @@ func (p *Pipeline) gather(
 }
 
 // ExtractRepository extracts for a single repository.
-func (p *Pipeline) ExtractRepository(repository provider.GitRepository, eventChan chan<- PipelineEvent) error {
+func (p *Pipeline) ExtractRepository(repository provider.GitRepository, after string, eventChan chan<- PipelineEvent) error { // nolint
 	defer p.publishEvent(eventChan, RepositoryPipelineEvent{true, repository.GetPrivate(), repository.GetName()})
 
 	log.Infof("Cloning repo %v\n", repository.GetName())
@@ -103,7 +103,7 @@ func (p *Pipeline) ExtractRepository(repository provider.GitRepository, eventCha
 	log.Infof("Cloned repo %v (size: %v KB)\n", repository.GetName(), repository.GetStorageSize())
 
 	extractorGitFile := NewFastExtractor()
-	extractorGitFile.Run(gitRepository)
+	extractorGitFile.Run(gitRepository, after)
 
 	for gitFile := range extractorGitFile.ChanGitFiles {
 		p.publishEvent(eventChan, ResultGitFilePipelineEvent{repository, gitFile})
@@ -119,7 +119,7 @@ const (
 )
 
 // ExtractRepositories extract repositories and analyze it for a given user and provider.
-func (p *Pipeline) ExtractRepositories(user string, eventChan chan<- PipelineEvent) {
+func (p *Pipeline) ExtractRepositories(user string, after string, eventChan chan<- PipelineEvent) {
 	log.Infof("Extracting user %v\n", user)
 
 	repositoryChannel := make(chan provider.GitRepository)
@@ -142,7 +142,7 @@ func (p *Pipeline) ExtractRepositories(user string, eventChan chan<- PipelineEve
 			defer wg.Done()
 
 			for repository := range repositoryChannel {
-				if err := p.ExtractRepository(repository, eventChan); err != nil {
+				if err := p.ExtractRepository(repository, after, eventChan); err != nil {
 					log.Errorf("extracting %v failed: %v\n", repository.GetName(), err)
 				}
 			}

pipeline_test.go

@@ -96,7 +96,7 @@ func (suite *PipelineTestSuite) TestExtractGitRepository() {
 	go func() {
 		defer close(eventChan)
 
-		pipeline.ExtractRepository(repository, eventChan)
+		pipeline.ExtractRepository(repository, "", eventChan)
 	}()
 
 	events := make([]PipelineEvent, 0)
@@ -136,7 +136,7 @@ func (suite *PipelineTestSuite) TestExtractRepositories() {
 	go func() {
 		defer close(eventChan)
 
-		pipeline.ExtractRepositories("user", eventChan)
+		pipeline.ExtractRepositories("user", "", eventChan)
 	}()
 
 	events := make([]PipelineEvent, 0)