feat(collect): Add timeout per object (-u) to the collect command

Author: Walz

.github/workflows/test.yml

@@ -9,12 +9,15 @@ jobs:
     name: Lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.16
+      - uses: actions/checkout@v3
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: v1.43
+          version: v1.45
   build:
     name: Build & Test
     runs-on: ubuntu-latest
@@ -77,6 +80,7 @@ jobs:
           GH_INTEGRATION_TESTS_TOKEN: ${{ secrets.GH_INTEGRATION_TESTS_TOKEN }}
           BITBUCKET_INTEGRATION_TESTS_TOKEN: ${{ secrets.BITBUCKET_INTEGRATION_TESTS_TOKEN }}
           BITBUCKET_INTEGRATION_TESTS_URL: ${{ secrets.BITBUCKET_INTEGRATION_TESTS_URL }}
+          REPOSITORY_TRIGGERING_TIMEOUT: ${{ secrets.REPOSITORY_TRIGGERING_TIMEOUT }}
         run: |
           python3 -m pip install pytest
           pytest tests

README.md

@@ -103,7 +103,8 @@ The default format is `gzip-jsonl` to minimize the size of the output file.
 The default output filepath is `./fingerprints.jsonl.gz`. Use `--output` to override this behavior.  
 Also, note that if you were to download fingerprints for repositories of a big organization, `src-fingerprint` has a limit to process no more than 100
 repositories. You can override this limit with the option `--limit`, a limit of 0 will process all repos of the organization.
-Note that if multiple organizations are passed, the limit is applied to each one independently.
+Note that if multiple organizations are passed, the limit is applied to each one independently.  
+There is no default timeout, it can be set with the option `--timeout`. Similarly to the limit, it is applied to each source independently.
 
 ### Sample output
 

cloner/clone_repository.go

@@ -0,0 +1,91 @@
+package cloner
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/shirou/gopsutil/v3/process"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func terminateProcessAndChildren(pid int32) error {
+	var (
+		proc     *process.Process
+		children []*process.Process
+		err      error
+	)
+
+	proc, err = process.NewProcess(pid)
+	if err != nil {
+		return err
+	}
+
+	children, err = proc.Children()
+	if err != nil {
+		log.WithError(err).Warnf("could not get children of pid#%v", pid)
+
+		children = []*process.Process{}
+	}
+
+	err = proc.Terminate()
+	if err != nil {
+		return err
+	}
+
+	for _, child := range children {
+		err = terminateProcessAndChildren(child.Pid)
+		if err != nil {
+			log.WithError(err).Warnf("could not terminate pid#%v", pid)
+		}
+	}
+
+	return nil
+}
+
+func cloneGitRepository(ctx context.Context, destDir, gitRepoURL string) error {
+	var outbuf, errbuf bytes.Buffer
+	// git clone github.com/author/name.git /tmp/workdir/author-name/clone
+	cmd := exec.Command("git", "clone", gitRepoURL, destDir)
+	cmd.Stdout = &outbuf
+	cmd.Stderr = &errbuf
+
+	cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
+
+	go func() {
+		<-ctx.Done()
+
+		if cmd.Process != nil && (cmd.ProcessState == nil || !cmd.ProcessState.Exited()) {
+			if err := terminateProcessAndChildren(int32(cmd.Process.Pid)); err != nil {
+				log.WithError(err).Error("Could not terminate git")
+			}
+		}
+	}()
+
+	err := cmd.Run()
+	if exitError, ok := err.(*exec.ExitError); ok {
+		stderr := strings.TrimSpace(errbuf.String())
+
+		if exitError.ExitCode() == gitExitUnclean {
+			log.WithError(err).WithFields(log.Fields{
+				"op":     "gitError",
+				"stderr": stderr,
+			}).Warnf("missing repo")
+		} else if ctx.Err() != nil {
+			log.Errorf("timeout reached while cloning the repository")
+		} else {
+			log.WithError(err).WithFields(log.Fields{
+				"op":     "gitError",
+				"stderr": stderr,
+			}).Errorf("unhandled git error")
+		}
+
+		return errors.New("")
+	}
+
+	return err
+}

cloner/cloner.go

@@ -1,12 +1,9 @@
 package cloner
 
 import (
-	"bytes"
-	"errors"
+	"context"
 	"os"
-	"os/exec"
 	"path/filepath"
-	"strings"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -15,7 +12,7 @@ const gitExitUnclean = 128
 
 // Cloner represents a cloner of git repository.
 type Cloner interface {
-	CloneRepository(url string) (string, error)
+	CloneRepository(ctx context.Context, url string) (string, error)
 }
 
 // DiskCloner closes a git repository on disk in a temporary file.
@@ -52,48 +49,17 @@ func NewDiskCloner(baseDir string) *DiskCloner {
 }
 
 // CloneRepository clones a git repository given its information.
-func (d *DiskCloner) CloneRepository(url string) (string, error) {
+func (d *DiskCloner) CloneRepository(ctx context.Context, url string) (string, error) {
 	tmpDir, err := os.MkdirTemp(d.BaseDir, "srcfingerprint-")
 	if err != nil {
 		return "", err
 	}
 
-	if err := cloneGitRepository(tmpDir, url); err != nil {
+	if err := cloneGitRepository(ctx, tmpDir, url); err != nil {
 		os.RemoveAll(tmpDir)
 
 		return "", err
 	}
 
 	return tmpDir, nil
 }
-
-func cloneGitRepository(destDir, gitRepoURL string) error {
-	var outbuf, errbuf bytes.Buffer
-	// git clone github.com/author/name.git /tmp/workdir/author-name/clone
-	cmd := exec.Command("git", "clone", gitRepoURL, destDir)
-	cmd.Stdout = &outbuf
-	cmd.Stderr = &errbuf
-
-	cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
-
-	err := cmd.Run()
-	if exitError, ok := err.(*exec.ExitError); ok {
-		stderr := strings.TrimSpace(errbuf.String())
-
-		if exitError.ExitCode() == gitExitUnclean {
-			log.WithError(err).WithFields(log.Fields{
-				"op":     "gitError",
-				"stderr": stderr,
-			}).Warnf("missing repo")
-		} else {
-			log.WithError(err).WithFields(log.Fields{
-				"op":     "gitError",
-				"stderr": stderr,
-			}).Errorf("unhandled git error")
-		}
-
-		return errors.New("")
-	}
-
-	return err
-}

cmd/src-fingerprint/main.go

@@ -25,7 +25,8 @@ func runExtract(
 	pipeline *srcfingerprint.Pipeline,
 	objects []string,
 	after string,
-	limit int) chan srcfingerprint.PipelineEvent {
+	limit int,
+	timeout time.Duration) chan srcfingerprint.PipelineEvent {
 	// If there is no object, default to an empty object
 	if len(objects) == 0 {
 		objects = []string{""}
@@ -38,7 +39,7 @@ func runExtract(
 		defer close(eventChannel)
 
 		for _, object := range objects {
-			pipeline.ExtractRepositories(object, after, eventChannel, limit)
+			pipeline.ExtractRepositories(object, after, eventChannel, limit, timeout)
 		}
 	}(ch)
 
@@ -84,6 +85,7 @@ type authorInfo struct {
 
 const DefaultClonerN = 8
 const DefaultLimit = 100
+const DefaultTimeout = 0
 
 func main() {
 	cli.VersionFlag = &cli.BoolFlag{
@@ -203,6 +205,11 @@ func main() {
 						Usage: "Maximum number of repositories to analyze (0 for unlimited). " +
 							"The limit is applied for independently to each object.",
 					},
+					&cli.DurationFlag{
+						Name:  "timeout",
+						Value: DefaultTimeout,
+						Usage: "Maximum time to process each object (0 for unlimited, min. 1s).",
+					},
 				},
 			},
 		},
@@ -267,6 +274,12 @@ func collectAction(c *cli.Context) error {
 		}
 	}()
 
+	timeout := c.Duration("timeout")
+	if timeout != 0 && timeout < time.Second {
+		log.Error("timeout must be 0 or >= 1")
+		cli.ShowAppHelpAndExit(c, 1)
+	}
+
 	srcProvider, err := getProvider(c.String("provider"), c.String("token"), providerOptions)
 	if err != nil {
 		log.Errorln(err)
@@ -288,7 +301,13 @@ func collectAction(c *cli.Context) error {
 
 	ticker := time.Tick(1 * time.Second)
 
-	eventChannel := runExtract(&pipeline, c.StringSlice("object"), c.String("after"), c.Int("limit"))
+	eventChannel := runExtract(
+		&pipeline,
+		c.StringSlice("object"),
+		c.String("after"),
+		c.Int("limit"),
+		timeout,
+	)
 
 	// runtime stats
 	var (

go.mod

@@ -1,6 +1,6 @@
 module srcfingerprint
 
-go 1.15
+go 1.16
 
 require (
 	github.com/Microsoft/go-winio v0.5.0 // indirect
@@ -13,16 +13,16 @@ require (
 	github.com/kevinburke/ssh_config v1.1.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.22.4 // indirect
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.7.1
 	github.com/suhaibmujahid/go-bitbucket-server v0.1.0
 	github.com/urfave/cli/v2 v2.3.0
 	github.com/xanzy/go-gitlab v0.50.1
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
 	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
 	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
 	golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
 	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect