Add git metrics for cloning and scanning (#4234)

Author: dustin-decker

pkg/sources/git/git.go

@@ -64,18 +64,15 @@ type Git struct {
 	jobID              sources.JobID
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
 	verify             bool
-	metrics            metrics
+	metrics            metricsCollector
 	concurrency        *semaphore.Weighted
 	skipBinaries       bool
 	skipArchives       bool
+	repoCommitsScanned uint64 // Atomic counter for commits scanned in the current repo
 
 	parser *gitparse.Parser
 }
 
-type metrics struct {
-	commitsScanned uint64
-}
-
 // Config for a Git source.
 type Config struct {
 	Concurrency        int
@@ -114,6 +111,7 @@ func NewGit(config *Config) *Git {
 		jobID:              config.JobID,
 		sourceMetadataFunc: config.SourceMetadataFunc,
 		verify:             config.Verify,
+		metrics:            metricsInstance,
 		concurrency:        semaphore.NewWeighted(int64(config.Concurrency)),
 		skipBinaries:       config.SkipBinaries,
 		skipArchives:       config.SkipArchives,
@@ -404,6 +402,9 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitURL string, authI
 		// DO NOT FORGET TO CLEAN UP THE CLONE PATH HERE!!
 		// If we don't, we'll end up with a bunch of orphaned directories in the temp dir.
 		CleanOnError(&err, clonePath)
+
+		// Note: We don't need to record the clone failure here as it's already
+		// recorded in executeClone when the error occurs
 		return "", nil, err
 	}
 
@@ -483,6 +484,10 @@ func executeClone(ctx context.Context, params cloneParams) (*git.Repository, err
 		return nil, fmt.Errorf("clone command exited with no output")
 	} else if cloneCmd.ProcessState.ExitCode() != 0 {
 		logger.V(1).Info("git clone failed", "error", err)
+		// Record the clone failure with the appropriate reason and exit code
+		failureReason := ClassifyCloneError(output)
+		exitCode := cloneCmd.ProcessState.ExitCode()
+		metricsInstance.RecordCloneOperation(statusFailure, failureReason, exitCode)
 		return nil, fmt.Errorf("could not clone repo: %s, %w", safeURL, err)
 	}
 
@@ -493,6 +498,9 @@ func executeClone(ctx context.Context, params cloneParams) (*git.Repository, err
 	}
 	logger.V(1).Info("successfully cloned repo")
 
+	// Record the successful clone operation
+	metricsInstance.RecordCloneOperation(statusSuccess, cloneSuccess, 0)
+
 	return repo, nil
 }
 
@@ -518,7 +526,18 @@ func PingRepoUsingToken(ctx context.Context, token, gitUrl, user string) error {
 	fakeRef := "TRUFFLEHOG_CHECK_GIT_REMOTE_URL_REACHABILITY"
 	gitArgs := []string{"ls-remote", lsUrl.String(), "--quiet", fakeRef}
 	cmd := exec.Command("git", gitArgs...)
-	_, err = cmd.CombinedOutput()
+	output, err := cmd.CombinedOutput()
+
+	if err != nil {
+		// Record the ping failure with the appropriate reason and exit code
+		failureReason := ClassifyCloneError(string(output))
+		exitCode := 0
+		if cmd.ProcessState != nil {
+			exitCode = cmd.ProcessState.ExitCode()
+		}
+		metricsInstance.RecordCloneOperation(statusFailure, failureReason, exitCode)
+	}
+
 	return err
 }
 
@@ -546,8 +565,9 @@ var codeCommitRE = regexp.MustCompile(`ssh://git-codecommit\.[\w-]+\.amazonaws\.
 
 func isCodeCommitURL(gitURL string) bool { return codeCommitRE.MatchString(gitURL) }
 
+// CommitsScanned returns the number of commits scanned
 func (s *Git) CommitsScanned() uint64 {
-	return atomic.LoadUint64(&s.metrics.commitsScanned)
+	return atomic.LoadUint64(&s.repoCommitsScanned)
 }
 
 const gitDirName = ".git"
@@ -627,7 +647,9 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 		if fullHash != lastCommitHash {
 			depth++
 			lastCommitHash = fullHash
-			atomic.AddUint64(&s.metrics.commitsScanned, 1)
+			s.metrics.RecordCommitScanned()
+			// Increment repo-specific commit counter
+			atomic.AddUint64(&s.repoCommitsScanned, 1)
 			logger.V(5).Info("scanning commit", "commit", fullHash)
 
 			// Scan the commit metadata.
@@ -869,7 +891,9 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 		if fullHash != lastCommitHash {
 			depth++
 			lastCommitHash = fullHash
-			atomic.AddUint64(&s.metrics.commitsScanned, 1)
+			s.metrics.RecordCommitScanned()
+			// Increment repo-specific commit counter
+			atomic.AddUint64(&s.repoCommitsScanned, 1)
 		}
 
 		if reachedBase && fullHash != scanOptions.BaseHash {
@@ -961,7 +985,12 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
 	}
 	start := time.Now().Unix()
 
+	// Reset the repo-specific commit counter
+	atomic.StoreUint64(&s.repoCommitsScanned, 0)
+
 	if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, reporter); err != nil {
+		// Record that we've failed to scan this repo
+		s.metrics.RecordRepoScanned(statusFailure)
 		return err
 	}
 	if !scanOptions.Bare {
@@ -970,6 +999,9 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
 		}
 	}
 
+	// Get the number of commits scanned in this repo
+	commitsScannedInRepo := atomic.LoadUint64(&s.repoCommitsScanned)
+
 	logger := ctx.Logger()
 	// We're logging time, but the repoPath is usually a dynamically generated folder in /tmp.
 	// To make this duration logging useful, we need to log the remote as well.
@@ -988,8 +1020,11 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
 		"scanning git repo complete",
 		"path", repoPath,
 		"time_seconds", scanTime,
-		"commits_scanned", atomic.LoadUint64(&s.metrics.commitsScanned),
+		"commits_scanned", commitsScannedInRepo,
 	)
+
+	// Record that we've scanned a repo successfully
+	s.metrics.RecordRepoScanned(statusSuccess)
 	return nil
 }
 

pkg/sources/git/metrics.go

@@ -0,0 +1,130 @@
+package git
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+)
+
+// metricsCollector defines the interface for recording Git scan metrics.
+type metricsCollector interface {
+	// Clone metrics
+	RecordCloneOperation(status string, reason string, exitCode int)
+
+	// Scan metrics
+	RecordCommitScanned()
+	RecordRepoScanned(status string)
+}
+
+// Predefined status values
+const (
+	statusSuccess = "success"
+	statusFailure = "failure"
+)
+
+// Predefined clone success reason
+const (
+	cloneSuccess = "success"
+)
+
+// Predefined clone failure reasons to avoid high cardinality
+const (
+	// Authentication/redirection errors
+	cloneFailureAuth = "auth_error"
+
+	// Rate limiting errors
+	cloneFailureRateLimit = "rate_limit"
+
+	// Permission errors
+	cloneFailurePermission = "permission_denied"
+
+	// Network/connection errors
+	cloneFailureNetwork = "network_error"
+
+	// Git reference errors
+	cloneFailureReference = "reference_error"
+
+	// Other/unknown errors
+	cloneFailureOther = "other_error"
+)
+
+type collector struct {
+	cloneOperations *prometheus.CounterVec
+	commitsScanned  prometheus.Counter
+	reposScanned    *prometheus.CounterVec
+}
+
+var metricsInstance metricsCollector
+
+func init() {
+	// These are package-level metrics that are incremented by all git scans across the lifetime of the process.
+	metricsInstance = &collector{
+		cloneOperations: promauto.NewCounterVec(prometheus.CounterOpts{
+			Namespace: common.MetricsNamespace,
+			Subsystem: common.MetricsSubsystem,
+			Name:      "git_clone_operations_total",
+			Help:      "Total number of git clone operations by status, reason, and exit code",
+		}, []string{"status", "reason", "exit_code"}),
+
+		commitsScanned: promauto.NewCounter(prometheus.CounterOpts{
+			Namespace: common.MetricsNamespace,
+			Subsystem: common.MetricsSubsystem,
+			Name:      "git_commits_scanned_total",
+			Help:      "Total number of git commits scanned",
+		}),
+
+		reposScanned: promauto.NewCounterVec(prometheus.CounterOpts{
+			Namespace: common.MetricsNamespace,
+			Subsystem: common.MetricsSubsystem,
+			Name:      "git_repos_scanned_total",
+			Help:      "Total number of git repositories scanned by status (success/failure)",
+		}, []string{"status"}),
+	}
+}
+
+func (c *collector) RecordCloneOperation(status string, reason string, exitCode int) {
+	c.cloneOperations.WithLabelValues(status, reason, fmt.Sprintf("%d", exitCode)).Inc()
+}
+
+func (c *collector) RecordCommitScanned() {
+	c.commitsScanned.Inc()
+}
+
+func (c *collector) RecordRepoScanned(status string) {
+	c.reposScanned.WithLabelValues(status).Inc()
+}
+
+// ClassifyCloneError analyzes the error message and returns the appropriate failure reason
+func ClassifyCloneError(errMsg string) string {
+	switch {
+	case strings.Contains(errMsg, "unable to update url base from redirection") &&
+		strings.Contains(errMsg, "redirect:") && strings.Contains(errMsg, "users/sign_in"):
+		return cloneFailureAuth
+
+	case strings.Contains(errMsg, "The requested URL returned error: 429") ||
+		strings.Contains(errMsg, "remote: Retry later"):
+		return cloneFailureRateLimit
+
+	case strings.Contains(errMsg, "The requested URL returned error: 403") ||
+		strings.Contains(errMsg, "remote: You are not allowed to download code from this project"):
+		return cloneFailurePermission
+
+	case strings.Contains(errMsg, "RPC failed") ||
+		strings.Contains(errMsg, "unexpected disconnect") ||
+		strings.Contains(errMsg, "early EOF") ||
+		strings.Contains(errMsg, "Problem (3) in the Chunked-Encoded data"):
+		return cloneFailureNetwork
+
+	case strings.Contains(errMsg, "cannot process") ||
+		strings.Contains(errMsg, "multiple updates for ref") ||
+		strings.Contains(errMsg, "invalid index-pack output"):
+		return cloneFailureReference
+
+	default:
+		return cloneFailureOther
+	}
+}