Allow filesystem exclusion to eagerly prune the enumerated tree (#4008)

* Allow filesystem exclusion to eagerly prune the enumerated tree rather than skipping files afterwards

* added test case

* resolved charlie's comment

* resolved cody comments

* updated test case

* remove shouldExclude method

* added test cases and fixed the logic

* removed print statement

* renamed test case

* assert->require in tests

Author: kashifkhan0771

pkg/common/filter.go

@@ -97,8 +97,10 @@ func (filter *Filter) Pass(object string) bool {
 	if filter == nil {
 		return true
 	}
+
 	excluded := filter.exclude.Matches(object)
 	included := filter.include.Matches(object)
+
 	return !excluded && included
 }
 
@@ -114,3 +116,8 @@ func (rules *FilterRuleSet) Matches(object string) bool {
 	}
 	return false
 }
+
+// ShouldExclude return true if any regular expressions in the exclude FilterRuleSet matches the path.
+func (filter *Filter) ShouldExclude(path string) bool {
+	return filter.exclude.Matches(path)
+}

pkg/sources/filesystem/filesystem.go

@@ -127,17 +127,25 @@ func (s *Source) scanDir(ctx context.Context, path string, chunksChan chan *sour
 			ctx.Logger().Error(err, "error walking directory")
 			return nil
 		}
+
 		fullPath := filepath.Join(path, relativePath)
 
+		// check if the full path is not matching any pattern in include FilterRuleSet and matching any exclude FilterRuleSet.
+		if s.filter != nil && !s.filter.Pass(fullPath) {
+			// skip excluded directories
+			if d.IsDir() && s.filter.ShouldExclude(fullPath) {
+				return fs.SkipDir
+			}
+
+			return nil // skip the file
+		}
+
 		// Skip over non-regular files. We do this check here to suppress noisy
 		// logs for trying to scan directories and other non-regular files in
 		// our traversal.
 		if !d.Type().IsRegular() {
 			return nil
 		}
-		if s.filter != nil && !s.filter.Pass(fullPath) {
-			return nil
-		}
 
 		workerPool.Go(func() error {
 			if err = s.scanFile(ctx, fullPath, chunksChan); err != nil {

pkg/sources/filesystem/filesystem_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/kylelemons/godebug/pretty"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -300,6 +301,83 @@ func TestChunkUnitReporterErr(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestSkipDir(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	// create a temp directory with files
+	ignoreDir, cleanupDir, err := createTempDir("", "ignore1", "ignore2", "ignore3")
+	require.NoError(t, err)
+	defer cleanupDir()
+
+	// create an ExcludePathsFile that contains the ignoreDir path
+	excludeFile, cleanupFile, err := createTempFile("", ignoreDir+"\n")
+	require.NoError(t, err)
+	defer cleanupFile()
+
+	conn, err := anypb.New(&sourcespb.Filesystem{
+		ExcludePathsFile: excludeFile.Name(),
+	})
+	require.NoError(t, err)
+
+	// initialize the source.
+	s := Source{}
+	err = s.Init(ctx, "exclude directory", 0, 0, true, conn, 1)
+	require.NoError(t, err)
+
+	reporter := sourcestest.TestReporter{}
+	err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
+		ID: ignoreDir,
+	}, &reporter)
+	require.NoError(t, err)
+
+	require.Equal(t, 0, len(reporter.Chunks), "Expected no chunks from excluded directory")
+	require.Equal(t, 0, len(reporter.ChunkErrs), "Expected no errors for excluded directory")
+}
+
+func TestScanSubDirFile(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	// create a temp directory with files
+	parentDir, cleanupParentDir, err := createTempDir("", "file1")
+	require.NoError(t, err)
+	defer cleanupParentDir()
+
+	childDir, cleanupChildDir, err := createTempDir(parentDir, "file2")
+	require.NoError(t, err)
+	defer cleanupChildDir()
+
+	// create a file in child directory
+	file, cleanupFile, err := createTempFile(childDir, "should scan this file")
+	require.NoError(t, err)
+	defer cleanupFile()
+
+	// create an IncludePathsFile that contains the file path
+	includeFile, cleanupFile, err := createTempFile("", file.Name()+"\n")
+	require.NoError(t, err)
+	defer cleanupFile()
+
+	conn, err := anypb.New(&sourcespb.Filesystem{
+		IncludePathsFile: includeFile.Name(),
+	})
+	require.NoError(t, err)
+
+	// initialize the source.
+	s := Source{}
+	err = s.Init(ctx, "include sub directory file", 0, 0, true, conn, 1)
+	require.NoError(t, err)
+
+	reporter := sourcestest.TestReporter{}
+	err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
+		ID: parentDir,
+	}, &reporter)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(reporter.Chunks), "Expected chunks from included file")
+	require.Equal(t, 0, len(reporter.ChunkErrs), "Expected no errors")
+}
+
 // createTempFile is a helper function to create a temporary file in the given
 // directory with the provided contents. If dir is "", the operating system's
 // temp directory is used.