feat: recusively scan folders [#120] (#124)

This PR refactors NewFileDirReader to use Go’s standard filepath.WalkDir (here) for directory traversal, and avoid custom recursive implementations. The function efficiently reads all files under a directory, optionally traversing subdirectories up to a configurable maxDepth. Traversal is stopped immediately for directories beyond the allowed depth or when recursion is disabled using fs.SkipDir, avoiding unnecessary filesystem reads and reducing resource usage. This approach is safe, efficient, and reduces the risk of errors compared to manual recursion, while remaining idiomatic and easy to maintain.

Usage:

```
vt scan file YOUR_DIRECTORY  (default recursive disabled + maxDepth == 1)
vt scan file YOUR_DIRECTORY -r (recursive enabled with default maxDepth == 1)
vt scan file YOUR_DIRECTORY -r -d 3 (recursive enabled + setting up maxDepth to read more subtrees until reaching depth == 3 on each subdirectory)
```

Author: JicLotus

cmd/cmd.go

@@ -74,6 +74,18 @@ func addThreadsFlag(flags *pflag.FlagSet) {
 		"number of threads working in parallel")
 }
 
+func addRecursive(flags *pflag.FlagSet) {
+	flags.BoolP(
+		"recursive", "r", false,
+		"enable recursive traversal of subdirectories")
+}
+
+func addMaxDepth(flags *pflag.FlagSet) {
+	flags.IntP(
+		"maxDepth", "d", 1,
+		"maximum recursion depth for directory traversal")
+}
+
 func addIDOnlyFlag(flags *pflag.FlagSet) {
 	flags.BoolP(
 		"identifiers-only", "I", false,

cmd/scan.go

@@ -164,7 +164,9 @@ func NewScanFileCmd() *cobra.Command {
 			if len(args) == 1 && args[0] == "-" {
 				argReader = utils.NewStringIOReader(os.Stdin)
 			} else if len(args) == 1 && utils.IsDir(args[0]) {
-				argReader, _ = utils.NewFileDirReader(args[0])
+				recursive := viper.GetBool("recursive")
+				maxDepth := viper.GetInt("maxDepth")
+				argReader, _ = utils.NewFileDirReader(args[0], recursive, maxDepth)
 			} else {
 				argReader = utils.NewStringArrayReader(args)
 			}
@@ -188,6 +190,8 @@ func NewScanFileCmd() *cobra.Command {
 		},
 	}
 
+	addRecursive(cmd.Flags())
+	addMaxDepth(cmd.Flags())
 	addThreadsFlag(cmd.Flags())
 	addOpenInVTFlag(cmd.Flags())
 	addPasswordFlag(cmd.Flags())

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect

go.sum

@@ -24,6 +24,8 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
 github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=

utils/file_utils.go

@@ -14,26 +14,56 @@
 package utils
 
 import (
+	"io/fs"
 	"os"
-	"path"
+	"path/filepath"
+	"strings"
 )
 
-// FileDirReader returns all files inside a given directory
-// as a StringArrayReader
-func NewFileDirReader(fileDir string) (*StringArrayReader, error) {
-	files, err := os.ReadDir(fileDir)
+// NewFileDirReader reads all files from the given directory `fileDir`.
+// It can optionally traverse subdirectories if `recursive` is true,
+// and will limit recursion to `maxDepth` levels if specified.
+//
+// Uses the standard library's `filepath.WalkDir` to traverse directories efficiently,
+// and `fs.SkipDir` to skip directories when recursion is disabled or maxDepth is reached.
+func NewFileDirReader(fileDir string, recursive bool, maxDepth int) (*StringArrayReader, error) {
+	var filePaths []string
+	rootDepth := pathDepth(fileDir)
+
+	// filePaths is safely appended within WalkDir because WalkDir executes the callback sequentially.
+	// No race conditions occur in this implementation, even with slice reallocation.
+	err := filepath.WalkDir(fileDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !d.IsDir() {
+			filePaths = append(filePaths, path)
+			return nil
+		}
+
+		currentDepth := pathDepth(path) - rootDepth
+		// we skip directory if recursive is disabled or
+		// if we reached configured maxDepth
+		if !recursive && path != fileDir ||
+			currentDepth >= maxDepth {
+			return fs.SkipDir
+		}
+
+		return nil
+	})
+
 	if err != nil {
 		return nil, err
 	}
-	fileNames := []string{}
-	for _, f := range files {
-		// Skip subdirectories
-		if f.IsDir() {
-			continue
-		}
-		fileNames = append(fileNames, path.Join(fileDir, f.Name()))
-	}
-	return &StringArrayReader{strings: fileNames}, nil
+	return &StringArrayReader{strings: filePaths}, nil
+}
+
+// pathDepth returns the depth of a given path by counting its components.
+// It uses filepath.Separator, which ensures correct behavior across all platforms
+// (Windows, macOS, Linux), regardless of the underlying path separator.
+func pathDepth(path string) int {
+	return len(strings.Split(filepath.Clean(path), string(filepath.Separator)))
 }
 
 // IsDir function returns whether a file is a directory or not

utils/file_utils_test.go

@@ -0,0 +1,226 @@
+// Copyright © 2017 The VirusTotal CLI authors. All Rights Reserved.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func Test_NewFileDirReader(t *testing.T) {
+	t.Parallel()
+
+	useCases := []struct {
+		name string
+
+		directories []string
+		files       []string
+		recursive   bool
+		maxDepth    int
+
+		want func(string) *StringArrayReader
+	}{
+
+		{
+			name:        "want get back empty files",
+			directories: []string{},
+			files:       []string{},
+			recursive:   true,
+			maxDepth:    2,
+			want: func(d string) *StringArrayReader {
+				return &StringArrayReader{
+					strings: nil,
+				}
+			},
+		},
+		{
+			name:        "want to read single file",
+			directories: []string{},
+			files:       []string{"z.txt"},
+			recursive:   true,
+			maxDepth:    2,
+			want: func(d string) *StringArrayReader {
+				return &StringArrayReader{
+					strings: []string{
+						filepath.Join(d, "z.txt"),
+					},
+				}
+			},
+		},
+		{
+			name:        "want read all files within all subdirectories",
+			directories: []string{"sub", "sub/sub", ".hidden"},
+			files:       []string{"a.txt", "b.txt", "sub/c.txt", "sub/sub/d.txt", ".hidden/config"},
+			recursive:   true,
+			maxDepth:    3,
+			want: func(d string) *StringArrayReader {
+				return &StringArrayReader{
+					strings: []string{
+						filepath.Join(d, ".hidden/config"),
+						filepath.Join(d, "a.txt"),
+						filepath.Join(d, "b.txt"),
+						filepath.Join(d, "sub/c.txt"),
+						filepath.Join(d, "sub/sub/d.txt"),
+					},
+				}
+			},
+		},
+		{
+			name:        "want to ignore all subdirectories",
+			directories: []string{"sub", "sub/sub"},
+			files:       []string{"a.txt", "b.txt", "sub/c.txt", "sub/sub/d.txt"},
+			recursive:   false,
+			maxDepth:    10,
+			want: func(d string) *StringArrayReader {
+				return &StringArrayReader{
+					strings: []string{
+						filepath.Join(d, "a.txt"),
+						filepath.Join(d, "b.txt"),
+					},
+				}
+			},
+		},
+		{
+			name:        "want to read until first depth",
+			directories: []string{"sub", "sub/sub"},
+			files:       []string{"a.txt", "b.txt", "sub/c.txt", "sub/sub/d.txt"},
+			recursive:   true,
+			maxDepth:    1,
+			want: func(d string) *StringArrayReader {
+				return &StringArrayReader{
+					strings: []string{
+						filepath.Join(d, "a.txt"),
+						filepath.Join(d, "b.txt"),
+					},
+				}
+			},
+		},
+		{
+			name:        "want to read until second depth",
+			directories: []string{"sub", "sub/sub"},
+			files:       []string{"a.txt", "b.txt", "sub/c.txt", "sub/sub/d.txt"},
+			recursive:   true,
+			maxDepth:    2,
+			want: func(d string) *StringArrayReader {
+				return &StringArrayReader{
+					strings: []string{
+						filepath.Join(d, "a.txt"),
+						filepath.Join(d, "b.txt"),
+						filepath.Join(d, "sub/c.txt"),
+					},
+				}
+			},
+		},
+	}
+
+	for _, uc := range useCases {
+		t.Run(uc.name, func(t *testing.T) {
+
+			// create a temp directory, and will clean up after test ends
+			rootDir := t.TempDir()
+
+			for _, d := range uc.directories {
+				path := filepath.Join(rootDir, d)
+				rwxPerm := os.FileMode(0755)
+				if err := os.Mkdir(path, rwxPerm); err != nil {
+					t.Fatalf("unexpected error while Mkdir %v", err)
+				}
+			}
+
+			for _, f := range uc.files {
+				path := filepath.Join(rootDir, f)
+				rwPerm := os.FileMode(0644)
+				if err := os.WriteFile(path, []byte("hello world!"), rwPerm); err != nil {
+					t.Fatalf("unexpected error while WriteFile %v", err)
+				}
+			}
+
+			got, err := NewFileDirReader(rootDir, uc.recursive, uc.maxDepth)
+			if err != nil {
+				t.Errorf("unexpected error while NewFileDirReader err:%v", err)
+			}
+			if diff := cmp.Diff(
+				uc.want(rootDir),
+				got,
+				cmp.AllowUnexported(StringArrayReader{}),
+			); diff != "" {
+				t.Errorf("unexpected StringArrayReader mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+func Test_NewFileDirReader_Error(t *testing.T) {
+	t.Parallel()
+
+	rootDir := t.TempDir()
+	noPerm := os.FileMode(0000)
+	if err := os.WriteFile(filepath.Join(rootDir, "a.txt"), []byte("hello world!"), noPerm); err != nil {
+		t.Fatalf("unexpected error while WriteFile %v", err)
+	}
+	path := filepath.Join(rootDir, "sub")
+	if err := os.Mkdir(path, noPerm); err != nil {
+		t.Fatalf("unexpected error while Mkdir %v", err)
+	}
+	_, err := NewFileDirReader(rootDir, false, 10)
+	if err != nil {
+		t.Errorf("unexpected error while NewFileDirReader err:%v", err)
+	}
+	_, err = NewFileDirReader(rootDir, true, 10)
+	if !strings.Contains(err.Error(), "permission denied") {
+		t.Errorf("unexpected error permissions denied message got:%v", err.Error())
+	}
+}
+
+func Test_pathDepth(t *testing.T) {
+	t.Parallel()
+
+	useCases := []struct {
+		name string
+		dir  string
+		want int
+	}{
+		{
+			name: "want one depth when empty directory",
+			want: 1,
+		},
+		{
+			name: "want one depth when simple directory",
+			dir:  "a",
+			want: 1,
+		},
+		{
+			name: "want 2 depth when sub directory",
+			dir:  "a/a",
+			want: 2,
+		},
+		{
+			name: "want 3 depth when sub directory",
+			dir:  "a-a/b_bb__/cccc/",
+			want: 3,
+		},
+	}
+
+	for _, uc := range useCases {
+		t.Run(uc.name, func(t *testing.T) {
+			if got, want := pathDepth(uc.dir), uc.want; got != want {
+				t.Errorf("unexpected pathDepth, got:%v, want:%v", got, want)
+			}
+		})
+	}
+}