Log recursion limiting (#4236)

* No need to take the address of a map; they're already pointer types

* Log whenever we hit the recursion limit

Author: camgunz

pkg/sources/postman/postman.go

@@ -329,10 +329,13 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu
 // scanCollection scans a collection and all its items, folders, and requests.
 // locally scoped Metadata is updated as we drill down into the collection.
 func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, collection Collection) {
-	ctx.Logger().V(2).Info("starting to scan collection",
+	ctx = context.WithValues(ctx,
 		"collection_name", collection.Info.Name,
 		"collection_uuid", collection.Info.Uid,
-		"variable_count", len(collection.Variables))
+	)
+	ctx.Logger().V(2).Info("starting to scan collection",
+		"variable_count", len(collection.Variables),
+	)
 	metadata.CollectionInfo = collection.Info
 	metadata.Type = COLLECTION_TYPE
 	s.attemptToAddKeyword(collection.Info.Name)
@@ -354,32 +357,34 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch
 	s.scanAuth(ctx, chunksChan, metadata, collection.Auth, URL{})
 
 	ctx.Logger().V(3).Info("Scanning events in collection",
-		"collection_uid", collection.Info.Uid,
 		"event_count", len(collection.Events),
 	)
 	for _, event := range collection.Events {
 		s.scanEvent(ctx, chunksChan, metadata, event)
 	}
 
 	ctx.Logger().V(3).Info("Scanning items in collection",
-		"collection_uid", collection.Info.Uid,
 		"item_ids", fp.Map(func(i Item) string { return i.Id })(collection.Items),
 	)
 	for _, item := range collection.Items {
-		s.scanItem(ctx, chunksChan, collection, metadata, item, "")
+		seenItemIds := make(map[string]struct{})
+		s.scanItem(ctx, chunksChan, collection, metadata, item, "", seenItemIds)
 	}
-
 }
 
-func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, collection Collection, metadata Metadata, item Item, parentItemId string) {
+func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, collection Collection, metadata Metadata, item Item, parentItemId string, seenItemIds map[string]struct{}) {
+	ctx = context.WithValue(ctx, "item_uid", item.Uid)
+
 	ctx.Logger().V(3).Info("Starting to scan item",
-		"item_uid", item.Uid,
 		"item_parent_item_id", parentItemId,
 		"item_descendent_item_uids", fp.Map(func(i Item) string { return i.Uid })(item.Items),
 		"item_event_count", len(item.Events),
 		"item_response_count", len(item.Response),
 		"item_variable_count", len(item.Variable),
 	)
+
+	seenItemIds[item.Uid] = struct{}{}
+
 	s.attemptToAddKeyword(item.Name)
 
 	// override the base collection metadata with item-specific metadata
@@ -397,7 +402,13 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c
 	}
 	// recurse through the folders
 	for _, subItem := range item.Items {
-		s.scanItem(ctx, chunksChan, collection, metadata, subItem, item.Uid)
+		if _, ok := seenItemIds[subItem.Uid]; ok {
+			ctx.Logger().Info("Skipping already-seen item",
+				"seen_item_id", subItem.Uid,
+			)
+			continue
+		}
+		s.scanItem(ctx, chunksChan, collection, metadata, subItem, item.Uid, seenItemIds)
 	}
 
 	// The assignment of the folder ID to be the current item UID is due to wanting to assume that your current item is a folder unless you have request data inside of your item.
@@ -463,7 +474,8 @@ func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk,
 		metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_SCRIPT
 	}
 
-	s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, data, DefaultMaxRecursionDepth)), metadata)
+	ctx = context.WithValue(ctx, "event_listen", event.Listen)
+	s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, metadata, data, DefaultMaxRecursionDepth)), metadata)
 	metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 }
 
@@ -561,7 +573,7 @@ func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m
 	} else if strings.Contains(m.Type, COLLECTION_TYPE) {
 		m.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_AUTHORIZATION
 	}
-	s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, authData, DefaultMaxRecursionDepth)), m)
+	s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, authData, DefaultMaxRecursionDepth)), m)
 	m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 }
 
@@ -591,7 +603,7 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C
 		metadata.Type = originalType + " > header"
 		metadata.Link = metadata.Link + "?tab=headers"
 		metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_HEADER
-		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, strings.Join(r.HeaderString, " "), DefaultMaxRecursionDepth)), metadata)
+		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, metadata, strings.Join(r.HeaderString, " "), DefaultMaxRecursionDepth)), metadata)
 		metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
@@ -600,7 +612,7 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C
 		// Note: query parameters are handled separately
 		u := fmt.Sprintf("%s://%s/%s", r.URL.Protocol, strings.Join(r.URL.Host, "."), strings.Join(r.URL.Path, "/"))
 		metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_URL
-		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, u, DefaultMaxRecursionDepth)), metadata)
+		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, metadata, u, DefaultMaxRecursionDepth)), metadata)
 		metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
@@ -655,13 +667,13 @@ func (s *Source) scanRequestBody(ctx context.Context, chunksChan chan *sources.C
 		m.Type = originalType + " > raw"
 		data := b.Raw
 		m.LocationType = source_metadatapb.PostmanLocationType_REQUEST_BODY_RAW
-		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, data, DefaultMaxRecursionDepth)), m)
+		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, data, DefaultMaxRecursionDepth)), m)
 		m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	case "graphql":
 		m.Type = originalType + " > graphql"
 		data := b.GraphQL.Query + " " + b.GraphQL.Variables
 		m.LocationType = source_metadatapb.PostmanLocationType_REQUEST_BODY_GRAPHQL
-		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, data, DefaultMaxRecursionDepth)), m)
+		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, data, DefaultMaxRecursionDepth)), m)
 		m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 }
@@ -687,15 +699,15 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.
 		m.Type = originalType + " > response header"
 		// TODO Note: for now, links to Postman responses do not include a more granular tab for the params/header/body, but when they do, we will need to update the metadata.Link info
 		m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_HEADER
-		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, strings.Join(response.HeaderString, " "), DefaultMaxRecursionDepth)), m)
+		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, strings.Join(response.HeaderString, " "), DefaultMaxRecursionDepth)), m)
 		m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
 	// Body in a response is just a string
 	if response.Body != "" {
 		m.Type = originalType + " > response body"
 		m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_BODY
-		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, response.Body, DefaultMaxRecursionDepth)), m)
+		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, response.Body, DefaultMaxRecursionDepth)), m)
 		m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
@@ -728,7 +740,7 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.
 		if valStr == "" {
 			continue
 		}
-		values = append(values, s.buildSubstituteSet(m, valStr, DefaultMaxRecursionDepth)...)
+		values = append(values, s.buildSubstituteSet(ctx, m, valStr, DefaultMaxRecursionDepth)...)
 	}
 
 	m.FieldType = m.Type + " variables"

pkg/sources/postman/substitution.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 )
 
 var subRe = regexp.MustCompile(`\{\{[^{}]+\}\}`)
@@ -52,12 +54,17 @@ func (s *Source) formatAndInjectKeywords(data []string) string {
 
 // buildSubstituteSet creates a set of substitutions for the given data
 // maxRecursionDepth is the maximum recursion depth to use for variable substitution
-func (s *Source) buildSubstituteSet(metadata Metadata, data string, maxRecursionDepth int) []string {
+func (s *Source) buildSubstituteSet(
+	ctx context.Context,
+	metadata Metadata,
+	data string,
+	maxRecursionDepth int,
+) []string {
 	var ret []string
 	combos := make(map[string]struct{})
 
 	// Call buildSubstitution with initial depth of 0 and the maxRecursionDepth
-	s.buildSubstitution(data, metadata, &combos, 0, maxRecursionDepth)
+	s.buildSubstitution(ctx, data, metadata, combos, 0, maxRecursionDepth)
 
 	for combo := range combos {
 		ret = append(ret, combo)
@@ -73,22 +80,26 @@ func (s *Source) buildSubstituteSet(metadata Metadata, data string, maxRecursion
 // depth is the current recursion depth
 // maxRecursionDepth is the maximum recursion depth to use for variable substitution
 func (s *Source) buildSubstitution(
+	ctx context.Context,
 	data string,
 	metadata Metadata,
-	combos *map[string]struct{},
+	combos map[string]struct{},
 	depth int,
 	maxRecursionDepth int,
 ) {
 	// Limit recursion depth to prevent stack overflow
 	if depth > maxRecursionDepth {
-		(*combos)[data] = struct{}{}
+		ctx.Logger().V(2).Info("Limited recursion depth",
+			"depth", depth,
+		)
+		combos[data] = struct{}{}
 		return
 	}
 
 	matches := removeDuplicateStr(subRe.FindAllString(data, -1))
 	if len(matches) == 0 {
 		// No more substitutions to make, add to combos
-		(*combos)[data] = struct{}{}
+		combos[data] = struct{}{}
 		return
 	}
 
@@ -117,14 +128,14 @@ func (s *Source) buildSubstitution(
 			// Only mark substitution as made if we actually changed something
 			if d != data {
 				substitutionMade = true
-				s.buildSubstitution(d, metadata, combos, depth+1, maxRecursionDepth)
+				s.buildSubstitution(ctx, d, metadata, combos, depth+1, maxRecursionDepth)
 			}
 		}
 	}
 
 	// If no substitutions were made, add the current data
 	if !substitutionMade {
-		(*combos)[data] = struct{}{}
+		combos[data] = struct{}{}
 	}
 }
 

pkg/sources/postman/substitution_test.go

@@ -5,6 +5,8 @@ import (
 	"sort"
 	"strings"
 	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 )
 
 func TestNewSubstitution(t *testing.T) {
@@ -63,6 +65,8 @@ func TestSource_KeywordCombinations(t *testing.T) {
 }
 
 func TestSource_BuildSubstituteSet(t *testing.T) {
+	ctx := context.Background()
+
 	s := &Source{
 		sub: NewSubstitution(),
 	}
@@ -89,7 +93,7 @@ func TestSource_BuildSubstituteSet(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		result := s.buildSubstituteSet(metadata, tc.data, DefaultMaxRecursionDepth)
+		result := s.buildSubstituteSet(ctx, metadata, tc.data, DefaultMaxRecursionDepth)
 		if !reflect.DeepEqual(result, tc.expected) {
 			t.Errorf("Expected substitution set: %v, got: %v", tc.expected, result)
 		}
@@ -158,6 +162,8 @@ func TestSource_FormatAndInjectKeywords(t *testing.T) {
 }
 
 func TestSource_BuildSubstitution_RecursionLimit(t *testing.T) {
+	ctx := context.Background()
+
 	s := &Source{
 		sub: NewSubstitution(),
 	}
@@ -229,9 +235,9 @@ func TestSource_BuildSubstitution_RecursionLimit(t *testing.T) {
 
 			// Use custom maxDepth if provided, otherwise use default
 			if tc.maxDepth > 0 {
-				s.buildSubstitution(tc.data, metadata, &combos, 0, tc.maxDepth)
+				s.buildSubstitution(ctx, tc.data, metadata, combos, 0, tc.maxDepth)
 			} else {
-				s.buildSubstitution(tc.data, metadata, &combos, 0, DefaultMaxRecursionDepth)
+				s.buildSubstitution(ctx, tc.data, metadata, combos, 0, DefaultMaxRecursionDepth)
 			}
 
 			var result []string