Merge pull request #719 from srivathsan-srinivasan/dev

Fix GCP provider to iterate over only provided project_ids in config instead of looping through All projects

Author: ShubhamRasal

PROVIDERS.md

@@ -152,6 +152,7 @@ Google Cloud Platform supports **two discovery approaches** and **two authentica
 - `service_account_email` (string, required if short-lived): Target service account to impersonate
 - `source_credentials` (string, optional): Path to source credentials file (uses ADC if not provided)
 - `token_lifetime` (string, optional): Token lifetime in seconds (e.g., "3600s") or Go duration format (e.g., "1h"). Range: 1s to 3600s (1 hour). Default: "3600s"
+- `project_ids` (list, optional): Comma-separated/list of project IDs to enumerate. When provided, Cloudlist skips discovery in every other accessible project, both for individual APIs and the organization-level Asset API.
 
 ---
 
@@ -191,8 +192,13 @@ Google Cloud Platform supports **two discovery approaches** and **two authentica
   use_short_lived_credentials: true
   service_account_email: "asset-viewer-sa@project.iam.gserviceaccount.com"
   token_lifetime: "7200s"  # 2 hours
+  project_ids:
+    - security-core
+    - shared-infra
 ```
 
+Add `project_ids` to either configuration style to limit enumeration strictly to the listed projects (Cloud Asset API requests are filtered too), which is helpful for large organizations or delegated-access service accounts.
+
 **Required Organization-Level Roles:**
 1. `roles/cloudasset.viewer` - Core Asset API access
 2. `roles/resourcemanager.viewer` - List projects in organization

internal/runner/runner.go

@@ -6,10 +6,12 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/projectdiscovery/cloudlist/pkg/inventory"
 	"github.com/projectdiscovery/cloudlist/pkg/schema"
+	"github.com/projectdiscovery/cloudlist/pkg/schema/validate"
 	"github.com/projectdiscovery/gologger"
 )
 
@@ -115,6 +117,8 @@ func (r *Runner) Enumerate() {
 				continue
 			}
 
+			sanitizePrivateIPs(instance, r.options.ExcludePrivate)
+
 			builder.Reset()
 
 			if r.options.JSON {
@@ -124,7 +128,7 @@ func (r *Runner) Enumerate() {
 				} else {
 					builder.Write(data)
 					builder.WriteString("\n")
-					output.Write(builder.Bytes()) //nolint
+					writeOutputBytes(output, builder.Bytes())
 
 					if instance.DNSName != "" {
 						hostsCount++
@@ -152,7 +156,7 @@ func (r *Runner) Enumerate() {
 					hostsCount++
 					builder.WriteString(instance.DNSName)
 					builder.WriteRune('\n')
-					output.WriteString(builder.String()) //nolint
+					writeOutputString(output, builder.String())
 					builder.Reset()
 					gologger.Silent().Msgf("%s", instance.DNSName)
 				}
@@ -163,31 +167,31 @@ func (r *Runner) Enumerate() {
 					ipCount++
 					builder.WriteString(instance.PublicIPv4)
 					builder.WriteRune('\n')
-					output.WriteString(builder.String()) //nolint
+					writeOutputString(output, builder.String())
 					builder.Reset()
 					gologger.Silent().Msgf("%s", instance.PublicIPv4)
 				}
 				if instance.PublicIPv6 != "" {
 					ipCount++
 					builder.WriteString(instance.PublicIPv6)
 					builder.WriteRune('\n')
-					output.WriteString(builder.String()) //nolint
+					writeOutputString(output, builder.String())
 					builder.Reset()
 					gologger.Silent().Msgf("%s", instance.PublicIPv6)
 				}
 				if instance.PrivateIpv4 != "" && !r.options.ExcludePrivate {
 					ipCount++
 					builder.WriteString(instance.PrivateIpv4)
 					builder.WriteRune('\n')
-					output.WriteString(builder.String()) //nolint
+					writeOutputString(output, builder.String())
 					builder.Reset()
 					gologger.Silent().Msgf("%s", instance.PrivateIpv4)
 				}
 				if instance.PrivateIpv6 != "" && !r.options.ExcludePrivate {
 					ipCount++
 					builder.WriteString(instance.PrivateIpv6)
 					builder.WriteRune('\n')
-					output.WriteString(builder.String()) //nolint
+					writeOutputString(output, builder.String())
 					builder.Reset()
 					gologger.Silent().Msgf("%s", instance.PrivateIpv6)
 				}
@@ -198,39 +202,39 @@ func (r *Runner) Enumerate() {
 				hostsCount++
 				builder.WriteString(instance.DNSName)
 				builder.WriteRune('\n')
-				output.WriteString(builder.String()) //nolint
+				writeOutputString(output, builder.String())
 				builder.Reset()
 				gologger.Silent().Msgf("%s", instance.DNSName)
 			}
 			if instance.PublicIPv4 != "" {
 				ipCount++
 				builder.WriteString(instance.PublicIPv4)
 				builder.WriteRune('\n')
-				output.WriteString(builder.String()) //nolint
+				writeOutputString(output, builder.String())
 				builder.Reset()
 				gologger.Silent().Msgf("%s", instance.PublicIPv4)
 			}
 			if instance.PublicIPv6 != "" {
 				ipCount++
 				builder.WriteString(instance.PublicIPv6)
 				builder.WriteRune('\n')
-				output.WriteString(builder.String()) //nolint
+				writeOutputString(output, builder.String())
 				builder.Reset()
 				gologger.Silent().Msgf("%s", instance.PublicIPv6)
 			}
 			if instance.PrivateIpv4 != "" && !r.options.ExcludePrivate {
 				ipCount++
 				builder.WriteString(instance.PrivateIpv4)
 				builder.WriteRune('\n')
-				output.WriteString(builder.String()) //nolint
+				writeOutputString(output, builder.String())
 				builder.Reset()
 				gologger.Silent().Msgf("%s", instance.PrivateIpv4)
 			}
 			if instance.PrivateIpv6 != "" && !r.options.ExcludePrivate {
 				ipCount++
 				builder.WriteString(instance.PrivateIpv6)
 				builder.WriteRune('\n')
-				output.WriteString(builder.String()) //nolint
+				writeOutputString(output, builder.String())
 				builder.Reset()
 				gologger.Silent().Msgf("%s", instance.PrivateIpv6)
 			}
@@ -263,3 +267,55 @@ func Contains(s []string, e string) bool {
 	}
 	return false
 }
+
+func sanitizePrivateIPs(resource *schema.Resource, exclude bool) {
+	if !exclude || resource == nil {
+		return
+	}
+	resource.PrivateIpv4 = ""
+	resource.PrivateIpv6 = ""
+
+	if isPrivateIP(resource.PublicIPv4) {
+		resource.PublicIPv4 = ""
+	}
+	if isPrivateIP(resource.PublicIPv6) {
+		resource.PublicIPv6 = ""
+	}
+}
+
+var (
+	ipValidatorOnce sync.Once
+	ipValidator     *validate.Validator
+)
+
+func isPrivateIP(ip string) bool {
+	if ip == "" {
+		return false
+	}
+	ipValidatorOnce.Do(func() {
+		var err error
+		ipValidator, err = validate.NewValidator()
+		if err != nil {
+			gologger.Warning().Msgf("could not initialize ip validator: %s", err)
+		}
+	})
+	if ipValidator == nil {
+		return false
+	}
+	resourceType := ipValidator.Identify(ip)
+	return resourceType == validate.PrivateIPv4 || resourceType == validate.PrivateIPv6
+}
+
+func writeOutputString(output *os.File, data string) {
+	if output == nil || data == "" {
+		return
+	}
+	_, _ = output.WriteString(data)
+}
+
+func writeOutputBytes(output *os.File, data []byte) {
+	if output == nil || len(data) == 0 {
+		return
+	}
+	_, _ = output.Write(data)
+}

internal/runner/runner_test.go

@@ -0,0 +1,58 @@
+package runner
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/cloudlist/pkg/schema"
+)
+
+func TestSanitizePrivateIPs(t *testing.T) {
+	resource := &schema.Resource{
+		PublicIPv4:  "1.2.3.4",
+		PrivateIpv4: "10.0.0.5",
+		PrivateIpv6: "fd00::1",
+		PublicIPv6:  "2606:4700:4700::1111",
+		DNSName:     "example.internal",
+	}
+
+	sanitizePrivateIPs(resource, true)
+
+	if resource.PrivateIpv4 != "" || resource.PrivateIpv6 != "" {
+		t.Fatalf("expected private addresses to be cleared, got %q and %q", resource.PrivateIpv4, resource.PrivateIpv6)
+	}
+
+	if resource.PublicIPv4 == "" || resource.PublicIPv6 == "" || resource.DNSName == "" {
+		t.Fatalf("public fields should remain untouched: %+v", resource)
+	}
+
+	resource.PrivateIpv4 = "10.0.0.5"
+	resource.PrivateIpv6 = "fd00::1"
+	sanitizePrivateIPs(resource, false)
+
+	if resource.PrivateIpv4 == "" || resource.PrivateIpv6 == "" {
+		t.Fatalf("expected private addresses to remain when exclusion disabled")
+	}
+
+	sanitizePrivateIPs(nil, true)
+}
+
+func TestSanitizePrivateIPsOnMisclassifiedFields(t *testing.T) {
+	resource := &schema.Resource{
+		PublicIPv4: "10.10.0.5",
+		PublicIPv6: "fd00::5",
+	}
+
+	sanitizePrivateIPs(resource, true)
+
+	if resource.PublicIPv4 != "" || resource.PublicIPv6 != "" {
+		t.Fatalf("expected private addresses in public fields to be cleared, got ipv4=%q ipv6=%q", resource.PublicIPv4, resource.PublicIPv6)
+	}
+
+	resource.PublicIPv4 = "8.8.8.8"
+	resource.PublicIPv6 = "2606:4700:4700::1111"
+	sanitizePrivateIPs(resource, true)
+
+	if resource.PublicIPv4 == "" || resource.PublicIPv6 == "" {
+		t.Fatalf("expected public addresses to remain when exclusion enabled")
+	}
+}