Fix paginations (projectdiscovery#1603)

* fix quake pagination

* update page param

* skip empty

* fix test

* minor

Author: dogancanbakir

v2/pkg/subscraping/sources/hunter/hunter.go

@@ -60,7 +60,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		for currentPage := 1; currentPage <= pages; currentPage++ {
 			// hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2
 			qbase64 := base64.URLEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain))
-			resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=1&page_size=100&is_web=3", randomApiKey, qbase64))
+			resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=%d&page_size=100&is_web=3", randomApiKey, qbase64, currentPage))
 			if err != nil && resp == nil {
 				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
 				s.errors++

v2/pkg/subscraping/sources/quake/quake.go

@@ -58,44 +58,58 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		}
 
 		// quake api doc https://quake.360.cn/quake/#/help
-		var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "start":0, "size":500}`, domain)
-		resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{
-			"Content-Type": "application/json", "X-QuakeToken": randomApiKey,
-		}, bytes.NewReader(requestBody))
-		if err != nil {
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
-			s.errors++
-			session.DiscardHTTPResponse(resp)
-			return
-		}
+		var pageSize = 500
+		var start = 0
+		var totalResults = -1
+
+		for {
+			var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start)
+			resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{
+				"Content-Type": "application/json", "X-QuakeToken": randomApiKey,
+			}, bytes.NewReader(requestBody))
+			if err != nil {
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				s.errors++
+				session.DiscardHTTPResponse(resp)
+				return
+			}
 
-		var response quakeResults
-		err = jsoniter.NewDecoder(resp.Body).Decode(&response)
-		if err != nil {
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
-			s.errors++
+			var response quakeResults
+			err = jsoniter.NewDecoder(resp.Body).Decode(&response)
+			if err != nil {
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				s.errors++
+				session.DiscardHTTPResponse(resp)
+				return
+			}
 			session.DiscardHTTPResponse(resp)
-			return
-		}
-		session.DiscardHTTPResponse(resp)
 
-		if response.Code != 0 {
-			results <- subscraping.Result{
-				Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message),
+			if response.Code != 0 {
+				results <- subscraping.Result{
+					Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message),
+				}
+				s.errors++
+				return
+			}
+
+			if totalResults == -1 {
+				totalResults = response.Meta.Pagination.Total
 			}
-			s.errors++
-			return
-		}
 
-		if response.Meta.Pagination.Total > 0 {
 			for _, quakeDomain := range response.Data {
 				subdomain := quakeDomain.Service.HTTP.Host
 				if strings.ContainsAny(subdomain, "暂无权限") {
-					subdomain = ""
+					continue
 				}
 				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
 				s.results++
 			}
+
+			if len(response.Data) == 0 || start+pageSize >= totalResults {
+				break
+			}
+
+			start += pageSize
 		}
 	}()
 

v2/pkg/subscraping/sources/rsecloud/rsecloud.go

@@ -96,7 +96,7 @@ func (s *Source) IsDefault() bool {
 }
 
 func (s *Source) HasRecursiveSupport() bool {
-	return true
+	return false
 }
 
 func (s *Source) NeedsKey() bool {