Adapt censys to new platform search

Use the new censys sdk for better stability in API updates. As censys now returns multiple endpoints per search, we now iterate over every endpoint and create a new result. This also changes the way ip are saved in the raw response, as we dont have the problem anymore, as we have one ip per endpoint.

Author: MaSven

.github/workflows/provider-integration.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Integration Tests
         env:
           GH_ACTION: true
-          CENSYS_API_KEY: ${{secrets.CENSYS_API_SECRET}}
+          CENSYS_API_KEY: ${{secrets.CENSYS_ORGANIZATION_ID}}
           FOFA_API_KEY: ${{secrets.FOFA_KEY}}
           SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}}
           ZOOMEYE_API_KEY: ${{secrets.ZOOMEYE_API_KEY}}

README.md

@@ -135,8 +135,8 @@ shodan:
   - SHODAN_API_KEY_1
   - SHODAN_API_KEY_2
 censys:
-  - CENSYS_API_ID_1:CENSYS_API_SECRET_1
-  - CENSYS_API_ID_2:CENSYS_API_SECRET_2
+  - CENSYS_API_TOKEN_1:CENSYS_ORGANIZATION_ID_1
+  - CENSYS_API_TOKEN_2:CENSYS_ORGANIZATION_ID_2
 fofa:
   - FOFA_EMAIL_1:FOFA_KEY_1
   - FOFA_EMAIL_2:FOFA_KEY_2
@@ -178,8 +178,8 @@ alternatively you can also set the API key as environment variable in your bash
 
 ```yaml
 export SHODAN_API_KEY=xxx
-export CENSYS_API_ID=xxx
-export CENSYS_API_SECRET=xxx
+export CENSYS_API_TOKEN=xxx
+export CENSYS_ORGANIZATION_ID=xxx
 export FOFA_EMAIL=xxx
 export FOFA_KEY=xxx
 export QUAKE_TOKEN=xxx
@@ -195,7 +195,7 @@ export ONYPHE_API_KEY=xxx
 export DRIFTNET_API_KEY=xxx
 ```
 
-Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1), ZoomEye [china](https://api.zoomeye.org) - [worldwide](https://api.zoomeye.hk), [Netlas](https://app.netlas.io/registration/), [CriminalIP](https://www.criminalip.io/register), [Publicwww](https://publicwww.com/profile/signup.html), Google [[1]](https://developers.google.com/custom-search/v1/introduction#identify_your_application_to_google_with_api_key),[[2]](https://programmablesearchengine.google.com/controlpanel/create), [Onyphe](https://search.onyphe.io/signup) and [Driftnet](https://driftnet.io/auth?state=signup).
+Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://docs.censys.com/reference/get-started), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1), ZoomEye [china](https://api.zoomeye.org) - [worldwide](https://api.zoomeye.hk), [Netlas](https://app.netlas.io/registration/), [CriminalIP](https://www.criminalip.io/register), [Publicwww](https://publicwww.com/profile/signup.html), Google [[1]](https://developers.google.com/custom-search/v1/introduction#identify_your_application_to_google_with_api_key),[[2]](https://programmablesearchengine.google.com/controlpanel/create), [Onyphe](https://search.onyphe.io/signup) and [Driftnet](https://driftnet.io/auth?state=signup).
 
 ### ZoomEye API
 

go.mod

@@ -3,6 +3,7 @@ module github.com/projectdiscovery/uncover
 go 1.23.0
 
 require (
+	github.com/censys/censys-sdk-go v0.19.1
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/logrusorgru/aurora v2.0.3+incompatible
@@ -63,6 +64,7 @@ require (
 	github.com/dlclark/regexp2 v1.11.4 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
+	github.com/ericlagergren/decimal v0.0.0-20221120152707-495c53812d05 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/gaissmai/bart v0.17.8 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect

go.sum

@@ -34,6 +34,8 @@ github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6
 github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY=
 github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs=
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/censys/censys-sdk-go v0.19.1 h1:CG8rQKgwrKuoICd3oU0uddALMfJnboeMkDg/e74HYyc=
+github.com/censys/censys-sdk-go v0.19.1/go.mod h1:DgPz5NgL+EfoueXLPG9UG1e7hS0OhtlywgpkIuu3ZRE=
 github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs=
 github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw=
 github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw=
@@ -62,6 +64,8 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY=
 github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/ericlagergren/decimal v0.0.0-20221120152707-495c53812d05 h1:S92OBrGuLLZsyM5ybUzgc/mPjIYk2AZqufieooe98uw=
+github.com/ericlagergren/decimal v0.0.0-20221120152707-495c53812d05/go.mod h1:M9R1FoZ3y//hwwnJtO51ypFGwm8ZfpxPT/ZLtO1mcgQ=
 github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
 github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=

sources/agent/censys/censys.go

@@ -1,18 +1,18 @@
 package censys
 
 import (
+	"context"
 	"encoding/json"
-	"fmt"
-	"net/http"
-	"net/url"
 
 	"errors"
 
+	censyssdkgo "github.com/censys/censys-sdk-go"
+	"github.com/censys/censys-sdk-go/models/components"
+	"github.com/censys/censys-sdk-go/models/operations"
 	"github.com/projectdiscovery/uncover/sources"
 )
 
 const (
-	URL        = "https://search.censys.io/api/v2/hosts/search?q=%s&per_page=%d&virtual_hosts=INCLUDE"
 	MaxPerPage = 100
 )
 
@@ -23,9 +23,10 @@ func (agent *Agent) Name() string {
 }
 
 func (agent *Agent) Query(session *sources.Session, query *sources.Query) (chan sources.Result, error) {
-	if session.Keys.CensysToken == "" || session.Keys.CensysSecret == "" {
+	if session.Keys.CensysToken == "" || session.Keys.CensysOrgId == "" {
 		return nil, errors.New("empty censys keys")
 	}
+
 	results := make(chan sources.Result)
 
 	go func() {
@@ -39,76 +40,82 @@ func (agent *Agent) Query(session *sources.Session, query *sources.Query) (chan
 				PerPage: MaxPerPage,
 				Cursor:  nextCursor,
 			}
-			censysResponse := agent.query(URL, session, censysRequest, results)
+			censysResponse := agent.query(session, censysRequest, results)
 			if censysResponse == nil {
 				break
 			}
-			nextCursor = censysResponse.Results.Links.Next
-			if nextCursor == "" || numberOfResults > query.Limit || len(censysResponse.Results.Hits) == 0 {
+			hasNextCursor := false
+			if censysResponse.ResponseEnvelopeSearchQueryResponse.Result.NextPageToken != nil {
+				hasNextCursor = true
+			}
+
+			if hasNextCursor || numberOfResults > query.Limit || len(censysResponse.ResponseEnvelopeSearchQueryResponse.Result.Hits) == 0 {
 				break
 			}
-			numberOfResults += len(censysResponse.Results.Hits)
+			nextCursor = censysResponse.ResponseEnvelopeSearchQueryResponse.Result.NextPageToken
+			numberOfResults += len(censysResponse.ResponseEnvelopeSearchQueryResponse.Result.Hits)
 		}
 	}()
 
 	return results, nil
 }
 
-func (agent *Agent) queryURL(session *sources.Session, URL string, censysRequest *CensysRequest) (*http.Response, error) {
-	censysURL := fmt.Sprintf(URL, url.QueryEscape(censysRequest.Query), censysRequest.PerPage)
-	if censysRequest.Cursor != "" {
-		censysURL += fmt.Sprintf("&cursor=%s", censysRequest.Cursor)
-	}
-	request, err := sources.NewHTTPRequest(http.MethodGet, censysURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	request.Header.Set("Accept", "application/json")
-	request.SetBasicAuth(session.Keys.CensysToken, session.Keys.CensysSecret)
-	return session.Do(request, agent.Name())
+func (agent *Agent) queryURL(session *sources.Session, censysRequest *CensysRequest) (*operations.V3GlobaldataSearchQueryResponse, error) {
+	ctx := context.Background()
+
+	s := censyssdkgo.New(
+		censyssdkgo.WithOrganizationID(session.Keys.CensysOrgId),
+		censyssdkgo.WithSecurity(session.Keys.CensysToken),
+		censyssdkgo.WithClient(
+			session.Client.HTTPClient,
+		),
+	)
+
+	return s.GlobalData.Search(ctx, operations.V3GlobaldataSearchQueryRequest{
+		SearchQueryInputBody: components.SearchQueryInputBody{
+			PageSize:  censyssdkgo.Int64(MaxPerPage),
+			Query:     censysRequest.Query,
+			PageToken: &censysRequest.Cursor,
+		},
+	})
+
 }
 
-func (agent *Agent) query(URL string, session *sources.Session, censysRequest *CensysRequest, results chan sources.Result) *CensysResponse {
+func (agent *Agent) query(session *sources.Session, censysRequest *CensysRequest, results chan sources.Result) *operations.V3GlobaldataSearchQueryResponse {
 	// query certificates
-	resp, err := agent.queryURL(session, URL, censysRequest)
+	resp, err := agent.queryURL(session, censysRequest)
 	if err != nil {
 		results <- sources.Result{Source: agent.Name(), Error: err}
 		// httputil.DrainResponseBody(resp)
 		return nil
 	}
 
-	censysResponse := &CensysResponse{}
-	if err := json.NewDecoder(resp.Body).Decode(censysResponse); err != nil {
-		results <- sources.Result{Source: agent.Name(), Error: err}
-		return nil
-	}
+	if result := resp.ResponseEnvelopeSearchQueryResponse.Result; result != nil {
+		for _, censysResult := range result.Hits {
 
-	for _, censysResult := range censysResponse.Results.Hits {
-		result := sources.Result{Source: agent.Name()}
-		if ip, ok := censysResult["ip"]; ok {
-			result.IP = ip.(string)
-		}
-		if name, ok := censysResult["name"]; ok {
-			result.Host = name.(string)
-		}
-		if services, ok := censysResult["services"]; ok {
-			for _, serviceData := range services.([]interface{}) {
-				if serviceData, ok := serviceData.(map[string]interface{}); ok {
-					result.Port = int(serviceData["port"].(float64))
-					raw, _ := json.Marshal(censysResult)
-					result.Raw = raw
-					results <- result
+			for _, host := range censysResult.WebpropertyV1.Resource.Endpoints {
+				result := sources.Result{Source: agent.Name()}
+				if host.IP != nil {
+					result.IP = *host.IP
 				}
+				if host.Hostname != nil {
+					result.Host = *host.Hostname
+				}
+				if host.Port != nil {
+					result.Port = *host.Port
+				}
+				if host.HTTP != nil && host.HTTP.URI != nil {
+					result.Url = *host.HTTP.URI
+				}
+				raw, _ := json.Marshal(host)
+				result.Raw = raw
+				results <- result
 			}
-		} else {
-			raw, _ := json.Marshal(censysResult)
-			result.Raw = raw
-			// only ip
-			results <- result
+
 		}
 	}
 
-	return censysResponse
+	return resp
 }
 
 type CensysRequest struct {

sources/agent/censys/example.json

@@ -2,7 +2,7 @@ package sources
 
 type Keys struct {
 	CensysToken     string
-	CensysSecret    string
+	CensysOrgId     string
 	Shodan          string
 	FofaEmail       string
 	FofaKey         string
@@ -22,7 +22,7 @@ type Keys struct {
 }
 
 func (keys Keys) Empty() bool {
-	return keys.CensysSecret == "" &&
+	return keys.CensysOrgId == "" &&
 		keys.CensysToken == "" &&
 		keys.Shodan == "" &&
 		keys.FofaEmail == "" &&

sources/agent/censys/response.go

@@ -56,7 +56,7 @@ func (provider *Provider) GetKeys() Keys {
 		parts := strings.Split(censysKeys, ":")
 		if len(parts) == 2 {
 			keys.CensysToken = parts[0]
-			keys.CensysSecret = parts[1]
+			keys.CensysOrgId = parts[1]
 		}
 	}
 
@@ -163,7 +163,7 @@ func (provider *Provider) LoadProviderKeysFromEnv() {
 		return arr
 	}
 	provider.Fofa = appendIfAllExists(provider.Fofa, "FOFA_EMAIL", "FOFA_KEY")
-	provider.Censys = appendIfAllExists(provider.Censys, "CENSYS_API_ID", "CENSYS_API_SECRET")
+	provider.Censys = appendIfAllExists(provider.Censys, "CENSYS_API_TOKEN", "CENSYS_ORGANIZATION_ID")
 	provider.Google = appendIfAllExists(provider.Google, "GOOGLE_API_KEY", "GOOGLE_API_CX")
 	provider.Odin = appendIfExists(provider.Odin, "ODIN_API_KEY")
 	provider.BinaryEdge = appendIfExists(provider.BinaryEdge, "BINARYEDGE_API_KEY")