Merge pull request projectdiscovery#7043 from projectdiscovery/dwisiswant0/fix/http/isolate-project-cache-keys-by-scheme-host

fix(http): isolate project cache keys by scheme & host

Author: dogancanbakir

pkg/protocols/http/request.go

@@ -691,9 +691,10 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 	}
 
 	var (
-		resp          *http.Response
-		fromCache     bool
-		dumpedRequest []byte
+		resp            *http.Response
+		fromCache       bool
+		dumpedRequest   []byte
+		projectCacheKey []byte
 	)
 
 	// Dump request for variables checks
@@ -731,6 +732,11 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		if dumpError != nil {
 			return dumpError
 		}
+		if generatedRequest.request != nil && generatedRequest.request.URL != nil {
+			projectCacheKey = getHTTPProjectCacheScope(dumpedRequest, generatedRequest.request.Scheme, generatedRequest.request.URL.Host)
+		} else {
+			projectCacheKey = dumpedRequest
+		}
 		dumpedRequestString := string(dumpedRequest)
 
 		if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil {
@@ -815,7 +821,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		if request.options.ProjectFile != nil {
 			// if unavailable fail silently
 			fromCache = true
-			resp, err = request.options.ProjectFile.Get(dumpedRequest)
+			resp, err = request.options.ProjectFile.Get(projectCacheKey)
 			if err != nil {
 				fromCache = false
 			}
@@ -965,7 +971,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 	onceFunc := sync.OnceFunc(func() {
 		// if nuclei-project is enabled store the response if not previously done
 		if request.options.ProjectFile != nil && !fromCache {
-			if err := request.options.ProjectFile.Set(dumpedRequest, resp, respChain.BodyBytes()); err != nil {
+			if err := request.options.ProjectFile.Set(projectCacheKey, resp, respChain.BodyBytes()); err != nil {
 				errx = errors.Wrap(err, "could not store in project file")
 			}
 		}

pkg/protocols/http/utils.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"bytes"
 	"io"
 	"strings"
 
@@ -26,3 +27,20 @@ func dump(req *generatedRequest, reqURL string) ([]byte, error) {
 	}
 	return bin, nil
 }
+
+func getHTTPProjectCacheScope(requestDump []byte, scheme, host string) []byte {
+	scheme = strings.ToLower(strings.TrimSpace(scheme))
+	host = strings.ToLower(strings.TrimSpace(host))
+	if scheme == "" || host == "" {
+		return requestDump
+	}
+
+	var scoped bytes.Buffer
+	scoped.Grow(len(scheme) + len(host) + len(requestDump) + 4)
+	_, _ = scoped.WriteString(scheme)
+	_, _ = scoped.WriteString("://")
+	_, _ = scoped.WriteString(host)
+	_, _ = scoped.WriteString("\n")
+	_, _ = scoped.Write(requestDump)
+	return scoped.Bytes()
+}

pkg/protocols/http/utils_test.go

@@ -0,0 +1,19 @@
+package http
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetHTTPProjectCacheScope_SeparatesSchemeAndPort(t *testing.T) {
+	requestDump := []byte("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n")
+
+	httpScoped := getHTTPProjectCacheScope(requestDump, "http", "example.com:80")
+	httpsScoped := getHTTPProjectCacheScope(requestDump, "https", "example.com:443")
+
+	require.NotEqual(t, httpScoped, httpsScoped)
+	require.True(t, bytes.HasSuffix(httpScoped, requestDump))
+	require.True(t, bytes.HasSuffix(httpsScoped, requestDump))
+}