Fixes #3388 (and refactors #1578)

Author: stamparm

lib/controller/controller.py

@@ -313,6 +313,7 @@ def start():
             conf.cookie = targetCookie
             conf.httpHeaders = list(initialHeaders)
             conf.httpHeaders.extend(targetHeaders or [])
+            conf.httpHeaders = [conf.httpHeaders[i] for i in xrange(len(conf.httpHeaders)) if conf.httpHeaders[i][0].upper() not in (__[0].upper() for __ in conf.httpHeaders[i + 1:])]
 
             initTargetEnv()
             parseTargetUrl()

lib/core/option.py

@@ -102,6 +102,7 @@
 from lib.core.settings import DEFAULT_PAGE_ENCODING
 from lib.core.settings import DEFAULT_TOR_HTTP_PORTS
 from lib.core.settings import DEFAULT_TOR_SOCKS_PORTS
+from lib.core.settings import DEFAULT_USER_AGENT
 from lib.core.settings import DUMMY_URL
 from lib.core.settings import IS_WIN
 from lib.core.settings import KB_CHARS_BOUNDARY_CHAR
@@ -112,7 +113,6 @@
 from lib.core.settings import NULL
 from lib.core.settings import PARAMETER_SPLITTING_REGEX
 from lib.core.settings import PRECONNECT_CANDIDATE_TIMEOUT
-from lib.core.settings import SITE
 from lib.core.settings import SOCKET_PRE_CONNECT_QUEUE_SIZE
 from lib.core.settings import SQLMAP_ENVIRONMENT_PREFIX
 from lib.core.settings import SUPPORTED_DBMS
@@ -122,7 +122,6 @@
 from lib.core.settings import UNION_CHAR_REGEX
 from lib.core.settings import UNKNOWN_DBMS_VERSION
 from lib.core.settings import URI_INJECTABLE_REGEX
-from lib.core.settings import VERSION_STRING
 from lib.core.threads import getCurrentThreadData
 from lib.core.threads import setDaemon
 from lib.core.update import update
@@ -1256,14 +1255,6 @@ def _setHTTPExtraHeaders():
         # Reference: http://stackoverflow.com/a/1383359
         conf.httpHeaders.append((HTTP_HEADER.CACHE_CONTROL, "no-cache"))
 
-def _defaultHTTPUserAgent():
-    """
-    @return: default sqlmap HTTP User-Agent header
-    @rtype: C{str}
-    """
-
-    return "%s (%s)" % (VERSION_STRING, SITE)
-
 def _setHTTPUserAgent():
     """
     Set the HTTP User-Agent header.
@@ -1308,7 +1299,7 @@ def _setHTTPUserAgent():
                 break
 
         if _:
-            conf.httpHeaders.append((HTTP_HEADER.USER_AGENT, _defaultHTTPUserAgent()))
+            conf.httpHeaders.append((HTTP_HEADER.USER_AGENT, DEFAULT_USER_AGENT))
 
     else:
         if not kb.userAgents:
@@ -1323,10 +1314,10 @@ def _setHTTPUserAgent():
                 warnMsg += "file '%s'" % paths.USER_AGENTS
                 logger.warn(warnMsg)
 
-                conf.httpHeaders.append((HTTP_HEADER.USER_AGENT, _defaultHTTPUserAgent()))
+                conf.httpHeaders.append((HTTP_HEADER.USER_AGENT, DEFAULT_USER_AGENT))
                 return
 
-        userAgent = random.sample(kb.userAgents or [_defaultHTTPUserAgent()], 1)[0]
+        userAgent = random.sample(kb.userAgents or [DEFAULT_USER_AGENT], 1)[0]
 
         infoMsg = "fetched random HTTP User-Agent header value '%s' from " % userAgent
         infoMsg += "file '%s'" % paths.USER_AGENTS

lib/core/settings.py

@@ -19,12 +19,13 @@
 from lib.core.enums import OS
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.2.11.18"
+VERSION = "1.2.11.19"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
 DESCRIPTION = "automatic SQL injection and database takeover tool"
 SITE = "http://sqlmap.org"
+DEFAULT_USER_AGENT = "%s (%s)" % (VERSION_STRING, SITE)
 DEV_EMAIL_ADDRESS = "[email protected]"
 ISSUES_PAGE = "https://github.com/sqlmapproject/sqlmap/issues/new"
 GIT_REPOSITORY = "https://github.com/sqlmapproject/sqlmap.git"

lib/request/connect.py

@@ -89,6 +89,7 @@ class WebSocketException(Exception):
 from lib.core.settings import DEFAULT_CONTENT_TYPE
 from lib.core.settings import DEFAULT_COOKIE_DELIMITER
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
+from lib.core.settings import DEFAULT_USER_AGENT
 from lib.core.settings import EVALCODE_KEYWORD_SUFFIX
 from lib.core.settings import HTTP_ACCEPT_HEADER_VALUE
 from lib.core.settings import HTTP_ACCEPT_ENCODING_HEADER_VALUE
@@ -361,14 +362,21 @@ def getPage(**kwargs):
             if kb.proxyAuthHeader:
                 headers[HTTP_HEADER.PROXY_AUTHORIZATION] = kb.proxyAuthHeader
 
-            if not getHeader(headers, HTTP_HEADER.ACCEPT):
-                headers[HTTP_HEADER.ACCEPT] = HTTP_ACCEPT_HEADER_VALUE
+            if not conf.requestFile or not target:
+                if not getHeader(headers, HTTP_HEADER.HOST):
+                    headers[HTTP_HEADER.HOST] = getHostHeader(url)
 
-            if not getHeader(headers, HTTP_HEADER.HOST) or not target:
-                headers[HTTP_HEADER.HOST] = getHostHeader(url)
+                if not getHeader(headers, HTTP_HEADER.ACCEPT):
+                    headers[HTTP_HEADER.ACCEPT] = HTTP_ACCEPT_HEADER_VALUE
 
-            if not getHeader(headers, HTTP_HEADER.ACCEPT_ENCODING):
-                headers[HTTP_HEADER.ACCEPT_ENCODING] = HTTP_ACCEPT_ENCODING_HEADER_VALUE if kb.pageCompress else "identity"
+                if not getHeader(headers, HTTP_HEADER.ACCEPT_ENCODING):
+                    headers[HTTP_HEADER.ACCEPT_ENCODING] = HTTP_ACCEPT_ENCODING_HEADER_VALUE if kb.pageCompress else "identity"
+
+            elif conf.requestFile and getHeader(headers, HTTP_HEADER.USER_AGENT) == DEFAULT_USER_AGENT:
+                for header in headers:
+                    if header.upper() == HTTP_HEADER.USER_AGENT.upper():
+                        del headers[header]
+                        break
 
             if post is not None and not multipart and not getHeader(headers, HTTP_HEADER.CONTENT_TYPE):
                 headers[HTTP_HEADER.CONTENT_TYPE] = POST_HINT_CONTENT_TYPES.get(kb.postHint, DEFAULT_CONTENT_TYPE)
@@ -385,10 +393,6 @@ def getPage(**kwargs):
             if conf.keepAlive:
                 headers[HTTP_HEADER.CONNECTION] = "keep-alive"
 
-            # Reset header values to original in case of provided request file
-            if target and conf.requestFile:
-                headers = forgeHeaders({HTTP_HEADER.COOKIE: cookie})
-
             if auxHeaders:
                 headers = forgeHeaders(auxHeaders, headers)
 

txt/checksum.md5

@@ -24,7 +24,7 @@ b3e60ea4e18a65c48515d04aab28ff68  extra/sqlharvest/sqlharvest.py
 c1bccc94522d3425a372dcd57f78418e  extra/wafdetectify/wafdetectify.py
 3459c562a6abb9b4bdcc36925f751f3e  lib/controller/action.py
 71334197c7ed28167cd66c17b2c21844  lib/controller/checks.py
-dd42ef140ffc0bd517128e6df369ab01  lib/controller/controller.py
+95cde6dc7efe2581a5936f0d4635cb3b  lib/controller/controller.py
 988b548f6578adf9cec17afdeee8291c  lib/controller/handler.py
 1e5532ede194ac9c083891c2f02bca93  lib/controller/__init__.py
 cb865cf6eff60118bc97a0f106af5e4d  lib/core/agent.py
@@ -42,14 +42,14 @@ cada93357a7321655927fc9625b3bfec  lib/core/exception.py
 1e5532ede194ac9c083891c2f02bca93  lib/core/__init__.py
 458a194764805cd8312c14ecd4be4d1e  lib/core/log.py
 7d6edc552e08c30f4f4d49fa93b746f1  lib/core/optiondict.py
-a24992df012aee6d5617808f1dbb70ec  lib/core/option.py
+7dacc178910ab4d57de36c3602bde17d  lib/core/option.py
 c8c386d644d57c659d74542f5f57f632  lib/core/patch.py
 6783160150b4711d02c56ee2beadffdb  lib/core/profiling.py
 6f654e1715571eff68a0f8af3d62dcf8  lib/core/readlineng.py
 0c3eef46bdbf87e29a3f95f90240d192  lib/core/replication.py
 a7db43859b61569b601b97f187dd31c5  lib/core/revision.py
 fcb74fcc9577523524659ec49e2e964b  lib/core/session.py
-c0d7976aabdffc78b22a9e63f3a51683  lib/core/settings.py
+9f209388d9fed41480e57c8574d0111a  lib/core/settings.py
 a971ce157d04de96ba6e710d3d38a9a8  lib/core/shell.py
 a7edc9250d13af36ac0108f259859c19  lib/core/subprocessng.py
 52642badbbca4c31a2fcdd754d67a983  lib/core/target.py
@@ -71,7 +71,7 @@ f6b5957bf2103c3999891e4f45180bce  lib/parse/payloads.py
 30eed3a92a04ed2c29770e1b10d39dc0  lib/request/basicauthhandler.py
 2b81435f5a7519298c15c724e3194a0d  lib/request/basic.py
 859b6ad583e0ffba154f17ee179b5b89  lib/request/comparison.py
-0113525b321d0d35cf973a9cff34850a  lib/request/connect.py
+77b24c30b1a2163add76652998e74127  lib/request/connect.py
 dd4598675027fae99f2e2475b05986da  lib/request/direct.py
 2044fce3f4ffa268fcfaaf63241b1e64  lib/request/dns.py
 98535d0efca5551e712fcc4b34a3f772  lib/request/httpshandler.py