--rewrite-host-header flag

Author: abhinavsingh

README.md

@@ -1075,7 +1075,7 @@ following `Nginx` config:
 
 ```console
 location /get {
-    proxy_pass http://httpbin.org/get
+    proxy_pass http://httpbin.org/get;
 }
 ```
 
@@ -1094,6 +1094,36 @@ Verify using `curl -v localhost:8899/get`:
 }
 ```
 
+#### Rewrite Host Header
+
+With above example, you may sometimes see:
+
+```console
+>
+* Empty reply from server
+* Closing connection
+curl: (52) Empty reply from server
+```
+
+This is happenening because our default reverse proxy plugin `ReverseProxyPlugin` is configured
+with a `http` and a `https` upstream server.  And, by default `ReverseProxyPlugin` preserves the
+original host header.  While this works with `https` upstreams, this doesn't work reliably with
+`http` upstreams.  To work around this problem use the `--rewrite-host-header` flags.
+
+Example:
+
+
+```console
+❯ proxy --enable-reverse-proxy \
+    --plugins proxy.plugin.ReverseProxyPlugin \
+    --rewrite-host-header
+```
+
+This will ensure that `Host` header field is set as `httpbin.org` and works with both `http` and
+`https` upstreams.
+
+> NOTE: Whether to use `--rewrite-host-header` or not depends upon your use-case.
+
 ## Plugin Ordering
 
 When using multiple plugins, depending upon plugin functionality,
@@ -2613,7 +2643,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT]
           [--proxy-pool PROXY_POOL] [--enable-web-server]
           [--enable-static-server] [--static-server-dir STATIC_SERVER_DIR]
           [--min-compression-length MIN_COMPRESSION_LENGTH]
-          [--enable-reverse-proxy] [--enable-metrics]
+          [--enable-reverse-proxy] [--rewrite-host-header] [--enable-metrics]
           [--metrics-path METRICS_PATH] [--pac-file PAC_FILE]
           [--pac-file-url-path PAC_FILE_URL_PATH]
           [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]
@@ -2622,7 +2652,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT]
           [--filtered-client-ips FILTERED_CLIENT_IPS]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
 
-proxy.py v2.4.6.dev25+g2754b928.d20240812
+proxy.py v2.4.8.dev8+gc703edac.d20241013
 
 options:
   -h, --help            show this help message and exit
@@ -2791,6 +2821,9 @@ options:
                         response that will be compressed (gzipped).
   --enable-reverse-proxy
                         Default: False. Whether to enable reverse proxy core.
+  --rewrite-host-header
+                        Default: False. If used, reverse proxy server will
+                        rewrite Host header field before sending to upstream.
   --enable-metrics      Default: False. Enables metrics.
   --metrics-path METRICS_PATH
                         Default: /metrics. Web server path to serve proxy.py

proxy/common/constants.py

@@ -165,6 +165,7 @@ def _env_threadless_compliant() -> bool:
     if sys.version_info >= (3, 10)
     else (ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1)
 )
+DEFAULT_ENABLE_REWRITE_HOST = False
 
 DEFAULT_DEVTOOLS_DOC_URL = 'http://proxy'
 DEFAULT_DEVTOOLS_FRAME_ID = secrets.token_hex(8)

proxy/common/utils.py

@@ -25,7 +25,8 @@
 from types import TracebackType
 from typing import Any, Dict, List, Type, Tuple, Callable, Optional
 
-import _ssl  # noqa: WPS436
+# noqa: WPS436
+import _ssl  # type: ignore[import-not-found]
 
 from .types import HostPort
 from .constants import (
@@ -322,6 +323,7 @@ def set_open_file_limit(soft_limit: int) -> None:
     if IS_WINDOWS:  # pragma: no cover
         return
 
+    # pylint: disable=possibly-used-before-assignment
     curr_soft_limit, curr_hard_limit = resource.getrlimit(
         resource.RLIMIT_NOFILE,
     )

proxy/http/server/reverse.py

@@ -128,11 +128,15 @@ def handle_request(self, request: HttpParser) -> None:
                 self.upstream.queue(
                     memoryview(
                         request.build(
-                            host=self.choice.hostname
-                            + (
-                                COLON + bytes_(self.choice.port)
-                                if self.choice.port is not None
-                                else b''
+                            host=(
+                                self.choice.hostname
+                                + (
+                                    COLON + bytes_(self.choice.port)
+                                    if self.choice.port is not None
+                                    else b''
+                                )
+                                if self.flags.rewrite_host_header
+                                else None
                             ),
                         ),
                     ),

proxy/http/server/web.py

@@ -29,8 +29,9 @@
 from ...common.utils import text_, build_websocket_handshake_response
 from ...common.constants import (
     DEFAULT_ENABLE_WEB_SERVER, DEFAULT_STATIC_SERVER_DIR,
-    DEFAULT_ENABLE_REVERSE_PROXY, DEFAULT_ENABLE_STATIC_SERVER,
-    DEFAULT_WEB_ACCESS_LOG_FORMAT, DEFAULT_MIN_COMPRESSION_LENGTH,
+    DEFAULT_ENABLE_REWRITE_HOST, DEFAULT_ENABLE_REVERSE_PROXY,
+    DEFAULT_ENABLE_STATIC_SERVER, DEFAULT_WEB_ACCESS_LOG_FORMAT,
+    DEFAULT_MIN_COMPRESSION_LENGTH,
 )
 
 
@@ -78,6 +79,16 @@
     help='Default: False.  Whether to enable reverse proxy core.',
 )
 
+flags.add_argument(
+    '--rewrite-host-header',
+    action='store_true',
+    default=DEFAULT_ENABLE_REWRITE_HOST,
+    help='Default: '
+    + str(DEFAULT_ENABLE_REWRITE_HOST)
+    + '.  '
+    + 'If used, reverse proxy server will rewrite Host header field before sending to upstream.',
+)
+
 
 class HttpWebServerPlugin(HttpProtocolHandlerPlugin):
     """HttpProtocolHandler plugin which handles incoming requests to local web server."""