Ability to override --data-dir for scenarios when proxy.py is running as a user with no home directory (#1389)

* Ability to override `--data-dir` for scenarios when `proxy.py` is running as a user with no home directory

* Single quotes

* Update expected tar.gz name and Default to `ms-python.black-formatter` in vscode settings

* Fix tests for Python 3.6 and 3.7

* Updated README

Author: abhinavsingh

.github/workflows/test-library.yml

@@ -188,7 +188,7 @@ jobs:
     - name: Set the expected dist artifact names
       id: artifact-name
       run: |
-        print('::set-output name=sdist::proxy.py-${{
+        print('::set-output name=sdist::proxy_py-${{
             steps.request-check.outputs.release-requested == 'true'
             && github.event.inputs.release-version
             || steps.scm-version.outputs.dist-version

.gitignore

@@ -36,3 +36,6 @@ build
 
 pyreverse.png
 profile.svg
+
+*-pre-push
+jaxl-api-credentials*.json

.vscode/settings.json

@@ -19,7 +19,7 @@
   "typescript.preferences.quoteStyle": "single",
   "[python]": {
     "editor.wordBasedSuggestions": "matchingDocuments",
-    "editor.defaultFormatter": null
+    "editor.defaultFormatter": "ms-python.black-formatter"
   },
   "python.testing.unittestEnabled": false,
   "python.testing.autoTestDiscoverOnSaveEnabled": true,

README.md

@@ -2357,8 +2357,9 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT]
           [--plugins PLUGINS [PLUGINS ...]] [--enable-dashboard]
           [--basic-auth BASIC_AUTH] [--enable-ssh-tunnel]
           [--work-klass WORK_KLASS] [--pid-file PID_FILE] [--openssl OPENSSL]
-          [--enable-proxy-protocol] [--enable-conn-pool] [--key-file KEY_FILE]
-          [--cert-file CERT_FILE] [--client-recvbuf-size CLIENT_RECVBUF_SIZE]
+          [--data-dir DATA_DIR] [--enable-proxy-protocol] [--enable-conn-pool]
+          [--key-file KEY_FILE] [--cert-file CERT_FILE]
+          [--client-recvbuf-size CLIENT_RECVBUF_SIZE]
           [--server-recvbuf-size SERVER_RECVBUF_SIZE]
           [--max-sendbuf-size MAX_SENDBUF_SIZE] [--timeout TIMEOUT]
           [--disable-http-proxy] [--disable-headers DISABLE_HEADERS]
@@ -2378,7 +2379,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT]
           [--filtered-client-ips FILTERED_CLIENT_IPS]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
 
-proxy.py v2.4.4rc6.dev11+gac1f05d7.d20240413
+proxy.py v2.4.4rc6.dev85+g9335918b
 
 options:
   -h, --help            show this help message and exit
@@ -2459,6 +2460,7 @@ options:
   --pid-file PID_FILE   Default: None. Save "parent" process ID to a file.
   --openssl OPENSSL     Default: openssl. Path to openssl binary. By default,
                         assumption is that openssl is in your PATH.
+  --data-dir DATA_DIR   Default: ~/.proxypy. Path to proxypy data directory.
   --enable-proxy-protocol
                         Default: False. If used, will enable proxy protocol.
                         Only version 1 is currently supported.
@@ -2501,9 +2503,9 @@ options:
                         Default: None. Signing certificate to use for signing
                         dynamically generated HTTPS certificates. If used,
                         must also pass --ca-key-file and --ca-signing-key-file
-  --ca-file CA_FILE     Default: /Users/abhinavsingh/Dev/proxy.py/.venv3118/li
-                        b/python3.11/site-packages/certifi/cacert.pem. Provide
-                        path to custom CA bundle for peer certificate
+  --ca-file CA_FILE     Default: /Users/abhinavsingh/Dev/proxy.py/.venv31010/l
+                        ib/python3.10/site-packages/certifi/cacert.pem.
+                        Provide path to custom CA bundle for peer certificate
                         verification
   --ca-signing-key-file CA_SIGNING_KEY_FILE
                         Default: None. CA signing key to use for dynamic

proxy/common/constants.py

@@ -47,6 +47,8 @@ def _env_threadless_compliant() -> bool:
 DOT = b'.'
 SLASH = b'/'
 AT = b'@'
+AND = b'&'
+EQUAL = b'='
 HTTP_PROTO = b'http'
 HTTPS_PROTO = HTTP_PROTO + b's'
 HTTP_1_0 = HTTP_PROTO.upper() + SLASH + b'1.0'

proxy/common/flag.py

@@ -381,7 +381,13 @@ def initialize(
             ),
         )
 
-        args.proxy_py_data_dir = DEFAULT_DATA_DIRECTORY_PATH
+        args.proxy_py_data_dir = cast(
+            str,
+            opts.get(
+                'data_dir',
+                args.data_dir or DEFAULT_DATA_DIRECTORY_PATH,
+            ),
+        )
         os.makedirs(args.proxy_py_data_dir, exist_ok=True)
 
         ca_cert_dir = opts.get('ca_cert_dir', args.ca_cert_dir)

proxy/proxy.py

@@ -145,6 +145,13 @@
     'By default, assumption is that openssl is in your PATH.',
 )
 
+flags.add_argument(
+    '--data-dir',
+    type=str,
+    default=None,
+    help='Default: ~/.proxypy. Path to proxypy data directory.',
+)
+
 
 class Proxy:
     """Proxy is a context manager to control proxy.py library core.

requirements-tunnel.txt

@@ -1,3 +1,5 @@
-paramiko==2.11.0
-types-paramiko==2.11.3
+paramiko==2.11.0; python_version < '3.11'
+paramiko==3.4.0; python_version >= '3.11'
+types-paramiko==2.11.3; python_version < '3.11'
+types-paramiko==3.4.0.20240311; python_version >= '3.11'
 cryptography==36.0.2; python_version <= '3.6'

tests/test_main.py

@@ -33,9 +33,9 @@
     DEFAULT_ENABLE_SSH_TUNNEL, DEFAULT_ENABLE_WEB_SERVER,
     DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_WEBSOCKET_TRANSPORT,
     DEFAULT_CA_SIGNING_KEY_FILE, DEFAULT_CLIENT_RECVBUF_SIZE,
-    DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CACHE_DIRECTORY_PATH,
-    DEFAULT_ENABLE_REVERSE_PROXY, DEFAULT_ENABLE_STATIC_SERVER,
-    _env_threadless_compliant,
+    DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_SERVER_RECVBUF_SIZE,
+    DEFAULT_CACHE_DIRECTORY_PATH, DEFAULT_ENABLE_REVERSE_PROXY,
+    DEFAULT_ENABLE_STATIC_SERVER, _env_threadless_compliant,
 )
 
 
@@ -83,6 +83,7 @@ def mock_default_args(mock_args: mock.Mock) -> None:
         mock_args.enable_ssh_tunnel = DEFAULT_ENABLE_SSH_TUNNEL
         mock_args.enable_reverse_proxy = DEFAULT_ENABLE_REVERSE_PROXY
         mock_args.unix_socket_path = None
+        mock_args.data_dir = DEFAULT_DATA_DIRECTORY_PATH
         mock_args.cache_dir = DEFAULT_CACHE_DIRECTORY_PATH
 
     @mock.patch('os.remove')