Pass separate --ca-cert-dir flag for parallel TLS interception tests (#984)

* Pass separate `--ca-cert-dir` flag for parallel TLS interception tests

* Temp disable `test_modify_post_response_integration`

* mock ca cert dir

* Is threaded an issue with TLS interception?

* Disable modify chunk response for python < 3.10

* Disable modify chunk response for python < 3.10

Author: abhinavsingh

proxy/common/flag.py

@@ -379,7 +379,7 @@ def initialize(
             args.ca_cert_dir = os.path.join(
                 args.proxy_py_data_dir, 'certificates',
             )
-            os.makedirs(args.ca_cert_dir, exist_ok=True)
+        os.makedirs(args.ca_cert_dir, exist_ok=True)
 
         return args
 

tests/integration/test_integration.py

@@ -10,6 +10,7 @@
 
     Test the simplest proxy use scenario for smoke.
 """
+import sys
 import time
 import pytest
 import tempfile
@@ -89,7 +90,9 @@ def proxy_py_subprocess(request: Any) -> Generator[int, None, None]:
 
     After the testing is over, tear it down.
     """
-    port_file = Path(tempfile.gettempdir()) / 'proxy.port'
+    temp_dir = Path(tempfile.gettempdir())
+    port_file = temp_dir / 'proxy.port'
+    ca_cert_dir = temp_dir / ('certificates-%s' % int(time.time()))
     proxy_cmd = (
         'python', '-m', 'proxy',
         '--hostname', '127.0.0.1',
@@ -98,6 +101,8 @@ def proxy_py_subprocess(request: Any) -> Generator[int, None, None]:
         '--enable-web-server',
         '--num-acceptors', '3',
         '--num-workers', '3',
+        '--ca-cert-dir', str(ca_cert_dir),
+        '--log-level', 'd',
     ) + tuple(request.param.split())
     proxy_proc = Popen(proxy_cmd)
     # Needed because port file might not be available immediately
@@ -148,21 +153,25 @@ def test_integration_with_interception_flags(proxy_py_subprocess: int) -> None:
     check_output([str(shell_script_test), str(proxy_py_subprocess)])
 
 
-# @pytest.mark.smoke  # type: ignore[misc]
-# @pytest.mark.parametrize(
-#     'proxy_py_subprocess',
-#     PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN,
-#     indirect=True,
-# )   # type: ignore[misc]
-# @pytest.mark.skipif(
-#     IS_WINDOWS,
-#     reason='OSError: [WinError 193] %1 is not a valid Win32 application',
-# )  # type: ignore[misc]
-# def test_modify_chunk_response_integration(proxy_py_subprocess: int) -> None:
-#     """An acceptance test for :py:class:`~proxy.plugin.ModifyChunkResponsePlugin`
-#     interception using ``curl`` through proxy.py."""
-#     shell_script_test = Path(__file__).parent / 'test_modify_chunk_response.sh'
-#     check_output([str(shell_script_test), str(proxy_py_subprocess)])
+@pytest.mark.smoke  # type: ignore[misc]
+@pytest.mark.parametrize(
+    'proxy_py_subprocess',
+    PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN,
+    indirect=True,
+)   # type: ignore[misc]
+@pytest.mark.skipif(
+    IS_WINDOWS,
+    reason='OSError: [WinError 193] %1 is not a valid Win32 application',
+)  # type: ignore[misc]
+@pytest.mark.skipif(
+    sys.version_info < (3, 10),
+    reason='For version < 3.10, GHA integration run into OSError when flushing to clients',
+)  # type: ignore[misc]
+def test_modify_chunk_response_integration(proxy_py_subprocess: int) -> None:
+    """An acceptance test for :py:class:`~proxy.plugin.ModifyChunkResponsePlugin`
+    interception using ``curl`` through proxy.py."""
+    shell_script_test = Path(__file__).parent / 'test_modify_chunk_response.sh'
+    check_output([str(shell_script_test), str(proxy_py_subprocess)])
 
 
 @pytest.mark.smoke  # type: ignore[misc]

tests/integration/test_modify_chunk_response.sh

@@ -76,8 +76,7 @@ plugin
 EOM
 
 echo "[Test ModifyChunkResponsePlugin]"
-CMD="curl -v -x $PROXY_URL --cacert ca-cert.pem https://httpbin.org/stream/5"
-RESPONSE=$($CMD 2> /dev/null)
+RESPONSE=$(curl -v -x $PROXY_URL --cacert ca-cert.pem https://httpbin.org/stream/5 2> /dev/null)
 verify_response "$RESPONSE" "$MODIFIED_CHUNK_RESPONSE"
 VERIFIED1=$?
 

tests/integration/test_modify_post_data.sh

@@ -51,23 +51,6 @@ while true; do
     sleep 1
 done
 
-verify_response() {
-    if [ "$1" == "" ];
-    then
-        echo "Empty response";
-        return 1;
-    else
-        if [ "$1" == "$2" ];
-        then
-            echo "Ok";
-            return 0;
-        else
-            echo "Invalid response: '$1', expected: '$2'";
-            return 1;
-        fi
-    fi;
-}
-
 verify_contains() {
     if [ "$1" == "" ];
     then

tests/test_main.py

@@ -17,7 +17,7 @@
 from proxy.proxy import main, entry_point
 from proxy.common.utils import bytes_
 from proxy.common.constants import (  # noqa: WPS450
-    DEFAULT_PORT, DEFAULT_PLUGINS, DEFAULT_TIMEOUT, DEFAULT_KEY_FILE,
+    DEFAULT_CA_CERT_DIR, DEFAULT_PORT, DEFAULT_PLUGINS, DEFAULT_TIMEOUT, DEFAULT_KEY_FILE,
     DEFAULT_LOG_FILE, DEFAULT_PAC_FILE, DEFAULT_PID_FILE, PLUGIN_DASHBOARD,
     DEFAULT_CERT_FILE, DEFAULT_LOG_LEVEL, DEFAULT_PORT_FILE, PLUGIN_HTTP_PROXY,
     PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER, DEFAULT_BASIC_AUTH,
@@ -45,6 +45,7 @@ def mock_default_args(mock_args: mock.Mock) -> None:
         mock_args.ca_key_file = DEFAULT_CA_KEY_FILE
         mock_args.ca_cert_file = DEFAULT_CA_CERT_FILE
         mock_args.ca_signing_key_file = DEFAULT_CA_SIGNING_KEY_FILE
+        mock_args.ca_cert_dir = DEFAULT_CA_CERT_DIR
         mock_args.pid_file = DEFAULT_PID_FILE
         mock_args.log_file = DEFAULT_LOG_FILE
         mock_args.log_level = DEFAULT_LOG_LEVEL