update dependecy versions (#8422)

DESCRIPTION: 
🚀 Motivation

- Upgrade mitmproxy (12.2.1) to address CVEs and enable downstream
dependency refreshes tied to security bulletins.
- Refresh Python packages used by the regression proxy harness, so they
remain compatible with the upgraded mitmproxy API and current patch
levels.
- Keep the [check-pytest] infrastructure resilient on both SSL-enabled
and SSL-less PostgreSQL builds encountered in CI and local development.

🔄 Summary of Changes

1. Mitmproxy Upgrade & Harness Adaptation

- Bumped mitmproxy to 12.2.1, which replaces the legacy TCP mode with
the new reverse:tcp:// listener and requires explicit connection
strategy flags.
- Updated pg_regress_multi.pl and the Python proxy scripts to:
-- Use the reverse:tcp:// endpoint format.
-- Pass [--set connection_strategy=lazy]so new connections are accepted
after mitmproxy’s change in default behavior.
-- Handle ResetHandler cleanup differences introduced in mitmproxy 12
(guard when the client socket is already closed).
- Adjusted columnar tests to accept the new psycopg EOF message variants
surfaced once mitmproxy’s forwarding changed, ensuring the failure
expectations still match.

2. Python Dependency Refresh
- Synchronized requirements.txt/Pipenv lockfiles to versions compatible
with mitmproxy 12 and recent security patches.
- Ensured pytest plugins referenced in regression tests remain up to
date (pytest 9, pytest-xdist 3.8, etc.), matching our supported Python
3.12 toolchain.

3. Pytest Harness Hardening

- Added runtime detection in [common.py] to determine whether PostgreSQL
was compiled with SSL ([pg_config --configure]
- If SSL support is absent (as in our CI images), we now skip writing
ssl = on into postgresql.conf and omit hostssl entries in pg_hba.conf.
- This prevents pg_ctl from failing with “SSL is not supported by this
build” when the tests restart cluster nodes.

Author: ihalatci

.devcontainer/src/test/regress/Pipfile

@@ -4,10 +4,19 @@ url = "https://pypi.python.org/simple"
 verify_ssl = true
 
 [packages]
-mitmproxy = {editable = true, ref = "main", git = "https://github.com/citusdata/mitmproxy.git"}
+mitmproxy = {git = "https://github.com/citusdata/mitmproxy.git", ref = "main"}
+"aioquic" = ">=1.2.0,<1.3.0"
+"mitmproxy-rs" = ">=0.12.6,<0.13.0"
+argon2-cffi = ">=23.1.0"
+bcrypt = ">=4.1.2"
+brotli = "<=1.2.0"
+h11 = "==0.16.0"
+h2 = "==4.3.0"
+tornado = ">=6.5.1,<6.6.0"
+zstandard = ">=0.25.0"
 construct = "*"
 docopt = "==0.6.2"
-cryptography = ">=41.0.4"
+cryptography = "==44.0.3"
 pytest = "*"
 psycopg = "*"
 filelock = "*"
@@ -16,13 +25,15 @@ pytest-timeout = "*"
 pytest-xdist = "*"
 pytest-repeat = "*"
 pyyaml = "*"
-werkzeug = "==3.0.6"
+werkzeug = "==3.1.0"
+"typing-extensions" = ">=4.13.2,<5"
+pyperclip = "==1.9.0"
 
 [dev-packages]
-black = "*"
+black = "==24.10.0"
 isort = "*"
 flake8 = "*"
 flake8-bugbear = "*"
 
 [requires]
-python_version = "3.9"
+python_version = "3.12"

.devcontainer/src/test/regress/Pipfile.lock

@@ -32,7 +32,7 @@ jobs:
       style_checker_image_name: "ghcr.io/citusdata/stylechecker"
       style_checker_tools_version: "0.8.33"
       sql_snapshot_pg_version: "18.1"
-      image_suffix: "-v15cd08f"
+      image_suffix: "-vaa2c190"
       pg16_version: '{ "major": "16", "full": "16.11" }'
       pg17_version: '{ "major": "17", "full": "17.7" }'
       pg18_version: '{ "major": "18", "full": "18.1" }'

.github/workflows/build_and_test.yml

@@ -4,10 +4,19 @@ url = "https://pypi.python.org/simple"
 verify_ssl = true
 
 [packages]
-mitmproxy = {editable = true, ref = "main", git = "https://github.com/citusdata/mitmproxy.git"}
+mitmproxy = {git = "https://github.com/citusdata/mitmproxy.git", ref = "main"}
+"aioquic" = ">=1.2.0,<1.3.0"
+"mitmproxy-rs" = ">=0.12.6,<0.13.0"
+argon2-cffi = ">=23.1.0"
+bcrypt = ">=4.1.2"
+brotli = "<=1.2.0"
+h11 = "==0.16.0"
+h2 = "==4.3.0"
+tornado = ">=6.5.1,<6.6.0"
+zstandard = ">=0.25.0"
 construct = "*"
 docopt = "==0.6.2"
-cryptography = ">=41.0.4"
+cryptography = "==44.0.3"
 pytest = "*"
 psycopg = "*"
 filelock = "*"
@@ -16,13 +25,15 @@ pytest-timeout = "*"
 pytest-xdist = "*"
 pytest-repeat = "*"
 pyyaml = "*"
-werkzeug = "==3.0.6"
+werkzeug = "==3.1.0"
+"typing-extensions" = ">=4.13.2,<5"
+pyperclip = "==1.9.0"
 
 [dev-packages]
-black = "*"
+black = "==24.10.0"
 isort = "*"
 flake8 = "*"
 flake8-bugbear = "*"
 
 [requires]
-python_version = "3.9"
+python_version = "3.12"

src/test/regress/Pipfile

@@ -74,6 +74,9 @@ def capture(command, *args, **kwargs):
 
 
 PG_CONFIG = os.environ.get("PG_CONFIG", "pg_config")
+PG_CONFIG_ARGS = capture([PG_CONFIG, "--configure"], shell=False).rstrip()
+PG_SUPPORTS_SSL = "--with-ssl" in PG_CONFIG_ARGS or "--with-openssl" in PG_CONFIG_ARGS
+
 PG_BINDIR = capture([PG_CONFIG, "--bindir"], shell=False).rstrip()
 os.environ["PATH"] = PG_BINDIR + os.pathsep + os.environ["PATH"]
 
@@ -850,7 +853,8 @@ def initdb(self):
             pgconf.write("restart_after_crash = off\n")
 
         os.truncate(self.hba_path, 0)
-        self.ssl_access("all", "trust")
+        if PG_SUPPORTS_SSL:
+            self.ssl_access("all", "trust")
         self.nossl_access("all", "trust")
         self.commit_hba()
 
@@ -859,11 +863,12 @@ def init_with_citus(self):
         self.start()
         self.sql("CREATE EXTENSION citus")
 
-        # Manually turn on ssl, so that we can safely truncate
-        # postgresql.auto.conf later. We can only do this after creating the
-        # citus extension because that creates the self signed certificates.
-        with self.conf_path.open(mode="a") as pgconf:
-            pgconf.write("ssl = on\n")
+        if PG_SUPPORTS_SSL:
+            # Manually turn on ssl, so that we can safely truncate
+            # postgresql.auto.conf later. We can only do this after creating the
+            # citus extension because that creates the self signed certificates.
+            with self.conf_path.open(mode="a") as pgconf:
+                pgconf.write("ssl = on\n")
 
     def pgctl(self, command, **kwargs):
         run(f"pg_ctl -w --pgdata {self.pgdata} {command}", **kwargs)

src/test/regress/Pipfile.lock

@@ -71,7 +71,12 @@ def test_recovery(coord):
     # test crashing while having an open transaction
     with pytest.raises(
         psycopg.OperationalError,
-        match="server closed the connection unexpectedly|consuming input failed: EOF detected",
+        match=(
+            "server closed the connection unexpectedly|"
+            "consuming input failed: EOF detected|"
+            "SSL SYSCALL error: EOF detected|"
+            "SSL error: unexpected eof while reading"
+        ),
     ):
         with coord.transaction() as cur:
             cur.execute(
@@ -88,7 +93,12 @@ def test_recovery(coord):
     # test crashing while having a prepared transaction
     with pytest.raises(
         psycopg.OperationalError,
-        match="server closed the connection unexpectedly|consuming input failed: EOF detected",
+        match=(
+            "server closed the connection unexpectedly|"
+            "consuming input failed: EOF detected|"
+            "consuming input failed: SSL SYSCALL error: EOF detected|"
+            "SSL error: unexpected eof while reading"
+        ),
     ):
         with coord.transaction() as cur:
             cur.execute(

src/test/regress/citus_tests/common.py

@@ -157,22 +157,27 @@ def _handle(self, flow, message):
         flow.kill()  # tell mitmproxy this connection should be closed
 
         # this is a mitmproxy.connections.ClientConnection(mitmproxy.tcp.BaseHandler)
-        client_conn = flow.client_conn
-        # this is a regular socket object
-        conn = client_conn.connection
-
-        # cause linux to send a RST
-        LINGER_ON, LINGER_TIMEOUT = 1, 0
-        conn.setsockopt(
-            socket.SOL_SOCKET,
-            socket.SO_LINGER,
-            struct.pack("ii", LINGER_ON, LINGER_TIMEOUT),
-        )
-        conn.close()
-
-        # closing the connection isn't ideal, this thread later crashes when mitmproxy
-        # tries to call conn.shutdown(), but there's nothing else to clean up so that's
-        # maybe okay
+        client_conn = getattr(flow, "client_conn", None)
+
+        # this is a regular socket object on mitmproxy versions < 12.2.
+        # Newer releases no longer expose the raw socket via the "connection"
+        # attribute, so guard access accordingly.
+        conn = getattr(client_conn, "connection", None)
+
+        if conn is not None:
+            # mitmproxy < 12 exposed the raw socket so we could force a TCP RST. Keep
+            # that behaviour for older versions to ensure tests stay reproducible.
+            LINGER_ON, LINGER_TIMEOUT = 1, 0
+            conn.setsockopt(
+                socket.SOL_SOCKET,
+                socket.SO_LINGER,
+                struct.pack("ii", LINGER_ON, LINGER_TIMEOUT),
+            )
+            conn.close()
+        else:
+            # mitmproxy >= 12 hides the socket; flow.kill() already tears the
+            # connection down, so there's nothing additional we can do here.
+            pass
 
         return "done"
 
@@ -460,7 +465,7 @@ def tcp_message(flow: tcp.TCPFlow):
     This callback is hit every time mitmproxy receives a packet. It's the main entrypoint
     into this script.
     """
-    global connection_count
+    global connection_count  # noqa: F824
 
     tcp_msg = flow.messages[-1]
 

src/test/regress/citus_tests/test/test_columnar.py

@@ -914,7 +914,7 @@ ()
   if ($mitmPid eq 0) {
     print("forked, about to exec mitmdump\n");
     setpgrp(0,0); # we're about to spawn both a shell and a mitmdump, kill them as a group
-    exec("mitmdump --rawtcp -p $mitmPort --mode reverse:localhost:57638 -s $regressdir/mitmscripts/fluent.py --set fifo=$mitmFifoPath --set flow_detail=0 --set termlog_verbosity=warn >proxy.output 2>&1");
+    exec("mitmdump --rawtcp -p $mitmPort --mode reverse:tcp://localhost:57638 -s $regressdir/mitmscripts/fluent.py --set fifo=$mitmFifoPath --set flow_detail=0 --set termlog_verbosity=warn --set connection_strategy=lazy >proxy.output 2>&1");
     die 'could not start mitmdump';
   }
 }