SNOW-926289 updating vendored libraries (#1793)

sfc-gh-mkeller · sfc-gh-stan · web-flow · commit 4f46237d6bbd · 2023-11-07T16:52:10.000-05:00
* updating vendored requests to 2.31.0

* updating vendored urllib3 to 1.26.18

* putting back urllib3 pin

* adding changelog entry

---------

Co-authored-by: Sophie Tan &lt;sophie.tan@snowflake.com&gt;
diff --git a/DESCRIPTION.md b/DESCRIPTION.md
@@ -9,6 +9,11 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne
 # Release Notes
 
 
+- v3.4.1(TBD)
+
+  - Bumped vendored `urllib3` to 1.26.18
+  - Bumped vendored `requests` to 2.31.0
+
 - v3.4.0(November 03,2023)
 
   - Added support for `use_logical_type` in `write_pandas`.
diff --git a/setup.cfg b/setup.cfg
@@ -53,7 +53,7 @@ install_requires =
     packaging
     charset_normalizer>=2,<4
     idna>=2.5,<4
-    urllib3>=1.21.1,<1.27
+    urllib3>=1.21.1,<2.0.0
     certifi>=2017.4.17
     typing_extensions>=4.3,<5
     filelock>=3.5,<4
diff --git a/src/snowflake/connector/vendored/requests/__init__.py b/src/snowflake/connector/vendored/requests/__init__.py
@@ -66,10 +66,10 @@ def check_compatibility(urllib3_version, chardet_version, charset_normalizer_ver
     # Check urllib3 for compatibility.
     major, minor, patch = urllib3_version  # noqa: F811
     major, minor, patch = int(major), int(minor), int(patch)
-    # urllib3 >= 1.21.1, <= 1.26
-    assert major == 1
-    assert minor >= 21
-    assert minor <= 26
+    # urllib3 >= 1.21.1
+    assert major >= 1
+    if major == 1:
+        assert minor >= 21
 
     # Check charset_normalizer for compatibility.
     if chardet_version:
diff --git a/src/snowflake/connector/vendored/requests/__version__.py b/src/snowflake/connector/vendored/requests/__version__.py
@@ -5,8 +5,8 @@
 __title__ = "requests"
 __description__ = "Python HTTP for Humans."
 __url__ = "https://requests.readthedocs.io"
-__version__ = "2.29.0"
-__build__ = 0x022900
+__version__ = "2.31.0"
+__build__ = 0x023100
 __author__ = "Kenneth Reitz"
 __author_email__ = "me@kennethreitz.org"
 __license__ = "Apache 2.0"
diff --git a/src/snowflake/connector/vendored/requests/adapters.py b/src/snowflake/connector/vendored/requests/adapters.py
@@ -193,7 +193,6 @@ def init_poolmanager(
             num_pools=connections,
             maxsize=maxsize,
             block=block,
-            strict=True,
             **pool_kwargs,
         )
 
diff --git a/src/snowflake/connector/vendored/requests/sessions.py b/src/snowflake/connector/vendored/requests/sessions.py
@@ -324,7 +324,9 @@ def rebuild_proxies(self, prepared_request, proxies):
         except KeyError:
             username, password = None, None
 
-        if username and password:
+        # urllib3 handles proxy authorization for us in the standard adapter.
+        # Avoid appending this to TLS tunneled requests where it may be leaked.
+        if not scheme.startswith('https') and username and password:
             headers["Proxy-Authorization"] = _basic_auth_str(username, password)
 
         return new_proxies
diff --git a/src/snowflake/connector/vendored/urllib3/_collections.py b/src/snowflake/connector/vendored/urllib3/_collections.py
@@ -268,6 +268,24 @@ def getlist(self, key, default=__marker):
         else:
             return vals[1:]
 
+    def _prepare_for_method_change(self):
+        """
+        Remove content-specific header fields before changing the request
+        method to GET or HEAD according to RFC 9110, Section 15.4.
+        """
+        content_specific_headers = [
+            "Content-Encoding",
+            "Content-Language",
+            "Content-Location",
+            "Content-Type",
+            "Content-Length",
+            "Digest",
+            "Last-Modified",
+        ]
+        for header in content_specific_headers:
+            self.discard(header)
+        return self
+
     # Backwards compatibility for httplib
     getheaders = getlist
     getallmatchingheaders = getlist
diff --git a/src/snowflake/connector/vendored/urllib3/_version.py b/src/snowflake/connector/vendored/urllib3/_version.py
@@ -1,2 +1,2 @@
 # This file is protected via CODEOWNERS
-__version__ = "1.26.15"
+__version__ = "1.26.18"
diff --git a/src/snowflake/connector/vendored/urllib3/connectionpool.py b/src/snowflake/connector/vendored/urllib3/connectionpool.py
@@ -9,6 +9,7 @@
 from socket import error as SocketError
 from socket import timeout as SocketTimeout
 
+from ._collections import HTTPHeaderDict
 from .connection import (
     BaseSSLError,
     BrokenPipeError,
@@ -50,6 +51,13 @@
 from .util.url import _normalize_host as normalize_host
 from .util.url import get_host, parse_url
 
+try:  # Platform-specific: Python 3
+    import weakref
+
+    weakref_finalize = weakref.finalize
+except AttributeError:  # Platform-specific: Python 2
+    from .packages.backports.weakref_finalize import weakref_finalize
+
 xrange = six.moves.xrange
 
 log = logging.getLogger(__name__)
@@ -220,6 +228,16 @@ def __init__(
             self.conn_kw["proxy"] = self.proxy
             self.conn_kw["proxy_config"] = self.proxy_config
 
+        # Do not pass 'self' as callback to 'finalize'.
+        # Then the 'finalize' would keep an endless living (leak) to self.
+        # By just passing a reference to the pool allows the garbage collector
+        # to free self if nobody else has a reference to it.
+        pool = self.pool
+
+        # Close all the HTTPConnections in the pool before the
+        # HTTPConnectionPool object is garbage collected.
+        weakref_finalize(self, _close_pool_connections, pool)
+
     def _new_conn(self):
         """
         Return a fresh :class:`HTTPConnection`.
@@ -489,14 +507,8 @@ def close(self):
         # Disable access to the pool
         old_pool, self.pool = self.pool, None
 
-        try:
-            while True:
-                conn = old_pool.get(block=False)
-                if conn:
-                    conn.close()
-
-        except queue.Empty:
-            pass  # Done.
+        # Close all the HTTPConnections in the pool.
+        _close_pool_connections(old_pool)
 
     def is_same_host(self, url):
         """
@@ -832,7 +844,11 @@ def _is_ssl_error_message_from_http_proxy(ssl_error):
         redirect_location = redirect and response.get_redirect_location()
         if redirect_location:
             if response.status == 303:
+                # Change the method according to RFC 9110, Section 15.4.4.
                 method = "GET"
+                # And lose the body not to transfer anything sensitive.
+                body = None
+                headers = HTTPHeaderDict(headers)._prepare_for_method_change()
 
             try:
                 retries = retries.increment(method, url, response=response, _pool=self)
@@ -1108,3 +1124,14 @@ def _normalize_host(host, scheme):
     if host.startswith("[") and host.endswith("]"):
         host = host[1:-1]
     return host
+
+
+def _close_pool_connections(pool):
+    """Drains a queue of connections and closes each one."""
+    try:
+        while True:
+            conn = pool.get(block=False)
+            if conn:
+                conn.close()
+    except queue.Empty:
+        pass  # Done.
diff --git a/src/snowflake/connector/vendored/urllib3/contrib/securetransport.py b/src/snowflake/connector/vendored/urllib3/contrib/securetransport.py
@@ -64,9 +64,8 @@
 import threading
 import weakref
 
-import six
-
 from .. import util
+from ..packages import six
 from ..util.ssl_ import PROTOCOL_TLS_CLIENT
 from ._securetransport.bindings import CoreFoundation, Security, SecurityConst
 from ._securetransport.low_level import (
diff --git a/src/snowflake/connector/vendored/urllib3/packages/backports/weakref_finalize.py b/src/snowflake/connector/vendored/urllib3/packages/backports/weakref_finalize.py
@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+"""
+backports.weakref_finalize
+~~~~~~~~~~~~~~~~~~
+
+Backports the Python 3 ``weakref.finalize`` method.
+"""
+from __future__ import absolute_import
+
+import itertools
+import sys
+from weakref import ref
+
+__all__ = ["weakref_finalize"]
+
+
+class weakref_finalize(object):
+    """Class for finalization of weakrefable objects
+    finalize(obj, func, *args, **kwargs) returns a callable finalizer
+    object which will be called when obj is garbage collected. The
+    first time the finalizer is called it evaluates func(*arg, **kwargs)
+    and returns the result. After this the finalizer is dead, and
+    calling it just returns None.
+    When the program exits any remaining finalizers for which the
+    atexit attribute is true will be run in reverse order of creation.
+    By default atexit is true.
+    """
+
+    # Finalizer objects don't have any state of their own.  They are
+    # just used as keys to lookup _Info objects in the registry.  This
+    # ensures that they cannot be part of a ref-cycle.
+
+    __slots__ = ()
+    _registry = {}
+    _shutdown = False
+    _index_iter = itertools.count()
+    _dirty = False
+    _registered_with_atexit = False
+
+    class _Info(object):
+        __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index")
+
+    def __init__(self, obj, func, *args, **kwargs):
+        if not self._registered_with_atexit:
+            # We may register the exit function more than once because
+            # of a thread race, but that is harmless
+            import atexit
+
+            atexit.register(self._exitfunc)
+            weakref_finalize._registered_with_atexit = True
+        info = self._Info()
+        info.weakref = ref(obj, self)
+        info.func = func
+        info.args = args
+        info.kwargs = kwargs or None
+        info.atexit = True
+        info.index = next(self._index_iter)
+        self._registry[self] = info
+        weakref_finalize._dirty = True
+
+    def __call__(self, _=None):
+        """If alive then mark as dead and return func(*args, **kwargs);
+        otherwise return None"""
+        info = self._registry.pop(self, None)
+        if info and not self._shutdown:
+            return info.func(*info.args, **(info.kwargs or {}))
+
+    def detach(self):
+        """If alive then mark as dead and return (obj, func, args, kwargs);
+        otherwise return None"""
+        info = self._registry.get(self)
+        obj = info and info.weakref()
+        if obj is not None and self._registry.pop(self, None):
+            return (obj, info.func, info.args, info.kwargs or {})
+
+    def peek(self):
+        """If alive then return (obj, func, args, kwargs);
+        otherwise return None"""
+        info = self._registry.get(self)
+        obj = info and info.weakref()
+        if obj is not None:
+            return (obj, info.func, info.args, info.kwargs or {})
+
+    @property
+    def alive(self):
+        """Whether finalizer is alive"""
+        return self in self._registry
+
+    @property
+    def atexit(self):
+        """Whether finalizer should be called at exit"""
+        info = self._registry.get(self)
+        return bool(info) and info.atexit
+
+    @atexit.setter
+    def atexit(self, value):
+        info = self._registry.get(self)
+        if info:
+            info.atexit = bool(value)
+
+    def __repr__(self):
+        info = self._registry.get(self)
+        obj = info and info.weakref()
+        if obj is None:
+            return "<%s object at %#x; dead>" % (type(self).__name__, id(self))
+        else:
+            return "<%s object at %#x; for %r at %#x>" % (
+                type(self).__name__,
+                id(self),
+                type(obj).__name__,
+                id(obj),
+            )
+
+    @classmethod
+    def _select_for_exit(cls):
+        # Return live finalizers marked for exit, oldest first
+        L = [(f, i) for (f, i) in cls._registry.items() if i.atexit]
+        L.sort(key=lambda item: item[1].index)
+        return [f for (f, i) in L]
+
+    @classmethod
+    def _exitfunc(cls):
+        # At shutdown invoke finalizers for which atexit is true.
+        # This is called once all other non-daemonic threads have been
+        # joined.
+        reenable_gc = False
+        try:
+            if cls._registry:
+                import gc
+
+                if gc.isenabled():
+                    reenable_gc = True
+                    gc.disable()
+                pending = None
+                while True:
+                    if pending is None or weakref_finalize._dirty:
+                        pending = cls._select_for_exit()
+                        weakref_finalize._dirty = False
+                    if not pending:
+                        break
+                    f = pending.pop()
+                    try:
+                        # gc is disabled, so (assuming no daemonic
+                        # threads) the following is the only line in
+                        # this function which might trigger creation
+                        # of a new finalizer
+                        f()
+                    except Exception:
+                        sys.excepthook(*sys.exc_info())
+                    assert f not in cls._registry
+        finally:
+            # prevent any more finalizers from executing during shutdown
+            weakref_finalize._shutdown = True
+            if reenable_gc:
+                gc.enable()
diff --git a/src/snowflake/connector/vendored/urllib3/poolmanager.py b/src/snowflake/connector/vendored/urllib3/poolmanager.py
@@ -4,7 +4,7 @@
 import functools
 import logging
 
-from ._collections import RecentlyUsedContainer
+from ._collections import HTTPHeaderDict, RecentlyUsedContainer
 from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
 from .exceptions import (
     LocationValueError,
@@ -171,7 +171,7 @@ class PoolManager(RequestMethods):
     def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
         RequestMethods.__init__(self, headers)
         self.connection_pool_kw = connection_pool_kw
-        self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close())
+        self.pools = RecentlyUsedContainer(num_pools)
 
         # Locally set the pool classes and keys so other PoolManagers can
         # override them.
@@ -382,9 +382,12 @@ def urlopen(self, method, url, redirect=True, **kw):
         # Support relative URLs for redirecting.
         redirect_location = urljoin(url, redirect_location)
 
-        # RFC 7231, Section 6.4.4
         if response.status == 303:
+            # Change the method according to RFC 9110, Section 15.4.4.
             method = "GET"
+            # And lose the body not to transfer anything sensitive.
+            kw["body"] = None
+            kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change()
 
         retries = kw.get("retries")
         if not isinstance(retries, Retry):
diff --git a/src/snowflake/connector/vendored/urllib3/request.py b/src/snowflake/connector/vendored/urllib3/request.py
diff --git a/src/snowflake/connector/vendored/urllib3/util/retry.py b/src/snowflake/connector/vendored/urllib3/util/retry.py

Original file line number	Diff line number	Diff line change
`@@ -193,7 +193,6 @@ def init_poolmanager(`
`193`	`193`	`num_pools=connections,`
`194`	`194`	`maxsize=maxsize,`
`195`	`195`	`block=block,`
`196`		`- strict=True,`
`197`	`196`	`**pool_kwargs,`
`198`	`197`	`)`
`199`	`198`
Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`# This file is protected via CODEOWNERS`
`2`		`-__version__ = "1.26.15"`
	`2`	`+__version__ = "1.26.18"`