fix: handle Minio client retries better (#511)

Author: joseph-sentry

shared/storage/minio.py

@@ -7,6 +7,8 @@
 from io import BytesIO
 from typing import BinaryIO, overload
 
+import certifi
+import urllib3
 from minio import Minio
 from minio.credentials import (
     ChainedProvider,
@@ -15,12 +17,17 @@
     IamAwsProvider,
 )
 from minio.error import MinioException, S3Error
+from urllib3 import Retry
+from urllib3.util import Timeout
 
 from shared.storage.base import CHUNK_SIZE, BaseStorageService
 from shared.storage.exceptions import BucketAlreadyExistsError, FileNotInStorageError
 
 log = logging.getLogger(__name__)
 
+CONNECT_TIMEOUT = 10
+READ_TIMEOUT = 60
+
 
 # Service class for interfacing with codecov's underlying storage layer, minio
 class MinioStorageService(BaseStorageService):
@@ -80,6 +87,24 @@ def init_minio_client(
         if port is not None:
             host = "{}:{}".format(host, port)
 
+        http_client = urllib3.PoolManager(
+            timeout=Timeout(connect=CONNECT_TIMEOUT, read=READ_TIMEOUT),
+            maxsize=10,
+            cert_reqs="CERT_REQUIRED",
+            ca_certs=os.environ.get("SSL_CERT_FILE") or certifi.where(),
+            retries=Retry(
+                total=5,
+                backoff_factor=1,
+                status_forcelist=[
+                    408,
+                    429,
+                    500,
+                    502,
+                    503,
+                    504,
+                ],  # https://cloud.google.com/storage/docs/retry-strategy#python
+            ),
+        )
         if iam_auth:
             return Minio(
                 host,
@@ -92,13 +117,15 @@ def init_minio_client(
                         EnvAWSProvider(),
                     ]
                 ),
+                http_client=http_client,
             )
         return Minio(
             host,
             access_key=access_key,
             secret_key=secret_key,
             secure=verify_ssl,
             region=region,
+            http_client=http_client,
         )
 
     # writes the initial storage bucket to storage via minio.

tests/unit/storage/test_minio.py

@@ -134,7 +134,11 @@ def test_minio_without_ports(self, mocker):
         storage = MinioStorageService(minio_no_ports_config)
         assert storage.minio_config == minio_no_ports_config
         mocked_minio_client.assert_called_with(
-            "cute_url_no_ports", credentials=mocker.ANY, secure=False, region=None
+            "cute_url_no_ports",
+            credentials=mocker.ANY,
+            http_client=mocker.ANY,
+            secure=False,
+            region=None,
         )
 
     def test_minio_with_ports(self, mocker):
@@ -151,7 +155,11 @@ def test_minio_with_ports(self, mocker):
         storage = MinioStorageService(minio_no_ports_config)
         assert storage.minio_config == minio_no_ports_config
         mocked_minio_client.assert_called_with(
-            "cute_url_no_ports:9000", credentials=mocker.ANY, secure=False, region=None
+            "cute_url_no_ports:9000",
+            credentials=mocker.ANY,
+            http_client=mocker.ANY,
+            secure=False,
+            region=None,
         )
 
     def test_minio_with_region(self, mocker):
@@ -171,6 +179,7 @@ def test_minio_with_region(self, mocker):
         mocked_minio_client.assert_called_with(
             "cute_url_no_ports:9000",
             credentials=mocker.ANY,
+            http_client=mocker.ANY,
             secure=False,
             region="example",
         )