Support newer versions of smart_open (#167)

* Support newer versions of smart_open

* Fix tests to support newer smart_open

* Fix types

* Remove unneeded smart_open comments

Author: cwegrzyn

metrics/bigfiles_high_water_mark

@@ -1 +1 @@
-1149
+1133

metrics/coverage_high_water_mark

@@ -1 +1 @@
-93.700
+93.6900

metrics/mypy_high_water_mark

@@ -1 +1 @@
-92.3000
+92.3100

records_mover/url/s3/s3_file_url.py

@@ -1,12 +1,17 @@
 import logging
+
+import smart_open
 from .s3_base_url import S3BaseUrl
 from ..base import BaseDirectoryUrl, BaseFileUrl
 from typing import IO, List, Optional
 import threading
 from time import sleep
 from s3_concat import S3Concat
 from smart_open.s3 import open as s3_open
+import packaging.version
 
+SMART_OPEN_VERSION = packaging.version.parse(smart_open.__version__)
+SMART_OPEN_USE_SESSION = SMART_OPEN_VERSION < packaging.version.parse("5.0.0")
 
 logger = logging.getLogger(__name__)
 
@@ -51,10 +56,16 @@ def __call__(self, bytes_amount: int) -> None:
 
     def open(self, mode: str = "rb") -> IO[bytes]:
         try:
-            return s3_open(bucket_id=self.bucket,
-                           key_id=self.key,
-                           mode=mode,
-                           session=self._boto3_session)
+            if SMART_OPEN_USE_SESSION:
+                return s3_open(bucket_id=self.bucket,
+                               key_id=self.key,
+                               mode=mode,
+                               session=self._boto3_session)
+            else:
+                return s3_open(bucket_id=self.bucket,
+                               key_id=self.key,
+                               mode=mode,
+                               client=self._boto3_session.client('s3'))
         except ValueError as e:
             # Example: ValueError: 'b0KD9AkG7XA/_manifest' does not
             #  exist in the bucket 'vince-scratch', or is forbidden

setup.py

@@ -151,23 +151,7 @@ def initialize_options(self) -> None:
 ]
 
 smart_open_dependencies = [
-    # smart_open requires rsa>=3.1.4, which causes pip 20.2.4 in
-    # default mode to install rsa 4.6.
-    #
-    # That then results in this error from pip at install-time:
-    #
-    # awscli 1.18.178 requires rsa<=4.5.0,>=3.1.2; python_version !=
-    # "3.4", but you'll have rsa 4.6 which is incompatible.
-    #
-    # Also, pipgrip (used to make Python formulas in Homebrew) takes
-    # somewhere between much longer and forever to provide its output
-    # without this line.
-    #
-    'rsa>=3.1.4,<=4.5.0',
-    # we rely on exception types from smart_open,
-    # which seem to change in feature releases
-    # without a major version bump
-    'smart_open>=2,<2.1',
+    'smart_open>=2',
 ]
 
 gcs_dependencies = [

tests/unit/url/test_s3_file.py

@@ -1,4 +1,4 @@
-from records_mover.url.s3.s3_file_url import S3FileUrl
+from records_mover.url.s3.s3_file_url import S3FileUrl, SMART_OPEN_USE_SESSION
 from mock import patch, Mock, MagicMock, ANY
 import unittest
 
@@ -12,6 +12,10 @@ def setUp(self):
                                      boto3_session=self.mock_boto3_session)
         self.mock_s3_resource = self.mock_boto3_session.resource.return_value
         self.mock_s3_client = self.mock_boto3_session.client.return_value
+        if SMART_OPEN_USE_SESSION:
+            self.open_boto_args = {"session": self.mock_boto3_session}
+        else:
+            self.open_boto_args = {"client": self.mock_s3_client}
 
     def test_aws_creds(self):
         self.assertEqual(self.s3_file_url.aws_creds(),
@@ -61,7 +65,7 @@ def test_wait_to_exist_exists_already(self, mock_s3_open):
         mock_s3_open.assert_called_with(bucket_id='bucket',
                                         key_id='topdir/bottomdir/file',
                                         mode='rb',
-                                        session=self.mock_boto3_session)
+                                        **self.open_boto_args)
 
     @patch('records_mover.url.s3.s3_file_url.s3_open')
     def test_wait_to_exist_one_loop(self, mock_s3_open):
@@ -77,7 +81,7 @@ def test_wait_to_exist_one_loop(self, mock_s3_open):
         mock_s3_open.assert_called_with(bucket_id='bucket',
                                         key_id='topdir/bottomdir/file',
                                         mode='rb',
-                                        session=self.mock_boto3_session)
+                                        **self.open_boto_args)
 
     @patch('records_mover.url.s3.s3_file_url.s3_open')
     def test_open_other_valueerror_passes_through(self, mock_s3_open):

types/stubs/smart_open/__init__.pyi

@@ -0,0 +1 @@
+__version__: str

types/stubs/smart_open/s3/__init__.pyi

@@ -13,6 +13,7 @@ def open(bucket_id: str,
          buffer_size: int = DEFAULT_BUFFER_SIZE,
          min_part_size: int = DEFAULT_MIN_PART_SIZE,
          session: Optional[boto3.session.Session] = None,
+         client: Optional[boto3.client] = None,
          resource_kwargs: Optional[dict] = None,
          multipart_upload_kwargs: Optional[Dict] = None) -> IO[bytes]:
     ...