build(ingest): bump moto[s3] to 5.x and re-add datahub-documents to dev deps (#16691)

Author: rajatoss

metadata-ingestion/constraints.txt

@@ -859,7 +859,7 @@ more-itertools==10.8.0
     #   jaraco-classes
     #   jaraco-functools
     #   simple-salesforce
-moto==4.2.14
+moto==5.1.22
     # via acryl-datahub
 msal==1.35.1
     # via
@@ -1124,7 +1124,7 @@ pure-sasl==0.6.2
     # via
     #   acryl-pyhive
     #   thrift-sasl
-py-partiql-parser==0.5.0
+py-partiql-parser==0.6.3
     # via moto
 py4j==0.10.9.9
     # via pyspark

metadata-ingestion/pyproject.toml

@@ -444,7 +444,7 @@ delta-lake = [
     "deltalake>=0.6.3, != 0.6.4, != 0.18.0, <1.0.0; platform_system == 'Darwin' and platform_machine == 'arm64'",
     "deltalake>=0.6.3, != 0.6.4, <1.0.0; platform_system != 'Darwin' or platform_machine != 'arm64'",
     "more-itertools>=8.12.0,<11.0.0",
-    "moto[s3]<5.0.0",
+    "moto[s3]>=5.0.0,<6.0.0",
     "parse>=1.19.0,<2.0.0",
     "pyarrow>14.0.0,<24.0.0",
     "pydeequ>=1.1.0,<2.0.0",
@@ -603,7 +603,7 @@ gcs = [
     "botocore!=1.23.0,<2.0.0",
     "cachetools<6.0.0",
     "more-itertools>=8.12.0,<11.0.0",
-    "moto[s3]<5.0.0",
+    "moto[s3]>=5.0.0,<6.0.0",
     "parse>=1.19.0,<2.0.0",
     "pyarrow>14.0.0,<24.0.0",
     "pydeequ>=1.1.0,<2.0.0",
@@ -620,7 +620,7 @@ gcs-slim = [
     "botocore!=1.23.0,<2.0.0",
     "cachetools<6.0.0",
     "more-itertools>=8.12.0,<11.0.0",
-    "moto[s3]<5.0.0",
+    "moto[s3]>=5.0.0,<6.0.0",
     "parse>=1.19.0,<2.0.0",
     "pyarrow>14.0.0,<24.0.0",
     "smart-open[gcs,s3]>=5.2.1,<8.0.0",
@@ -1091,7 +1091,7 @@ s3 = [
     "botocore!=1.23.0,<2.0.0",
     "cachetools<6.0.0",
     "more-itertools>=8.12.0,<11.0.0",
-    "moto[s3]<5.0.0",
+    "moto[s3]>=5.0.0,<6.0.0",
     "parse>=1.19.0,<2.0.0",
     "pyarrow>14.0.0,<24.0.0",
     "pydeequ>=1.1.0,<2.0.0",
@@ -1108,7 +1108,7 @@ s3-slim = [
     "botocore!=1.23.0,<2.0.0",
     "cachetools<6.0.0",
     "more-itertools>=8.12.0,<11.0.0",
-    "moto[s3]<5.0.0",
+    "moto[s3]>=5.0.0,<6.0.0",
     "parse>=1.19.0,<2.0.0",
     "pyarrow>14.0.0,<24.0.0",
     "smart-open[s3]>=5.2.1,<8.0.0",
@@ -1473,7 +1473,7 @@ all = [
     "looker-sdk>=23.0.0,<26.0.0",
     "mlflow-skinny>=2.3.0,<2.21.0",
     "more-itertools>=8.12.0,<11.0.0",
-    "moto[s3]<5.0.0",
+    "moto[s3]>=5.0.0,<6.0.0",
     "msal>=1.31.1,<2.0.0",
     "neo4j<7.0.0",
     "nest-asyncio<2.0.0",
@@ -1627,7 +1627,7 @@ dev = [
     "mixpanel>=4.9.0,<6.0.0",
     "mlflow-skinny>=2.3.0,<2.21.0",
     "more-itertools>=8.12.0,<11.0.0",
-    "moto[s3]<5.0.0",
+    "moto[s3]>=5.0.0,<6.0.0",
     "msal>=1.31.1,<2.0.0",
     "mypy==1.17.1",
     "neo4j<7.0.0",
@@ -1820,7 +1820,7 @@ docs = [
     "mixpanel>=4.9.0,<6.0.0",
     "mlflow-skinny>=2.3.0,<2.21.0",
     "more-itertools>=8.12.0,<11.0.0",
-    "moto[s3]<5.0.0",
+    "moto[s3]>=5.0.0,<6.0.0",
     "msal>=1.31.1,<2.0.0",
     "mypy==1.17.1",
     "neo4j<7.0.0",
@@ -1977,7 +1977,7 @@ integration-tests = [
     "jupyter_server>=2.14.1,<3.0.0",
     "kerberos>=1.3.0,<2.0.0",
     "more-itertools>=8.12.0,<11.0.0",
-    "moto[s3]<5.0.0",
+    "moto[s3]>=5.0.0,<6.0.0",
     "numpy<2",
     "openpyxl>=3.1.5,<4.0.0",
     "pandas<3.0.0",

metadata-ingestion/setup.py

@@ -389,8 +389,7 @@
     # ujson 5.2.0 has the JSONDecodeError exception type, which we need for error handling.
     "ujson>=5.2.0,<6.0.0",
     "smart-open[s3]>=5.2.1,<8.0.0",
-    # moto 5.0.0 drops support for Python 3.7
-    "moto[s3]<5.0.0",
+    "moto[s3]>=5.0.0,<6.0.0",
     *path_spec_common,
     # cachetools is used by operation_config which is imported by profiling config
     *cachetools_lib,
@@ -884,9 +883,7 @@
             "clickhouse-usage",
             "cockroachdb",
             "confluence",
-            # Note: datahub-documents removed from dev deps due to Python 3.10+ requirement
-            # It's available as a separate extra and in the docs extra for doc generation
-            # TODO: Re-add datahub-documents here now that Python 3.9 support was dropped
+            "datahub-documents",
             "dataplex",
             "delta-lake",
             "dremio",

metadata-ingestion/tests/integration/dynamodb/test_dynamodb.py

@@ -4,7 +4,7 @@
 import boto3
 import pytest
 from freezegun import freeze_time
-from moto import mock_dynamodb
+from moto import mock_aws
 from mypy_boto3_dynamodb.type_defs import TagTypeDef
 
 from datahub.ingestion.glossary.classification_mixin import ClassificationConfig
@@ -23,7 +23,7 @@
 
 
 @freeze_time(FROZEN_TIME)
-@mock_dynamodb
+@mock_aws
 @pytest.mark.integration
 def test_dynamodb(pytestconfig, tmp_path):
     boto3.setup_default_session()

metadata-ingestion/tests/integration/excel/test_excel_s3.py

@@ -4,7 +4,7 @@
 import moto
 import pytest
 from boto3.session import Session
-from moto import mock_s3
+from moto import mock_aws
 
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.testing import mce_helpers
@@ -16,7 +16,7 @@
 
 @pytest.fixture(scope="module")
 def s3_setup():
-    with mock_s3():
+    with mock_aws():
         session = Session(
             aws_access_key_id="test",
             aws_secret_access_key="test",

metadata-ingestion/tests/integration/s3/test_s3.py

@@ -7,7 +7,7 @@
 import moto.s3
 import pytest
 from boto3.session import Session
-from moto import mock_s3
+from moto import mock_aws
 from pydantic import ValidationError
 
 from datahub.ingestion.run.pipeline import Pipeline, PipelineContext
@@ -75,7 +75,7 @@ def bucket_names():
 
 @pytest.fixture(scope="module", autouse=True)
 def s3():
-    with mock_s3():
+    with mock_aws():
         conn = Session(
             aws_access_key_id="test",
             aws_secret_access_key="test",
@@ -86,14 +86,14 @@ def s3():
 
 @pytest.fixture(scope="module", autouse=True)
 def s3_resource(s3):
-    with mock_s3():
+    with mock_aws():
         conn = s3.resource("s3")
         yield conn
 
 
 @pytest.fixture(scope="module", autouse=True)
 def s3_client(s3):
-    with mock_s3():
+    with mock_aws():
         conn = s3.client("s3")
         yield conn
 

metadata-ingestion/tests/unit/s3/test_s3_source.py

@@ -6,7 +6,7 @@
 import pytest
 from boto3.session import Session
 from freezegun import freeze_time
-from moto import mock_s3
+from moto import mock_aws
 
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
@@ -26,7 +26,7 @@
 
 @pytest.fixture(autouse=True)
 def s3():
-    with mock_s3():
+    with mock_aws():
         conn = Session(
             aws_access_key_id="test",
             aws_secret_access_key="test",

metadata-ingestion/tests/unit/test_aws_common.py

@@ -4,7 +4,7 @@
 
 import boto3
 import pytest
-from moto import mock_iam, mock_lambda, mock_sts
+from moto import mock_aws
 
 from datahub.ingestion.source.aws.aws_common import (
     AwsConnectionConfig,
@@ -143,9 +143,7 @@ def test_is_running_on_ec2_failure(self, mock_put, mock_get):
         mock_get.return_value.status_code = 404
         assert is_running_on_ec2() is False
 
-    @mock_sts
-    @mock_lambda
-    @mock_iam
+    @mock_aws
     def test_get_current_identity_lambda(self):
         """Test getting identity in Lambda environment"""
         with patch.dict(
@@ -186,7 +184,7 @@ def test_get_current_identity_lambda(self):
 
     @patch("requests.get")
     @patch("requests.put")
-    @mock_sts
+    @mock_aws
     def test_get_instance_role_arn_success(self, mock_put, mock_get):
         """Test getting EC2 instance role ARN"""
         mock_put.return_value.status_code = 200
@@ -206,15 +204,15 @@ def test_get_instance_role_arn_success(self, mock_put, mock_get):
                 role_arn == "arn:aws:sts::123456789012:assumed-role/test-role/instance"
             )
 
-    @mock_sts
+    @mock_aws
     def test_aws_connection_config_basic(self, mock_aws_config):
         """Test basic AWS connection configuration"""
         session = mock_aws_config.get_session()
         creds = session.get_credentials()
         assert creds.access_key == "test-key"
         assert creds.secret_key == "test-secret"
 
-    @mock_sts
+    @mock_aws
     def test_aws_connection_config_with_session_token(self):
         """Test AWS connection with session token"""
         config = AwsConnectionConfig(
@@ -229,7 +227,7 @@ def test_aws_connection_config_with_session_token(self):
         assert creds is not None
         assert creds.token == "test-token"
 
-    @mock_sts
+    @mock_aws
     def test_aws_connection_config_role_assumption(self):
         """Test AWS connection with role assumption"""
         config = AwsConnectionConfig(
@@ -247,7 +245,7 @@ def test_aws_connection_config_role_assumption(self):
             creds = session.get_credentials()
             assert creds is not None
 
-    @mock_sts
+    @mock_aws
     def test_aws_connection_config_skip_role_assumption(self):
         """Test AWS connection skipping role assumption when already in role"""
         config = AwsConnectionConfig(
@@ -265,7 +263,7 @@ def test_aws_connection_config_skip_role_assumption(self):
             session = config.get_session()
             assert session is not None
 
-    @mock_sts
+    @mock_aws
     def test_aws_connection_config_multiple_roles(self):
         """Test AWS connection with multiple role assumption"""
         config = AwsConnectionConfig(
@@ -341,7 +339,7 @@ def test_environment_detection_parametrized(
         with patch.dict(os.environ, env_vars, clear=True):
             assert detect_aws_environment() == expected_environment
 
-    @mock_sts
+    @mock_aws
     def test_role_assumption_credentials_cached_across_sessions(self):
         """
         Test that assumed role credentials are cached and reused across multiple
@@ -413,7 +411,7 @@ def test_role_assumption_credentials_cached_across_sessions(self):
             # assume_role should still only be called once
             assert mock_assume_role.call_count == 1
 
-    @mock_sts
+    @mock_aws
     def test_role_assumption_refreshes_expired_credentials(self):
         """
         Test that expired credentials trigger a new role assumption.
@@ -471,7 +469,7 @@ def test_role_assumption_refreshes_expired_credentials(self):
             # assume_role should be called again due to expiration
             assert mock_assume_role.call_count == 2
 
-    @mock_sts
+    @mock_aws
     def test_multiple_clients_use_same_cached_credentials(self):
         """
         Test that multiple AWS clients (glue, s3, lakeformation) created from
@@ -521,7 +519,7 @@ def test_multiple_clients_use_same_cached_credentials(self):
             # assume_role should only be called once
             assert mock_assume_role.call_count == 1
 
-    @mock_sts
+    @mock_aws
     def test_role_assumption_without_caching_before_fix(self):
         """
         This test demonstrates the bug that existed before the fix.
@@ -576,7 +574,7 @@ def test_role_assumption_without_caching_before_fix(self):
             assert creds2.secret_key == "ASSUMED_SECRET"
             assert creds2.token == "ASSUMED_TOKEN"
 
-    @mock_sts
+    @mock_aws
     def test_role_assumption_with_explicit_credentials(self):
         """
         Test that explicit credentials (aws_access_key_id/aws_secret_access_key)
@@ -597,7 +595,7 @@ def test_role_assumption_with_explicit_credentials(self):
         assert creds.access_key == "EXPLICIT_KEY"
         assert creds.secret_key == "EXPLICIT_SECRET"
 
-    @mock_sts
+    @mock_aws
     def test_role_assumption_chain(self):
         """
         Test assuming multiple roles in a chain.