chore(ingest): start using explicit exports (#11899)

Author: hsheth2

metadata-ingestion/scripts/avro_codegen.py

@@ -769,7 +769,7 @@ def generate(
 import importlib
 from typing import TYPE_CHECKING
 
-from datahub._codegen.aspect import _Aspect
+from datahub._codegen.aspect import _Aspect as _Aspect
 from datahub.utilities.docs_build import IS_SPHINX_BUILD
 from datahub.utilities._custom_package_loader import get_custom_models_package
 
@@ -802,7 +802,7 @@ def generate(
 
 from datahub.utilities.docs_build import IS_SPHINX_BUILD
 from datahub.utilities._custom_package_loader import get_custom_urns_package
-from datahub.utilities.urns._urn_base import Urn  # noqa: F401
+from datahub.utilities.urns._urn_base import Urn as Urn  # noqa: F401
 
 _custom_package_path = get_custom_urns_package()
 

metadata-ingestion/setup.cfg

@@ -31,7 +31,7 @@ exclude =
     __pycache__
 per-file-ignores =
     # imported but unused
-    __init__.py: F401
+    __init__.py: F401, I250
 ban-relative-imports = true
 
 [mypy]
@@ -53,6 +53,14 @@ disallow_untyped_defs = no
 # try to be a bit more strict in certain areas of the codebase
 [mypy-datahub.*]
 ignore_missing_imports = no
+implicit_reexport = no
+[mypy-datahub.metadata.*]
+# TODO: Remove this once all the code has been updated.
+implicit_reexport = yes
+[mypy-datahub.ingestion.*]
+# TODO: Remove this once all the code has been updated.
+implicit_reexport = yes
+
 [mypy-datahub_provider.*]
 ignore_missing_imports = no
 [mypy-tests.*]

metadata-ingestion/src/datahub/api/circuit_breaker/__init__.py

@@ -12,3 +12,10 @@
 )
 
 requests_logger.setLevel(logging.WARNING)
+
+__all__ = [
+    "AssertionCircuitBreaker",
+    "AssertionCircuitBreakerConfig",
+    "OperationCircuitBreaker",
+    "OperationCircuitBreakerConfig",
+]

metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py

@@ -6,7 +6,7 @@
 from gql.transport.requests import RequestsHTTPTransport
 from pydantic import Field
 
-from datahub.configuration import ConfigModel
+from datahub.configuration.common import ConfigModel
 
 logger = logging.getLogger(__name__)
 

metadata-ingestion/src/datahub/api/entities/datajob/__init__.py

@@ -1,2 +1,5 @@
 from datahub.api.entities.datajob.dataflow import DataFlow
 from datahub.api.entities.datajob.datajob import DataJob
+
+# TODO: Remove this and start importing directly from the inner files.
+__all__ = ["DataFlow", "DataJob"]

metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py

@@ -3,7 +3,6 @@
 from typing import Callable, Dict, Iterable, List, Optional, Set, cast
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.source_common import ALL_ENV_TYPES
 from datahub.emitter.generic_emitter import Emitter
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.schema_classes import (
@@ -114,7 +113,7 @@ def generate_tags_aspect(self) -> List[GlobalTagsClass]:
 
     def _get_env(self) -> Optional[str]:
         env: Optional[str] = None
-        if self.env and self.env.upper() in ALL_ENV_TYPES:
+        if self.env and self.env.upper() in builder.ALL_ENV_TYPES:
             env = self.env.upper()
         else:
             logger.debug(

metadata-ingestion/src/datahub/api/entities/datajob/datajob.py

@@ -3,7 +3,6 @@
 from typing import Callable, Dict, Iterable, List, Optional, Set
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.source_common import ALL_ENV_TYPES
 from datahub.emitter.generic_emitter import Emitter
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.schema_classes import (
@@ -109,7 +108,7 @@ def generate_mcp(
         self, materialize_iolets: bool = True
     ) -> Iterable[MetadataChangeProposalWrapper]:
         env: Optional[str] = None
-        if self.flow_urn.cluster.upper() in ALL_ENV_TYPES:
+        if self.flow_urn.cluster.upper() in builder.ALL_ENV_TYPES:
             env = self.flow_urn.cluster.upper()
         else:
             logger.debug(

metadata-ingestion/src/datahub/api/graphql/__init__.py

@@ -1,2 +1,4 @@
 from datahub.api.graphql.assertion import Assertion
 from datahub.api.graphql.operation import Operation
+
+__all__ = ["Assertion", "Operation"]

metadata-ingestion/src/datahub/cli/put_cli.py

@@ -6,11 +6,12 @@
 
 from datahub.cli.cli_utils import post_entity
 from datahub.configuration.config_loader import load_config_file
-from datahub.emitter.mcp import MetadataChangeProposalWrapper, SystemMetadataClass
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.metadata.schema_classes import (
     DataPlatformInfoClass as DataPlatformInfo,
     PlatformTypeClass,
+    SystemMetadataClass,
 )
 from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade

metadata-ingestion/src/datahub/configuration/__init__.py

@@ -1,5 +1,4 @@
 from datahub.configuration.common import (
-    ConfigModel,
-    ConfigurationMechanism,
-    DynamicTypedConfig,
+    ConfigModel as ConfigModel,
+    DynamicTypedConfig as DynamicTypedConfig,
 )