[DOP-22120] Import file connectors on demand

Author: dolfinus

syncmaster/worker/handlers/file/ftp.py

@@ -5,8 +5,6 @@
 
 from typing import TYPE_CHECKING
 
-from onetl.connection import FTP, SparkLocalFS
-
 from syncmaster.dto.connections import FTPConnectionDTO
 from syncmaster.worker.handlers.file.local_df import LocalDFFileHandler
 
@@ -18,12 +16,15 @@ class FTPHandler(LocalDFFileHandler):
     connection_dto: FTPConnectionDTO
 
     def connect(self, spark: SparkSession) -> None:
+        from onetl.connection import FTP, SparkLocalFS
+
         self.file_connection = FTP(
             host=self.connection_dto.host,
             port=self.connection_dto.port,
             user=self.connection_dto.user,
             password=self.connection_dto.password,
         ).check()
+
         self.local_df_connection = SparkLocalFS(
             spark=spark,
         ).check()

syncmaster/worker/handlers/file/ftps.py

@@ -5,8 +5,6 @@
 
 from typing import TYPE_CHECKING
 
-from onetl.connection import FTPS, SparkLocalFS
-
 from syncmaster.dto.connections import FTPSConnectionDTO
 from syncmaster.worker.handlers.file.local_df import LocalDFFileHandler
 
@@ -18,12 +16,15 @@ class FTPSHandler(LocalDFFileHandler):
     connection_dto: FTPSConnectionDTO
 
     def connect(self, spark: SparkSession) -> None:
+        from onetl.connection import FTPS, SparkLocalFS
+
         self.file_connection = FTPS(
             host=self.connection_dto.host,
             port=self.connection_dto.port,
             user=self.connection_dto.user,
             password=self.connection_dto.password,
         ).check()
+
         self.local_df_connection = SparkLocalFS(
             spark=spark,
         ).check()

syncmaster/worker/handlers/file/hdfs.py

@@ -5,8 +5,6 @@
 
 from typing import TYPE_CHECKING
 
-from onetl.connection import HDFS, SparkHDFS
-
 from syncmaster.dto.connections import HDFSConnectionDTO
 from syncmaster.worker.handlers.file.remote_df import RemoteDFFileHandler
 
@@ -18,10 +16,13 @@ class HDFSHandler(RemoteDFFileHandler):
     connection_dto: HDFSConnectionDTO
 
     def connect(self, spark: SparkSession):
+        from onetl.connection import HDFS, SparkHDFS
+
         self.df_connection = SparkHDFS(
             cluster=self.connection_dto.cluster,
             spark=spark,
         ).check()
+
         self.file_connection = HDFS(
             cluster=self.connection_dto.cluster,
         ).check()

syncmaster/worker/handlers/file/s3.py

@@ -5,7 +5,6 @@
 
 from typing import TYPE_CHECKING
 
-from onetl.connection import S3, SparkS3
 from onetl.file import FileDFReader
 
 from syncmaster.dto.connections import S3ConnectionDTO
@@ -19,6 +18,8 @@ class S3Handler(RemoteDFFileHandler):
     connection_dto: S3ConnectionDTO
 
     def connect(self, spark: SparkSession):
+        from onetl.connection import S3, SparkS3
+
         self.df_connection = SparkS3(
             host=self.connection_dto.host,
             port=self.connection_dto.port,
@@ -30,6 +31,7 @@ def connect(self, spark: SparkSession):
             extra=self.connection_dto.additional_params,
             spark=spark,
         ).check()
+
         self.file_connection = S3(
             host=self.connection_dto.host,
             port=self.connection_dto.port,

syncmaster/worker/handlers/file/samba.py

@@ -5,8 +5,6 @@
 
 from typing import TYPE_CHECKING
 
-from onetl.connection import Samba, SparkLocalFS
-
 from syncmaster.dto.connections import SambaConnectionDTO
 from syncmaster.worker.handlers.file.local_df import LocalDFFileHandler
 
@@ -18,6 +16,8 @@ class SambaHandler(LocalDFFileHandler):
     connection_dto: SambaConnectionDTO
 
     def connect(self, spark: SparkSession) -> None:
+        from onetl.connection import Samba, SparkLocalFS
+
         self.file_connection = Samba(
             host=self.connection_dto.host,
             port=self.connection_dto.port,
@@ -28,6 +28,7 @@ def connect(self, spark: SparkSession) -> None:
             password=self.connection_dto.password,
             auth_type=self.connection_dto.auth_type,
         ).check()
+
         self.local_df_connection = SparkLocalFS(
             spark=spark,
         ).check()

syncmaster/worker/handlers/file/sftp.py

@@ -5,8 +5,6 @@
 
 from typing import TYPE_CHECKING
 
-from onetl.connection import SFTP, SparkLocalFS
-
 from syncmaster.dto.connections import SFTPConnectionDTO
 from syncmaster.worker.handlers.file.local_df import LocalDFFileHandler
 
@@ -18,13 +16,16 @@ class SFTPHandler(LocalDFFileHandler):
     connection_dto: SFTPConnectionDTO
 
     def connect(self, spark: SparkSession) -> None:
+        from onetl.connection import SFTP, SparkLocalFS
+
         self.file_connection = SFTP(
             host=self.connection_dto.host,
             port=self.connection_dto.port,
             user=self.connection_dto.user,
             password=self.connection_dto.password,
             compress=False,  # to avoid errors from combining file and SCP-level compression
         ).check()
+
         self.local_df_connection = SparkLocalFS(
             spark=spark,
         ).check()