rm excessive objs

Author: Ilyas Gasanov

tests/test_integration/test_run_transfer/test_clickhouse.py

@@ -211,12 +211,7 @@ async def test_run_transfer_postgres_to_clickhouse_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, postgres_to_clickhouse.id)
 
-    reader = DBReader(
-        connection=clickhouse,
-        table=f"{clickhouse.user}.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(df_with_increment, init_df)
     assert df_with_increment.sort("ID").collect() == init_df.sort("ID").collect()
 
@@ -343,11 +338,6 @@ async def test_run_transfer_clickhouse_to_postgres_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, clickhouse_to_postgres.id)
 
-    reader = DBReader(
-        connection=postgres,
-        table="public.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(df_with_increment, init_df)
     assert df_with_increment.sort("ID").collect() == init_df.sort("ID").collect()

tests/test_integration/test_run_transfer/test_ftp.py

@@ -241,12 +241,7 @@ async def test_run_transfer_ftp_to_postgres_with_incremental_strategy(
 
     await run_transfer_and_verify(client, group_owner, ftp_to_postgres.id)
 
-    reader = DBReader(
-        connection=postgres,
-        table="public.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(
         df_with_increment,
         init_df,
@@ -415,23 +410,10 @@ async def test_run_transfer_postgres_to_ftp_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, postgres_to_ftp.id)
 
-    downloader = FileDownloader(
-        connection=ftp_file_connection,
-        source_path=f"/target/{format_name}/{file_format_flavor}",
-        local_path=tmp_path,
-    )
     downloader.run()
-
     verify_file_name_template(os.listdir(tmp_path), expected_extension)
 
-    reader = FileDFReader(
-        connection=ftp_file_df_connection,
-        format=format,
-        source_path=tmp_path,
-        df_schema=init_df.schema,
-    )
     df_with_increment = reader.run()
-
     df_with_increment, second_transfer_df = prepare_dataframes_for_comparison(
         df_with_increment,
         second_transfer_df,

tests/test_integration/test_run_transfer/test_ftps.py

@@ -205,12 +205,7 @@ async def test_run_transfer_ftps_to_postgres_with_incremental_strategy(
 
     await run_transfer_and_verify(client, group_owner, ftps_to_postgres.id)
 
-    reader = DBReader(
-        connection=postgres,
-        table="public.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(
         df_with_increment,
         init_df,
@@ -344,23 +339,10 @@ async def test_run_transfer_postgres_to_ftps_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, postgres_to_ftps.id)
 
-    downloader = FileDownloader(
-        connection=ftps_file_connection,
-        source_path=f"/target/{format_name}/{file_format_flavor}",
-        local_path=tmp_path,
-    )
     downloader.run()
-
     verify_file_name_template(os.listdir(tmp_path), expected_extension)
 
-    reader = FileDFReader(
-        connection=ftps_file_df_connection,
-        format=format,
-        source_path=tmp_path,
-        df_schema=init_df.schema,
-    )
     df_with_increment = reader.run()
-
     df_with_increment, second_transfer_df = prepare_dataframes_for_comparison(
         df_with_increment,
         second_transfer_df,

tests/test_integration/test_run_transfer/test_hdfs.py

@@ -329,15 +329,7 @@ async def test_run_transfer_postgres_to_hdfs_with_incremental_strategy(
     files = [os.fspath(file) for file in hdfs_file_connection.list_dir(target_path) if file.is_file()]
     verify_file_name_template(files, expected_extension)
 
-    reader = FileDFReader(
-        connection=hdfs_file_df_connection,
-        format=format,
-        source_path=target_path,
-        df_schema=init_df.schema,
-        options={},
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(
         df_with_increment,
         init_df,

tests/test_integration/test_run_transfer/test_hive.py

@@ -197,12 +197,7 @@ async def test_run_transfer_postgres_to_hive_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, postgres_to_hive.id)
 
-    reader = DBReader(
-        connection=hive,
-        table="default.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(df_with_increment, init_df)
     assert df_with_increment.sort("ID").collect() == init_df.sort("ID").collect()
 
@@ -329,11 +324,6 @@ async def test_run_transfer_hive_to_postgres_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, hive_to_postgres.id)
 
-    reader = DBReader(
-        connection=postgres,
-        table="public.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(df_with_increment, init_df)
     assert df_with_increment.sort("ID").collect() == init_df.sort("ID").collect()

tests/test_integration/test_run_transfer/test_mssql.py

@@ -210,12 +210,7 @@ async def test_run_transfer_postgres_to_mssql_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, postgres_to_mssql.id)
 
-    reader = DBReader(
-        connection=mssql,
-        table="dbo.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(df_with_increment, init_df, db_type="mssql")
     assert df.sort("ID").collect() == init_df.sort("ID").collect()
 
@@ -351,11 +346,6 @@ async def test_run_transfer_mssql_to_postgres_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, mssql_to_postgres.id)
 
-    reader = DBReader(
-        connection=postgres,
-        table="public.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(df_with_increment, init_df, db_type="mssql")
     assert df_with_increment.sort("ID").collect() == init_df.sort("ID").collect()

tests/test_integration/test_run_transfer/test_mysql.py

@@ -212,12 +212,7 @@ async def test_run_transfer_postgres_to_mysql_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, postgres_to_mysql.id)
 
-    reader = DBReader(
-        connection=mysql,
-        table=f"{mysql.database}.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(df_with_increment, init_df, db_type="mysql")
     assert df_with_increment.sort("ID").collect() == init_df.sort("ID").collect()
 
@@ -355,11 +350,6 @@ async def test_run_transfer_mysql_to_postgres_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, mysql_to_postgres.id)
 
-    reader = DBReader(
-        connection=postgres,
-        table="public.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(df_with_increment, init_df, db_type="mysql")
     assert df_with_increment.sort("ID").collect() == init_df.sort("ID").collect()

tests/test_integration/test_run_transfer/test_oracle.py

@@ -200,12 +200,7 @@ async def test_run_transfer_postgres_to_oracle_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, postgres_to_oracle.id)
 
-    reader = DBReader(
-        connection=oracle,
-        table=f"{oracle.user}.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(df_with_increment, init_df)
     assert df_with_increment.sort("ID").collect() == init_df.sort("ID").collect()
 
@@ -332,11 +327,6 @@ async def test_run_transfer_oracle_to_postgres_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, oracle_to_postgres.id)
 
-    reader = DBReader(
-        connection=postgres,
-        table="public.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(df_with_increment, init_df)
     assert df_with_increment.sort("ID").collect() == init_df.sort("ID").collect()

tests/test_integration/test_run_transfer/test_s3.py

@@ -368,15 +368,7 @@ async def test_run_transfer_postgres_to_s3_with_incremental_strategy(
     files = [os.fspath(file) for file in s3_file_connection.list_dir(target_path)]
     verify_file_name_template(files, expected_extension)
 
-    reader = FileDFReader(
-        connection=s3_file_df_connection,
-        format=format,
-        source_path=target_path,
-        df_schema=init_df.schema,
-        options={},
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(
         df_with_increment,
         init_df,

tests/test_integration/test_run_transfer/test_samba.py

@@ -205,12 +205,7 @@ async def test_run_transfer_samba_to_postgres_with_incremental_strategy(
 
     await run_transfer_and_verify(client, group_owner, samba_to_postgres.id)
 
-    reader = DBReader(
-        connection=postgres,
-        table="public.target_table",
-    )
     df_with_increment = reader.run()
-
     df_with_increment, init_df = prepare_dataframes_for_comparison(
         df_with_increment,
         init_df,
@@ -344,23 +339,10 @@ async def test_run_transfer_postgres_to_samba_with_incremental_strategy(
     fill_with_data(second_transfer_df)
     await run_transfer_and_verify(client, group_owner, postgres_to_samba.id)
 
-    downloader = FileDownloader(
-        connection=samba_file_connection,
-        source_path=f"/target/{format_name}/{file_format_flavor}",
-        local_path=tmp_path,
-    )
     downloader.run()
-
     verify_file_name_template(os.listdir(tmp_path), expected_extension)
 
-    reader = FileDFReader(
-        connection=samba_file_df_connection,
-        format=format,
-        source_path=tmp_path,
-        df_schema=init_df.schema,
-    )
     df_with_increment = reader.run()
-
     df_with_increment, second_transfer_df = prepare_dataframes_for_comparison(
         df_with_increment,
         second_transfer_df,