[DOP-21450] Add compressions for file formats

Author: Ilyas Gasanov

syncmaster/schemas/v1/transfers/file_format.py

@@ -18,15 +18,13 @@
 
 
 class ORCCompression(str, Enum):
-    UNCOMPRESSED = "uncompressed"
     NONE = "none"
     SNAPPY = "snappy"
     ZLIB = "zlib"
     LZ4 = "lz4"
 
 
 class ParquetCompression(str, Enum):
-    UNCOMPRESSED = "uncompressed"
     NONE = "none"
     SNAPPY = "snappy"
     GZIP = "gzip"
@@ -66,21 +64,21 @@ class CSV(BaseModel):
     escape: str = "\\"
     include_header: bool = False
     line_sep: str = "\n"
-    compression: CSVCompression = CSVCompression.NONE
+    compression: CSVCompression = CSVCompression.GZIP
 
 
 class JSONLine(BaseModel):
     type: JSONLINE_FORMAT
     encoding: str = "utf-8"
     line_sep: str = "\n"
-    compression: JSONCompression = CSVCompression.NONE
+    compression: JSONCompression = JSONCompression.GZIP
 
 
 class JSON(BaseModel):
     type: JSON_FORMAT
     encoding: str = "utf-8"
     line_sep: str = "\n"
-    compression: JSONCompression = CSVCompression.NONE
+    compression: JSONCompression = JSONCompression.GZIP
 
 
 class Excel(BaseModel):
@@ -93,14 +91,14 @@ class XML(BaseModel):
     type: XML_FORMAT
     root_tag: str
     row_tag: str
-    compression: XMLCompression | None = None
+    compression: XMLCompression = XMLCompression.GZIP
 
 
 class ORC(BaseModel):
     type: ORC_FORMAT
-    compression: ORCCompression = CSVCompression.NONE
+    compression: ORCCompression = ORCCompression.ZLIB
 
 
 class Parquet(BaseModel):
     type: PARQUET_FORMAT
-    compression: ParquetCompression = CSVCompression.NONE
+    compression: ParquetCompression = ParquetCompression.SNAPPY

tests/test_integration/test_run_transfer/test_hdfs.py

@@ -276,7 +276,7 @@ async def test_run_transfer_postgres_to_hdfs(
     df = reader.run()
 
     # as Excel does not support datetime values with precision greater than milliseconds
-    if format_name in ("excel", "parquet"):
+    if format_name == "excel":
         init_df = init_df.withColumn(
             "REGISTERED_AT",
             to_timestamp(date_format(col("REGISTERED_AT"), "yyyy-MM-dd HH:mm:ss.SSS")),

tests/test_integration/test_run_transfer/test_s3.py

@@ -277,7 +277,7 @@ async def test_run_transfer_postgres_to_s3(
     df = reader.run()
 
     # as Excel does not support datetime values with precision greater than milliseconds
-    if format_name in ("excel", "parquet"):
+    if format_name == "excel":
         init_df = init_df.withColumn(
             "REGISTERED_AT",
             to_timestamp(date_format(col("REGISTERED_AT"), "yyyy-MM-dd HH:mm:ss.SSS")),