[DOP-21442] Add Excel API schema

Ilyas Gasanov · Ilyas Gasanov · commit e16f448c7642 · 2024-11-25T08:44:30.000+03:00
diff --git a/docs/changelog/next_release/139.feature.rst b/docs/changelog/next_release/139.feature.rst
@@ -0,0 +1 @@
+Add Excel API schema
diff --git a/syncmaster/schemas/v1/__init__.py b/syncmaster/schemas/v1/__init__.py
@@ -36,7 +36,7 @@
     PostgresReadTransferSourceAndTarget,
     ReadDBTransfer,
 )
-from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, JSONLine
+from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, Excel, JSONLine
 from syncmaster.schemas.v1.transfers.run import (
     CreateRunSchema,
     ReadRunSchema,
diff --git a/syncmaster/schemas/v1/file_formats.py b/syncmaster/schemas/v1/file_formats.py
@@ -5,3 +5,4 @@
 CSV_FORMAT = Literal["csv"]
 JSONLINE_FORMAT = Literal["jsonline"]
 JSON_FORMAT = Literal["json"]
+EXCEL_FORMAT = Literal["excel"]
diff --git a/syncmaster/schemas/v1/transfers/file/base.py b/syncmaster/schemas/v1/transfers/file/base.py
@@ -7,28 +7,28 @@
 
 from pydantic import BaseModel, Field, field_validator
 
-from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, JSONLine
+from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, Excel, JSONLine
 
 
 # At the moment the ReadTransferSourceParams and ReadTransferTargetParams
 # classes are identical but may change in the future
 class ReadFileTransferSource(BaseModel):
     directory_path: str
-    file_format: CSV | JSONLine | JSON = Field(..., discriminator="type")
+    file_format: CSV | JSONLine | JSON | Excel = Field(..., discriminator="type")
     options: dict[str, Any]
 
 
 class ReadFileTransferTarget(BaseModel):
     directory_path: str
-    file_format: CSV | JSONLine = Field(..., discriminator="type")  # JSON format is not supported for writing
+    file_format: CSV | JSONLine | Excel = Field(..., discriminator="type")  # JSON format is not supported for writing
     options: dict[str, Any]
 
 
 # At the moment the CreateTransferSourceParams and CreateTransferTargetParams
 # classes are identical but may change in the future
 class CreateFileTransferSource(BaseModel):
     directory_path: str
-    file_format: CSV | JSONLine | JSON = Field(..., discriminator="type")
+    file_format: CSV | JSONLine | JSON | Excel = Field(..., discriminator="type")
     options: dict[str, Any] = Field(default_factory=dict)
 
     class Config:
@@ -44,7 +44,7 @@ def _directory_path_is_valid_path(cls, value):
 
 class CreateFileTransferTarget(BaseModel):
     directory_path: str
-    file_format: CSV | JSONLine = Field(..., discriminator="type")  # JSON FORMAT IS NOT SUPPORTED AS A TARGET !
+    file_format: CSV | JSONLine | Excel = Field(..., discriminator="type")  # JSON FORMAT IS NOT SUPPORTED AS A TARGET !
     options: dict[str, Any] = Field(default_factory=dict)
 
     class Config:
diff --git a/syncmaster/schemas/v1/transfers/file_format.py b/syncmaster/schemas/v1/transfers/file_format.py
@@ -4,7 +4,12 @@
 
 from pydantic import BaseModel
 
-from syncmaster.schemas.v1.file_formats import CSV_FORMAT, JSON_FORMAT, JSONLINE_FORMAT
+from syncmaster.schemas.v1.file_formats import (
+    CSV_FORMAT,
+    EXCEL_FORMAT,
+    JSON_FORMAT,
+    JSONLINE_FORMAT,
+)
 
 
 class CSV(BaseModel):
@@ -27,3 +32,9 @@ class JSON(BaseModel):
     type: JSON_FORMAT
     encoding: str = "utf-8"
     line_sep: str = "\n"
+
+
+class Excel(BaseModel):
+    type: EXCEL_FORMAT
+    include_header: bool
+    start_cell: str | None = None
diff --git a/tests/test_unit/test_transfers/test_file_transfers/test_create_transfer.py b/tests/test_unit/test_transfers/test_file_transfers/test_create_transfer.py
@@ -28,6 +28,24 @@
             "directory_path": "/some/pure/path",
             "file_format": {
                 "type": "csv",
+                "delimiter": ",",
+                "encoding": "utf-8",
+                "quote": '"',
+                "escape": "\\",
+                "header": False,
+                "line_sep": "\n",
+            },
+            "options": {
+                "some": "option",
+            },
+        },
+        {
+            "type": "s3",
+            "directory_path": "/some/excel/path",
+            "file_format": {
+                "type": "excel",
+                "include_header": True,
+                "start_cell": "A1",
             },
             "options": {
                 "some": "option",
@@ -94,11 +112,26 @@ async def test_developer_plus_can_create_s3_transfer(
         "queue_id": transfer.queue_id,
     }
 
+    expected_file_formats = {
+        "csv": {
+            "delimiter": ",",
+            "encoding": "utf-8",
+            "quote": '"',
+            "escape": "\\",
+            "header": False,
+            "line_sep": "\n",
+        },
+        "excel": {
+            "include_header": True,
+            "start_cell": "A1",
+        },
+    }
+
     for params in (transfer.source_params, transfer.target_params):
-        assert params["type"] == "s3"
-        assert params["directory_path"] == "/some/pure/path"
-        assert params["file_format"]["type"] == "csv"
+        assert params["type"] == target_source_params["type"]
+        assert params["directory_path"] == target_source_params["directory_path"]
         assert params["options"] == {"some": "option"}
+        assert params["file_format"] == expected_file_formats[params["type"]]
 
 
 @pytest.mark.parametrize(
@@ -121,6 +154,15 @@ async def test_developer_plus_can_create_s3_transfer(
                 "type": "csv",
             },
         },
+        {
+            "type": "hdfs",
+            "directory_path": "/some/excel/path",
+            "file_format": {
+                "type": "excel",
+                "include_header": True,
+                "start_cell": "A1",
+            },
+        },
     ],
 )
 async def test_developer_plus_can_create_hdfs_transfer(
@@ -183,10 +225,27 @@ async def test_developer_plus_can_create_hdfs_transfer(
         "queue_id": transfer.queue_id,
     }
 
+    expected_file_formats = {
+        "csv": {
+            "type": "csv",
+            "delimiter": ",",
+            "encoding": "utf-8",
+            "quote": '"',
+            "escape": "\\",
+            "header": False,
+            "line_sep": "\n",
+        },
+        "excel": {
+            "type": "excel",
+            "include_header": True,
+            "start_cell": "A1",
+        },
+    }
+
     for params in (transfer.source_params, transfer.target_params):
-        assert params["type"] == "hdfs"
-        assert params["directory_path"] == "/some/pure/path"
-        assert params["file_format"]["type"] == "csv"
+        assert params["type"] == target_source_params["type"]
+        assert params["directory_path"] == target_source_params["directory_path"]
+        assert params["file_format"] == expected_file_formats[params["file_format"]["type"]]
         assert params["options"] == {}
 
 
@@ -211,6 +270,14 @@ async def test_developer_plus_can_create_hdfs_transfer(
                 "type": "csv",
             },
         },
+        {
+            "type": "s3",
+            "directory_path": "some/path",
+            "file_format": {
+                "type": "excel",
+                "include_header": True,
+            },
+        },
     ],
 )
 async def test_cannot_create_file_transfer_with_relative_path(
diff --git a/tests/test_unit/test_transfers/test_file_transfers/test_read_transfer.py b/tests/test_unit/test_transfers/test_file_transfers/test_read_transfer.py
@@ -23,6 +23,16 @@
             },
             "options": {},
         },
+        {
+            "type": "s3",
+            "directory_path": "/some/excel/path",
+            "file_format": {
+                "type": "excel",
+                "include_header": True,
+                "start_cell": "A1",
+            },
+            "options": {},
+        },
     ],
 )
 @pytest.mark.parametrize(
diff --git a/tests/test_unit/test_transfers/test_file_transfers/test_update_transfer.py b/tests/test_unit/test_transfers/test_file_transfers/test_update_transfer.py
@@ -23,6 +23,16 @@
             },
             "options": {},
         },
+        {
+            "type": "s3",
+            "directory_path": "/some/excel/path",
+            "file_format": {
+                "type": "excel",
+                "include_header": True,
+                "start_cell": "A1",
+            },
+            "options": {},
+        },
     ],
 )
 @pytest.mark.parametrize(
@@ -54,7 +64,7 @@ async def test_developer_plus_can_update_s3_transfer(
             "source_params": {
                 "type": "s3",
                 "directory_path": "/some/new/test/directory",
-                "file_format": {"type": "jsonline"},
+                "file_format": create_transfer_data["file_format"],
                 "options": {"some": "option"},
             },
         },
@@ -65,14 +75,11 @@ async def test_developer_plus_can_update_s3_transfer(
     source_params.update(
         {
             "directory_path": "/some/new/test/directory",
-            "file_format": {
-                "encoding": "utf-8",
-                "line_sep": "\n",
-                "type": "jsonline",
-            },
+            "file_format": create_transfer_data["file_format"],
             "options": {"some": "option"},
         },
     )
+
     # Assert
     assert result.status_code == 200
     assert result.json() == {

Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@`
`36`	`36`	`PostgresReadTransferSourceAndTarget,`
`37`	`37`	`ReadDBTransfer,`
`38`	`38`	`)`
`39`		`-from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, JSONLine`
	`39`	`+from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, Excel, JSONLine`
`40`	`40`	`from syncmaster.schemas.v1.transfers.run import (`
`41`	`41`	`CreateRunSchema,`
`42`	`42`	`ReadRunSchema,`