Minor - Adding partition params (#1035)

Author: jaidisido

awswrangler/catalog/_add.py

@@ -53,6 +53,7 @@ def add_csv_partitions(
     serde_parameters: Optional[Dict[str, str]] = None,
     boto3_session: Optional[boto3.Session] = None,
     columns_types: Optional[Dict[str, str]] = None,
+    partitions_parameters: Optional[Dict[str, str]] = None,
 ) -> None:
     r"""Add partitions (metadata) to a CSV Table in the AWS Glue Catalog.
 
@@ -89,6 +90,8 @@ def add_csv_partitions(
         Only required for Hive compability.
         Dictionary with keys as column names and values as data types (e.g. {'col0': 'bigint', 'col1': 'double'}).
         P.S. Only materialized columns please, not partition columns.
+    partitions_parameters: Optional[Dict[str, str]]
+        Dictionary with key-value pairs defining partition parameters.
 
     Returns
     -------
@@ -120,6 +123,7 @@ def add_csv_partitions(
             columns_types=columns_types,
             serde_library=serde_library,
             serde_parameters=serde_parameters,
+            partitions_parameters=partitions_parameters,
         )
         for k, v in partitions_values.items()
     ]
@@ -138,6 +142,7 @@ def add_json_partitions(
     serde_parameters: Optional[Dict[str, str]] = None,
     boto3_session: Optional[boto3.Session] = None,
     columns_types: Optional[Dict[str, str]] = None,
+    partitions_parameters: Optional[Dict[str, str]] = None,
 ) -> None:
     r"""Add partitions (metadata) to a JSON Table in the AWS Glue Catalog.
 
@@ -172,6 +177,8 @@ def add_json_partitions(
         Only required for Hive compability.
         Dictionary with keys as column names and values as data types (e.g. {'col0': 'bigint', 'col1': 'double'}).
         P.S. Only materialized columns please, not partition columns.
+    partitions_parameters: Optional[Dict[str, str]]
+        Dictionary with key-value pairs defining partition parameters.
 
     Returns
     -------
@@ -202,6 +209,7 @@ def add_json_partitions(
             columns_types=columns_types,
             serde_library=serde_library,
             serde_parameters=serde_parameters,
+            partitions_parameters=partitions_parameters,
         )
         for k, v in partitions_values.items()
     ]
@@ -218,6 +226,7 @@ def add_parquet_partitions(
     compression: Optional[str] = None,
     boto3_session: Optional[boto3.Session] = None,
     columns_types: Optional[Dict[str, str]] = None,
+    partitions_parameters: Optional[Dict[str, str]] = None,
 ) -> None:
     """Add partitions (metadata) to a Parquet Table in the AWS Glue Catalog.
 
@@ -245,6 +254,8 @@ def add_parquet_partitions(
         Only required for Hive compability.
         Dictionary with keys as column names and values as data types (e.g. {'col0': 'bigint', 'col1': 'double'}).
         P.S. Only materialized columns please, not partition columns.
+    partitions_parameters: Optional[Dict[str, str]]
+        Dictionary with key-value pairs defining partition parameters.
 
     Returns
     -------
@@ -274,6 +285,7 @@ def add_parquet_partitions(
                 bucketing_info=bucketing_info,
                 compression=compression,
                 columns_types=columns_types,
+                partitions_parameters=partitions_parameters,
             )
             for k, v in partitions_values.items()
         ]

awswrangler/catalog/_definitions.py

@@ -71,6 +71,7 @@ def _parquet_partition_definition(
     bucketing_info: Optional[Tuple[List[str], int]],
     compression: Optional[str],
     columns_types: Optional[Dict[str, str]],
+    partitions_parameters: Optional[Dict[str, str]],
 ) -> Dict[str, Any]:
     compressed: bool = compression is not None
     definition: Dict[str, Any] = {
@@ -88,6 +89,7 @@ def _parquet_partition_definition(
             "BucketColumns": [] if bucketing_info is None else bucketing_info[0],
         },
         "Values": values,
+        "Parameters": {} if partitions_parameters is None else partitions_parameters,
     }
     if columns_types is not None:
         definition["StorageDescriptor"]["Columns"] = [
@@ -155,6 +157,7 @@ def _csv_partition_definition(
     serde_library: Optional[str],
     serde_parameters: Optional[Dict[str, str]],
     columns_types: Optional[Dict[str, str]],
+    partitions_parameters: Optional[Dict[str, str]],
 ) -> Dict[str, Any]:
     compressed: bool = compression is not None
     serde_info = {
@@ -175,6 +178,7 @@ def _csv_partition_definition(
             "BucketColumns": [] if bucketing_info is None else bucketing_info[0],
         },
         "Values": values,
+        "Parameters": {} if partitions_parameters is None else partitions_parameters,
     }
     if columns_types is not None:
         definition["StorageDescriptor"]["Columns"] = [
@@ -232,6 +236,7 @@ def _json_partition_definition(
     serde_library: Optional[str],
     serde_parameters: Optional[Dict[str, str]],
     columns_types: Optional[Dict[str, str]],
+    partitions_parameters: Optional[Dict[str, str]],
 ) -> Dict[str, Any]:
     compressed: bool = compression is not None
     serde_info = {
@@ -250,6 +255,7 @@ def _json_partition_definition(
             "BucketColumns": [] if bucketing_info is None else bucketing_info[0],
         },
         "Values": values,
+        "Parameters": {} if partitions_parameters is None else partitions_parameters,
     }
     if columns_types is not None:
         definition["StorageDescriptor"]["Columns"] = [

tests/test_catalog.py

@@ -378,6 +378,7 @@ def test_catalog_json(path: str, glue_database: str, glue_table: str, account_id
         table=glue_table,
         partitions_values={f"{path}y=2020/m=1/": ["2020", "1"], f"{path}y=2021/m=2/": ["2021", "2"]},
         compression="snappy",
+        partitions_parameters={"retention": "365"},
     )
     partitions_values = wr.catalog.get_partitions(database=glue_database, table=glue_table)
     assert len(partitions_values) == 2