fix(partition, csv): increase csv field limit (#4046)

Increase the csv field limit to support partitioning of files with large
data in fields.

---------

Co-authored-by: Filip Knefel <[email protected]>

Author: ds-filipknefel

CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.18.4
+
+### Enhancements
+
+### Features
+
+### Fixes
+- **Increase CSV field limit** Addresses failures in partition for csv files with large fields
+
 ## 0.18.3
 
 ### Enhancements

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.18.3"  # pragma: no cover
+__version__ = "0.18.4"  # pragma: no cover

unstructured/partition/csv.py

@@ -14,6 +14,7 @@
 from unstructured.utils import is_temp_file_path, lazyproperty
 
 DETECTION_ORIGIN: str = "csv"
+CSV_FIELD_LIMIT = 10 * 1048576  # 10MiB
 
 
 @apply_metadata(FileType.CSV)
@@ -54,6 +55,7 @@ def partition_csv(
         infer_table_structure=infer_table_structure,
     )
 
+    csv.field_size_limit(CSV_FIELD_LIMIT)
     with ctx.open() as file:
         dataframe = pd.read_csv(file, header=ctx.header, sep=ctx.delimiter, encoding=ctx.encoding)