windows tests

Author: vdusek

src/crawlee/_utils/file.py

@@ -163,6 +163,13 @@ async def export_csv_to_stream(
     dst: TextIO,
     **kwargs: Unpack[ExportDataCsvKwargs],
 ) -> None:
+    # Set lineterminator to '\n' if not explicitly provided. This prevents double line endings on Windows.
+    # The csv.writer default is '\r\n', which when written to a file in text mode on Windows gets converted
+    # to '\r\r\n' due to newline translation. By using '\n', we let the platform handle the line ending
+    # conversion: '\n' stays as '\n' on Unix, and becomes '\r\n' on Windows.
+    if 'lineterminator' not in kwargs:
+        kwargs['lineterminator'] = '\n'
+
     writer = csv.writer(dst, **kwargs)  # type: ignore[arg-type]
     write_header = True
 

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -732,7 +732,13 @@ async def test_crawler_push_and_export_data(tmp_path: Path) -> None:
         {'id': 1, 'test': 'test'},
         {'id': 2, 'test': 'test'},
     ]
-    assert (tmp_path / 'dataset.csv').read_bytes() == b'id,test\r\n0,test\r\n1,test\r\n2,test\r\n'
+
+    # On Windows, text mode file writes convert \n to \r\n, resulting in \r\n line endings.
+    # On Unix/Linux, \n remains as \n.
+    if sys.platform == 'win32':
+        assert (tmp_path / 'dataset.csv').read_bytes() == b'id,test\r\n0,test\r\n1,test\r\n2,test\r\n'
+    else:
+        assert (tmp_path / 'dataset.csv').read_bytes() == b'id,test\n0,test\n1,test\n2,test\n'
 
 
 async def test_crawler_export_data_additional_kwargs(tmp_path: Path) -> None:
@@ -770,7 +776,12 @@ async def handler(context: BasicCrawlingContext) -> None:
         {'id': 2, 'test': 'test'},
     ]
 
-    assert (tmp_path / 'dataset.csv').read_bytes() == b'id,test\r\n0,test\r\n1,test\r\n2,test\r\n'
+    # On Windows, text mode file writes convert \n to \r\n, resulting in \r\n line endings.
+    # On Unix/Linux, \n remains as \n.
+    if sys.platform == 'win32':
+        assert (tmp_path / 'dataset.csv').read_bytes() == b'id,test\r\n0,test\r\n1,test\r\n2,test\r\n'
+    else:
+        assert (tmp_path / 'dataset.csv').read_bytes() == b'id,test\n0,test\n1,test\n2,test\n'
 
 
 async def test_context_update_kv_store() -> None: