fix: scanCSV eolChar bug (#335)

Fixing scanCSV eolChar bug to close #333

Author: Bidek56

__tests__/examples/datasets/pipe-eol.csv

@@ -0,0 +1 @@
+a,b|1,foo|2,boo

__tests__/io.test.ts

@@ -9,6 +9,8 @@ const tsvpath = path.resolve(__dirname, "./examples/datasets/data.tsv");
 // eslint-disable-next-line no-undef
 const emptycsvpath = path.resolve(__dirname, "./examples/datasets/empty.csv");
 // eslint-disable-next-line no-undef
+const pipecsvpath = path.resolve(__dirname, "./examples/datasets/pipe-eol.csv");
+// eslint-disable-next-line no-undef
 const parquetpath = path.resolve(__dirname, "./examples/foods.parquet");
 // eslint-disable-next-line no-undef
 const avropath = path.resolve(__dirname, "./examples/foods.avro");
@@ -52,6 +54,19 @@ describe("read:csv", () => {
       csvString.slice(0, 22),
     );
   });
+  it("can read from a csv file with eolChar", async () => {
+    const actual = pl.readCSV(pipecsvpath, { eolChar: "|" });
+    const expected = `shape: (2, 2)
+┌─────┬─────┐
+│ a   ┆ b   │
+│ --- ┆ --- │
+│ i64 ┆ str │
+╞═════╪═════╡
+│ 1   ┆ foo │
+│ 2   ┆ boo │
+└─────┴─────┘`;
+    expect(actual.toString()).toEqual(expected);
+  });
   it("can read from a csv buffer with newline in the header", () => {
     const csvBuffer = Buffer.from(
       '"name\na","height\nb"\n"John",172.23\n"Anna",1653.34',
@@ -264,6 +279,19 @@ describe("scan", () => {
 
     expect(df.shape).toEqual({ height: 4, width: 4 });
   });
+  it("can lazy load (scan) from a csv file with eolChar", async () => {
+    const actual = pl.scanCSV(pipecsvpath, { eolChar: "|" }).collectSync();
+    const expected = `shape: (2, 2)
+┌─────┬─────┐
+│ a   ┆ b   │
+│ --- ┆ --- │
+│ i64 ┆ str │
+╞═════╪═════╡
+│ 1   ┆ foo │
+│ 2   ┆ boo │
+└─────┴─────┘`;
+    expect(actual.toString()).toEqual(expected);
+  });
 });
 
 describe("parquet", () => {

polars/io.ts

@@ -232,6 +232,7 @@ const scanCsvDefaultOptions: Partial<ScanCsvOptions> = {
   ignoreErrors: true,
   skipRows: 0,
   sep: ",",
+  eolChar: "\n",
   rechunk: false,
   encoding: "utf8",
   lowMemory: false,

src/lazy/dataframe.rs

@@ -618,7 +618,7 @@ pub struct ScanCsvOptions {
     pub encoding: String,
     pub low_memory: Option<bool>,
     pub comment_prefix: Option<String>,
-    pub eol_char: Option<u8>,
+    pub eol_char: String,
     pub quote_char: Option<String>,
     pub parse_dates: Option<bool>,
     pub skip_rows_after_header: u32,
@@ -675,7 +675,7 @@ pub fn scan_csv(path: String, options: ScanCsvOptions) -> napi::Result<JsLazyFra
                 .map_or(None, |s| Some(PlSmallStr::from_string(s))),
         )
         .with_quote_char(quote_char)
-        .with_eol_char(options.eol_char.unwrap_or(b'\n'))
+        .with_eol_char(options.eol_char.as_bytes()[0])
         .with_rechunk(options.rechunk.unwrap_or(false))
         .with_skip_rows_after_header(options.skip_rows_after_header as usize)
         .with_encoding(encoding)