feat: release GIL when possible (#388)

* - release python's GIL when opening an excel file in read_excel() and when reading a sheet "not eagerly" in ExcelReader.build_sheet

* - also release the gil in the "eager" case of ExcelReader.build_sheet

* - release GIL for reading tables too
- re-instate reading range for both eager&not eager code paths for ExcelReader.build_sheet

* Update src/types/excelreader/python.rs

* feat: release GIL in to_arrow and to_arrow_with_errors as well

Signed-off-by: Luka Peschke <luka.peschke@toucantoco.com>

* feat: also release the gil when eagerly loading a sheet

Signed-off-by: Luka Peschke <luka.peschke@toucantoco.com>

---------

Signed-off-by: Luka Peschke <luka.peschke@toucantoco.com>
Co-authored-by: jonathan <jonathan.clarke@fivesigma.co.uk>
Co-authored-by: Luka Peschke <luka.peschke@toucantoco.com>

Author: 3 people

src/lib.rs

@@ -30,15 +30,15 @@ pub fn read_excel<S: AsRef<str> + Display>(path: S) -> FastExcelResult<ExcelRead
 #[cfg(feature = "python")]
 /// Reads an excel file and returns an object allowing to access its sheets, tables, and a bit of metadata
 #[pyfunction(name = "read_excel")]
-fn py_read_excel(source: &Bound<'_, PyAny>) -> PyResult<ExcelReader> {
+fn py_read_excel<'py>(source: &Bound<'_, PyAny>, py: Python<'py>) -> PyResult<ExcelReader> {
     use py_errors::IntoPyResult;
 
     if let Ok(path) = source.extract::<String>() {
-        ExcelReader::try_from_path(&path)
+        py.allow_threads(|| ExcelReader::try_from_path(&path))
             .with_context(|| format!("could not load excel file at {path}"))
             .into_pyresult()
     } else if let Ok(bytes) = source.extract::<&[u8]>() {
-        ExcelReader::try_from(bytes)
+        py.allow_threads(|| ExcelReader::try_from(bytes))
             .with_context(|| "could not load excel file for those bytes")
             .into_pyresult()
     } else {

src/types/excelreader/python.rs

@@ -84,24 +84,29 @@ impl ExcelReader {
             .to_owned();
 
         if eager && self.sheets.supports_by_ref() {
-            let range = self
-                .sheets
-                .with_header_row(calamine_header_row)
-                .worksheet_range_ref(&sheet_meta.name)
+            let range = py
+                .allow_threads(|| {
+                    self.sheets
+                        .with_header_row(calamine_header_row)
+                        .worksheet_range_ref(&sheet_meta.name)
+                })
                 .into_pyresult()?;
             let pagination =
                 Pagination::try_new(opts.skip_rows, opts.n_rows, &range).into_pyresult()?;
             let header = Header::new(data_header_row, opts.column_names);
-            let rb = Self::load_sheet_eager(
-                &range.into(),
-                pagination,
-                header,
-                opts.schema_sample_rows,
-                &opts.selected_columns,
-                opts.dtypes.as_ref(),
-                &opts.dtype_coercion,
-            )
-            .into_pyresult()?;
+            let rb = py
+                .allow_threads(|| {
+                    Self::load_sheet_eager(
+                        &range.into(),
+                        pagination,
+                        header,
+                        opts.schema_sample_rows,
+                        &opts.selected_columns,
+                        opts.dtypes.as_ref(),
+                        &opts.dtype_coercion,
+                    )
+                })
+                .into_pyresult()?;
 
             #[cfg(feature = "pyarrow")]
             {
@@ -115,10 +120,12 @@ impl ExcelReader {
                 ))
             }
         } else {
-            let range = self
-                .sheets
-                .with_header_row(calamine_header_row)
-                .worksheet_range(&sheet_meta.name)
+            let range = py
+                .allow_threads(|| {
+                    self.sheets
+                        .with_header_row(calamine_header_row)
+                        .worksheet_range(&sheet_meta.name)
+                })
                 .into_pyresult()?;
             let pagination =
                 Pagination::try_new(opts.skip_rows, opts.n_rows, &range).into_pyresult()?;
@@ -160,7 +167,9 @@ impl ExcelReader {
         eager: bool,
         py: Python<'py>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let excel_table = self.load_table(name, opts).into_pyresult()?;
+        let excel_table = py
+            .allow_threads(|| self.load_table(name, opts))
+            .into_pyresult()?;
 
         if eager {
             #[cfg(feature = "pyarrow")]

src/types/excelsheet/python.rs

@@ -272,7 +272,7 @@ impl ExcelSheet {
 
         use crate::error::py_errors::IntoPyResult;
 
-        RecordBatch::try_from(self)
+        py.allow_threads(|| RecordBatch::try_from(self))
             .with_context(|| {
                 format!(
                     "could not create RecordBatch from sheet \"{}\"",
@@ -305,18 +305,21 @@ impl ExcelSheet {
         let offset = self.offset();
         let limit = self.limit();
 
-        let (rb, errors) = record_batch_from_data_and_columns_with_errors(
-            &self.selected_columns,
-            self.data(),
-            offset,
-            limit,
-        )
-        .with_context(|| {
-            format!(
-                "could not create RecordBatch from sheet \"{}\"",
-                self.name()
-            )
-        })?;
+        let (rb, errors) = py
+            .allow_threads(|| {
+                record_batch_from_data_and_columns_with_errors(
+                    &self.selected_columns,
+                    self.data(),
+                    offset,
+                    limit,
+                )
+            })
+            .with_context(|| {
+                format!(
+                    "could not create RecordBatch from sheet \"{}\"",
+                    self.name()
+                )
+            })?;
 
         let rb = rb
             .into_pyarrow(py)