feat(file-based): Add Calamine-first with Openpyxl fallback for Excel parser

airbyte_cdk/sources/file_based/exceptions.py

@@ -92,6 +92,12 @@ class RecordParseError(BaseFileBasedSourceError):
     pass
 
 
+class ExcelCalamineParsingError(BaseFileBasedSourceError):
+    """Raised when Calamine engine fails to parse an Excel file."""
+
+    pass
+
+
 class SchemaInferenceError(BaseFileBasedSourceError):
     pass
 

airbyte_cdk/sources/file_based/file_types/excel_parser.py

@@ -3,9 +3,10 @@
 #
 
 import logging
+import warnings
 from io import IOBase
 from pathlib import Path
-from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, Union
+from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, Type, Union, cast
 
 import orjson
 import pandas as pd
@@ -17,6 +18,7 @@
 )
 from airbyte_cdk.sources.file_based.exceptions import (
     ConfigValidationError,
+    ExcelCalamineParsingError,
     FileBasedSourceError,
     RecordParseError,
 )
@@ -64,7 +66,7 @@ async def infer_schema(
         fields: Dict[str, str] = {}
 
         with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp:
-            df = self.open_and_parse_file(fp)
+            df = self.open_and_parse_file(fp, logger, file)
             for column, df_type in df.dtypes.items():
                 # Choose the broadest data type if the column's data type differs in dataframes
                 prev_frame_column_type = fields.get(column)  # type: ignore [call-overload]
@@ -92,7 +94,7 @@ def parse_records(
         discovered_schema: Optional[Mapping[str, SchemaType]] = None,
     ) -> Iterable[Dict[str, Any]]:
         """
-        Parses records from an Excel file based on the provided configuration.
+        Parses records from an Excel file with fallback error handling.
 
         Args:
             config (FileBasedStreamConfig): Configuration for the file-based stream.
@@ -111,7 +113,7 @@ def parse_records(
         try:
             # Open and parse the file using the stream reader
             with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp:
-                df = self.open_and_parse_file(fp)
+                df = self.open_and_parse_file(fp, logger, file)
                 # Yield records as dictionaries
                 # DataFrame.to_dict() method returns datetime values in pandas.Timestamp values, which are not serializable by orjson
                 # DataFrame.to_json() returns string with datetime values serialized to iso8601 with microseconds to align with pydantic behavior
@@ -181,14 +183,91 @@ def validate_format(excel_format: BaseModel, logger: logging.Logger) -> None:
             raise ConfigValidationError(FileBasedSourceError.CONFIG_VALIDATION_ERROR)
 
     @staticmethod
-    def open_and_parse_file(fp: Union[IOBase, str, Path]) -> pd.DataFrame:
+    def _open_and_parse_file_with_calamine(
+        fp: Union[IOBase, str, Path],
+        logger: logging.Logger,
+        file_info: RemoteFile,
+    ) -> pd.DataFrame:
+        """Opens and parses Excel file using Calamine engine.
+
+        Args:
+            fp: File pointer to the Excel file.
+            logger: Logger for logging information and errors.
+            file_info: Remote file information for logging context.
+
+        Returns:
+            pd.DataFrame: Parsed data from the Excel file.
+
+        Raises:
+            ExcelCalamineParsingError: If Calamine fails to parse the file.
         """
-        Opens and parses the Excel file.
+        try:
+            return pd.ExcelFile(fp, engine="calamine").parse()  # type: ignore [arg-type, call-overload, no-any-return]
+        except BaseException as exc:
+            # Calamine engine raises PanicException(child of BaseException) if Calamine fails to parse the file.
+            # Checking if ValueError in exception arg to know if it was actually an error during parsing due to invalid values in cells.
+            # Otherwise, raise an exception.
+            if "ValueError" in str(exc):
+                logger.warning(
+                    f"Calamine parsing failed for {file_info.file_uri_for_logging}, falling back to openpyxl: {exc}"
+                )
+                raise ExcelCalamineParsingError(
+                    f"Calamine engine failed to parse {file_info.file_uri_for_logging}",
+                    filename=file_info.uri,
+                ) from exc
+            raise exc
+
+    @staticmethod
+    def _open_and_parse_file_with_openpyxl(
+        fp: Union[IOBase, str, Path],
+        logger: logging.Logger,
+        file_info: RemoteFile,
+    ) -> pd.DataFrame:
+        """Opens and parses Excel file using Openpyxl engine.
 
         Args:
             fp: File pointer to the Excel file.
+            logger: Logger for logging information and errors.
+            file_info: Remote file information for logging context.
 
         Returns:
             pd.DataFrame: Parsed data from the Excel file.
         """
-        return pd.ExcelFile(fp, engine="calamine").parse()  # type: ignore [arg-type, call-overload, no-any-return]
+        with warnings.catch_warnings(record=True) as warning_records:
+            warnings.simplefilter("always")
+            df = pd.ExcelFile(fp, engine="openpyxl").parse()  # type: ignore [arg-type, call-overload]
+
+        for warning in warning_records:
+            logger.warning(
+                f"Openpyxl warning for {file_info.file_uri_for_logging}: {warning.message}"
+            )
+
+        return df  # type: ignore [no-any-return]
+
+    @staticmethod
+    def open_and_parse_file(
+        fp: Union[IOBase, str, Path],
+        logger: logging.Logger,
+        file_info: RemoteFile,
+    ) -> pd.DataFrame:
+        """Opens and parses the Excel file with Calamine-first and Openpyxl fallback.
+
+        Args:
+            fp: File pointer to the Excel file.
+            logger: Logger for logging information and errors.
+            file_info: Remote file information for logging context.
+
+        Returns:
+            pd.DataFrame: Parsed data from the Excel file.
+        """
+        try:
+            return ExcelParser._open_and_parse_file_with_calamine(fp, logger, file_info)
+        except ExcelCalamineParsingError:
+            # Fallback to openpyxl
+            try:
+                fp.seek(0)  # type: ignore [union-attr]
+            except (AttributeError, OSError):
+                # Some file-like objects may not be seekable; attempt openpyxl parsing anyway
+                pass
+
+            return ExcelParser._open_and_parse_file_with_openpyxl(fp, logger, file_info)

airbyte_cdk/sources/file_based/remote_file.py

@@ -17,6 +17,11 @@ class RemoteFile(BaseModel):
     last_modified: datetime
     mime_type: Optional[str] = None
 
+    @property
+    def file_uri_for_logging(self) -> str:
+        """Returns a user-friendly identifier for logging."""
+        return self.uri
+
 
 class UploadableRemoteFile(RemoteFile, ABC):
     """
@@ -48,10 +53,3 @@ def source_file_relative_path(self) -> str:
         Returns the relative path of the source file.
         """
         return self.uri
-
-    @property
-    def file_uri_for_logging(self) -> str:
-        """
-        Returns the URI for the file being logged.
-        """
-        return self.uri

pyproject.toml

@@ -73,6 +73,7 @@ pdf2image = { version = "1.16.3", optional = true }
 pyarrow = { version = "^19.0.0", optional = true }
 pytesseract = { version = "0.3.10", optional = true }  # Used indirectly by unstructured library
 python-calamine = { version = "0.2.3", optional = true }  # TODO: Remove if unused
+openpyxl = { version = "^3.1.0", optional = true }
 python-snappy = { version = "0.7.3", optional = true }  # TODO: remove if unused
 tiktoken = { version = "0.8.0", optional = true }
 nltk = { version = "3.9.1", optional = true }
@@ -120,7 +121,7 @@ deptry = "^0.23.0"
 dagger-io = "0.19.0"
 
 [tool.poetry.extras]
-file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfminer.six", "unstructured.pytesseract", "pytesseract", "markdown", "python-calamine", "python-snappy"]
+file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfminer.six", "unstructured.pytesseract", "pytesseract", "markdown", "python-calamine", "openpyxl", "python-snappy"]
 vector-db-based = ["langchain_community", "langchain_core", "langchain_text_splitters", "openai", "cohere", "tiktoken"]
 sql = ["sqlalchemy"]
 dev = ["pytest"]
@@ -252,6 +253,7 @@ DEP002 = [
   "cohere",
   "markdown",
   "openai",
+  "openpyxl",
   "pdf2image",
   "pdfminer.six",
   "pytesseract",

unit_tests/sources/file_based/file_types/test_excel_parser.py

@@ -4,6 +4,7 @@
 
 
 import datetime
+import warnings
 from io import BytesIO
 from unittest.mock import MagicMock, Mock, mock_open, patch
 
@@ -136,3 +137,55 @@ def test_file_read_error(mock_stream_reader, mock_logger, file_config, remote_fi
                 list(
                     parser.parse_records(file_config, remote_file, mock_stream_reader, mock_logger)
                 )
+
+
+class FakePanic(BaseException):
+    """Simulates the PyO3 PanicException which does not inherit from Exception."""
+
+
+def test_open_and_parse_file_falls_back_to_openpyxl(mock_logger):
+    parser = ExcelParser()
+    fp = BytesIO(b"test")
+    remote_file = RemoteFile(uri="s3://mybucket/test.xlsx", last_modified=datetime.datetime.now())
+
+    fallback_df = pd.DataFrame({"a": [1]})
+
+    calamine_excel_file = MagicMock()
+
+    def calamine_parse_side_effect():
+        raise FakePanic(
+            "failed to construct date: PyErr { type: <class 'ValueError'>, value: ValueError('year 20225 is out of range'), traceback: None }"
+        )
+
+    calamine_excel_file.parse.side_effect = calamine_parse_side_effect
+
+    openpyxl_excel_file = MagicMock()
+
+    def openpyxl_parse_side_effect():
+        warnings.warn("Cell A146 has invalid date", UserWarning)
+        return fallback_df
+
+    openpyxl_excel_file.parse.side_effect = openpyxl_parse_side_effect
+
+    with (
+        patch("airbyte_cdk.sources.file_based.file_types.excel_parser.pd.ExcelFile") as mock_excel,
+    ):
+        mock_excel.side_effect = [calamine_excel_file, openpyxl_excel_file]
+
+        result = parser.open_and_parse_file(fp, mock_logger, remote_file)
+
+    pd.testing.assert_frame_equal(result, fallback_df)
+    assert mock_logger.warning.call_count == 2
+    assert "Openpyxl warning" in mock_logger.warning.call_args_list[1].args[0]
+
+
+def test_open_and_parse_file_does_not_swallow_keyboard_interrupt(mock_logger):
+    parser = ExcelParser()
+    fp = BytesIO(b"test")
+    remote_file = RemoteFile(uri="s3://mybucket/test.xlsx", last_modified=datetime.datetime.now())
+
+    with patch("airbyte_cdk.sources.file_based.file_types.excel_parser.pd.ExcelFile") as mock_excel:
+        mock_excel.return_value.parse.side_effect = KeyboardInterrupt()
+
+        with pytest.raises(KeyboardInterrupt):
+            parser.open_and_parse_file(fp, mock_logger, remote_file)