feat(file-based): Add Calamine-first with Openpyxl fallback for Excel parser

airbyte_cdk/sources/file_based/file_types/excel_parser.py

@@ -3,6 +3,7 @@
 #
 
 import logging
+import warnings
 from io import IOBase
 from pathlib import Path
 from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, Union
@@ -64,7 +65,7 @@ async def infer_schema(
         fields: Dict[str, str] = {}
 
         with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp:
-            df = self.open_and_parse_file(fp)
+            df = self.open_and_parse_file(fp, logger, file)
             for column, df_type in df.dtypes.items():
                 # Choose the broadest data type if the column's data type differs in dataframes
                 prev_frame_column_type = fields.get(column)  # type: ignore [call-overload]
@@ -92,7 +93,7 @@ def parse_records(
         discovered_schema: Optional[Mapping[str, SchemaType]] = None,
     ) -> Iterable[Dict[str, Any]]:
         """
-        Parses records from an Excel file based on the provided configuration.
+        Parses records from an Excel file with fallback error handling.
 
         Args:
             config (FileBasedStreamConfig): Configuration for the file-based stream.
@@ -111,7 +112,7 @@ def parse_records(
         try:
             # Open and parse the file using the stream reader
             with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp:
-                df = self.open_and_parse_file(fp)
+                df = self.open_and_parse_file(fp, logger, file)
                 # Yield records as dictionaries
                 # DataFrame.to_dict() method returns datetime values in pandas.Timestamp values, which are not serializable by orjson
                 # DataFrame.to_json() returns string with datetime values serialized to iso8601 with microseconds to align with pydantic behavior
@@ -181,14 +182,84 @@ def validate_format(excel_format: BaseModel, logger: logging.Logger) -> None:
             raise ConfigValidationError(FileBasedSourceError.CONFIG_VALIDATION_ERROR)
 
     @staticmethod
-    def open_and_parse_file(fp: Union[IOBase, str, Path]) -> pd.DataFrame:
+    def open_and_parse_file(
+        fp: Union[IOBase, str, Path],
+        logger: Optional[logging.Logger] = None,
+        file_info: Optional[Union[str, RemoteFile]] = None,
+    ) -> pd.DataFrame:
         """
-        Opens and parses the Excel file.
-
-        Args:
-            fp: File pointer to the Excel file.
+        Opens and parses the Excel file with Calamine-first and Openpyxl fallback.
 
         Returns:
             pd.DataFrame: Parsed data from the Excel file.
         """
-        return pd.ExcelFile(fp, engine="calamine").parse()  # type: ignore [arg-type, call-overload, no-any-return]
+        file_label = "file"
+        file_url = None
+        if isinstance(file_info, RemoteFile):
+            file_label = file_info.file_uri_for_logging
+            file_url = getattr(file_info, "url", None)
+        elif isinstance(file_info, str):
+            file_label = file_info
+        calamine_exc: Optional[BaseException] = None
+        try:
+            with pd.ExcelFile(fp, engine="calamine") as excel_file:  # type: ignore [arg-type, call-overload]
+                return excel_file.parse(sheet_name=0)  # type: ignore [no-any-return]
+        except Exception as exc:
+            calamine_exc = exc
+            if logger:
+                logger.warning(
+                    ExcelParser._format_message_with_link(
+                        f"Calamine parsing failed for {file_label}, falling back to openpyxl: {exc}",
+                        file_url,
+                    )
+                )
+        except BaseException as exc:  # noqa: BLE001
+            # PyO3 PanicException from Calamine inherits from BaseException, not Exception
+            if isinstance(exc, (KeyboardInterrupt, SystemExit)):
+                raise
+            calamine_exc = exc
+            if logger:
+                logger.warning(
+                    ExcelParser._format_message_with_link(
+                        f"Calamine parsing failed for {file_label}, falling back to openpyxl: {exc}",
+                        file_url,
+                    )
+                )
+
+        # Fallback to openpyxl
+        try:
+            fp.seek(0)  # type: ignore [union-attr]
+        except (AttributeError, OSError):
+            # Some file-like objects may not be seekable; attempt openpyxl parsing anyway
+            pass
+
+        try:
+            with warnings.catch_warnings(record=True) as warning_records:
+                warnings.simplefilter("always")
+                with pd.ExcelFile(fp, engine="openpyxl") as excel_file:  # type: ignore [arg-type, call-overload]
+                    df = excel_file.parse(sheet_name=0)  # type: ignore [no-any-return]
+            if logger:
+                for warning in warning_records:
+                    logger.warning(
+                        ExcelParser._format_message_with_link(
+                            f"Openpyxl warning for {file_label}: {warning.message}",
+                            file_url,
+                        )
+                    )
+            return df
+        except Exception as openpyxl_exc:
+            # If both engines fail, raise the original calamine exception
+            if logger:
+                logger.error(
+                    ExcelParser._format_message_with_link(
+                        f"Both Calamine and Openpyxl parsing failed for {file_label}. Calamine error: {calamine_exc}, Openpyxl error: {openpyxl_exc}",
+                        file_url,
+                    )
+                )
+            raise calamine_exc if calamine_exc else openpyxl_exc
+
+    @staticmethod
+    def _format_message_with_link(message: str, file_url: Optional[str]) -> str:
+        if file_url:
+            return f"{message} (view: {file_url})"
+        return message

airbyte_cdk/sources/file_based/remote_file.py

@@ -17,6 +17,11 @@ class RemoteFile(BaseModel):
     last_modified: datetime
     mime_type: Optional[str] = None
 
+    @property
+    def file_uri_for_logging(self) -> str:
+        """Returns a user-friendly identifier for logging."""
+        return self.uri
+
 
 class UploadableRemoteFile(RemoteFile, ABC):
     """

pyproject.toml

@@ -73,6 +73,7 @@ pdf2image = { version = "1.16.3", optional = true }
 pyarrow = { version = "^19.0.0", optional = true }
 pytesseract = { version = "0.3.10", optional = true }  # Used indirectly by unstructured library
 python-calamine = { version = "0.2.3", optional = true }  # TODO: Remove if unused
+openpyxl = { version = "^3.1.0", optional = true }
 python-snappy = { version = "0.7.3", optional = true }  # TODO: remove if unused
 tiktoken = { version = "0.8.0", optional = true }
 nltk = { version = "3.9.1", optional = true }
@@ -120,7 +121,7 @@ deptry = "^0.23.0"
 dagger-io = "0.19.0"
 
 [tool.poetry.extras]
-file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfminer.six", "unstructured.pytesseract", "pytesseract", "markdown", "python-calamine", "python-snappy"]
+file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfminer.six", "unstructured.pytesseract", "pytesseract", "markdown", "python-calamine", "openpyxl", "python-snappy"]
 vector-db-based = ["langchain_community", "langchain_core", "langchain_text_splitters", "openai", "cohere", "tiktoken"]
 sql = ["sqlalchemy"]
 dev = ["pytest"]
@@ -252,6 +253,7 @@ DEP002 = [
   "cohere",
   "markdown",
   "openai",
+  "openpyxl",
   "pdf2image",
   "pdfminer.six",
   "pytesseract",

unit_tests/sources/file_based/file_types/test_excel_parser.py

@@ -4,6 +4,7 @@
 
 
 import datetime
+import warnings
 from io import BytesIO
 from unittest.mock import MagicMock, Mock, mock_open, patch
 
@@ -136,3 +137,43 @@ def test_file_read_error(mock_stream_reader, mock_logger, file_config, remote_fi
                 list(
                     parser.parse_records(file_config, remote_file, mock_stream_reader, mock_logger)
                 )
+
+
+class FakePanic(BaseException):
+    """Simulates the PyO3 PanicException which does not inherit from Exception."""
+
+
+def test_open_and_parse_file_falls_back_to_openpyxl(mock_logger):
+    parser = ExcelParser()
+    fp = BytesIO(b"test")
+
+    fallback_df = pd.DataFrame({"a": [1]})
+
+    calamine_ctx = MagicMock()
+    calamine_excel_file = MagicMock()
+    calamine_ctx.__enter__.return_value = calamine_excel_file
+
+    def calamine_parse_side_effect(sheet_name=None):
+        raise FakePanic("calamine panic")
+
+    calamine_excel_file.parse.side_effect = calamine_parse_side_effect
+
+    openpyxl_ctx = MagicMock()
+    openpyxl_excel_file = MagicMock()
+    openpyxl_ctx.__enter__.return_value = openpyxl_excel_file
+
+    def openpyxl_parse_side_effect(sheet_name=None):
+        warnings.warn("Cell A146 has invalid date", UserWarning)
+        return fallback_df
+
+    openpyxl_excel_file.parse.side_effect = openpyxl_parse_side_effect
+
+    with patch("airbyte_cdk.sources.file_based.file_types.excel_parser.pd.ExcelFile") as mock_excel:
+        mock_excel.side_effect = [calamine_ctx, openpyxl_ctx]
+
+        result = parser.open_and_parse_file(fp, mock_logger, "file.xlsx")
+
+    pd.testing.assert_frame_equal(result, fallback_df)
+    assert mock_logger.warning.call_count == 2
+    assert "Openpyxl warning" in mock_logger.warning.call_args_list[1].args[0]
+    mock_logger.error.assert_not_called()