fix: restore files unintentionally reverted from main

The T2E commit was built from an older snapshot of main, which
caused it to revert parquet utility functions, pyarrow dependency,
FAQ content, docs references, and related usages across the codebase.

This commit restores all those files to their current main state,
leaving only the actual T2E changes in the PR diff.

Author: anwurl

docs/reference/feature_importance.md docs/concepts/feature_importance.mddocs/reference/feature_importance.md renamed to docs/concepts/feature_importance.md

@@ -1 +1,13 @@
 # FAQ
+
+## When loading `.parquet` files, categorical columns seem to be returned as `int`, losing the information that they were categorical.
+
+This is a known issue with parquet file support in Python.
+Both existing libraries, `pyarrow` as well as `fastparquet` do not exactly reproduce original input data types when it comes to categorical columns.
+See e.g. [Issue 29017](https://github.com/apache/arrow/issues/29017) and [Issue 27067](https://github.com/apache/arrow/issues/27067).
+
+To ensure proper data type roundtrip, the module `octopus.utils` provides the functions `parquet_load()` and `parquet_save()` to store and reconstruct precise dtype information in the parquet metadata.
+Files written with `parquet_save()` are expected to be readable with every parquet-compatible code.
+Still, proper dtypes are only guaranteed to be reconstructed using `parquet_load()`.
+
+For details on which dtypes are tested and supported, see [tests/infrastructure/test_file_io.py](https://github.com/emdgroup/octopus/blob/main/tests/infrastructure/test_file_io.py).

docs/faq.md

@@ -0,0 +1 @@
+::: octopus.diagnostics

docs/reference/diagnostics.md

@@ -0,0 +1 @@
+::: octopus.predict

docs/reference/predict.md

@@ -87,12 +87,15 @@ nav:
       - "concepts/concepts.md"
       - Nested Cross-Validation: "concepts/nested_cv.md"
       - Understanding the Results: "concepts/understanding_results.md"
+      - Feature Importance: "concepts/feature_importance.md"
   - API Reference:
       - "reference/reference.md"
+      - octopus.diagnostics: "reference/diagnostics.md"
       - octopus.manager: "reference/manager.md"
       - octopus.metrics: "reference/metrics.md"
       - octopus.models: "reference/models.md"
       - octopus.modules: "reference/modules.md"
+      - octopus.predict: "reference/predict.md"
       - octopus.study: "reference/study.md"
 
   - Project Info:

mkdocs.yml

@@ -12,8 +12,10 @@
 
 import pandas as pd
 
+from octopus.utils import parquet_load
+
 if TYPE_CHECKING:
-    from pathlib import Path
+    from upath import UPath
 
 
 def _extract_id_from_dirname(dirname: str, prefix: str) -> int | None:
@@ -32,7 +34,7 @@ def _extract_id_from_dirname(dirname: str, prefix: str) -> int | None:
     return None
 
 
-def load_parquet_glob(study_path: Path, pattern: str) -> pd.DataFrame:
+def load_parquet_glob(study_path: UPath, pattern: str) -> pd.DataFrame:
     """Load and concatenate parquet files matching a glob pattern.
 
     Extracts ``outersplit_id`` and ``task_id`` from the directory structure,
@@ -51,7 +53,7 @@ def load_parquet_glob(study_path: Path, pattern: str) -> pd.DataFrame:
     dfs: list[pd.DataFrame] = []
     for parquet_file in sorted(study_path.glob(pattern)):
         try:
-            df = pd.read_parquet(parquet_file)
+            df = parquet_load(parquet_file)
         except Exception:
             continue
 
@@ -72,7 +74,7 @@ def load_parquet_glob(study_path: Path, pattern: str) -> pd.DataFrame:
     return pd.concat(dfs, ignore_index=True)
 
 
-def load_predictions(study_path: Path) -> pd.DataFrame:
+def load_predictions(study_path: UPath) -> pd.DataFrame:
     """Load all predictions parquet files across outersplits and tasks.
 
     Searches in ``outersplit*/task*/results/*/predictions.parquet`` to pick up
@@ -87,7 +89,7 @@ def load_predictions(study_path: Path) -> pd.DataFrame:
     return load_parquet_glob(study_path, "outersplit*/task*/results/*/predictions.parquet")
 
 
-def load_feature_importances(study_path: Path) -> pd.DataFrame:
+def load_feature_importances(study_path: UPath) -> pd.DataFrame:
     """Load all feature importance parquet files across outersplits and tasks.
 
     Searches in ``outersplit*/task*/results/*/feature_importances.parquet`` to pick up
@@ -102,7 +104,7 @@ def load_feature_importances(study_path: Path) -> pd.DataFrame:
     return load_parquet_glob(study_path, "outersplit*/task*/results/*/feature_importances.parquet")
 
 
-def load_optuna(study_path: Path) -> pd.DataFrame:
+def load_optuna(study_path: UPath) -> pd.DataFrame:
     """Load all Optuna parquet files across outersplits and tasks.
 
     Args:
@@ -114,7 +116,7 @@ def load_optuna(study_path: Path) -> pd.DataFrame:
     return load_parquet_glob(study_path, "outersplit*/task*/results/optuna_results.parquet")
 
 
-def load_scores(study_path: Path) -> pd.DataFrame:
+def load_scores(study_path: UPath) -> pd.DataFrame:
     """Load all scores parquet files across outersplits and tasks.
 
     Searches in ``outersplit*/task*/results/*/scores.parquet`` to pick up

octopus/diagnostics/_data_loader.py

@@ -12,7 +12,7 @@
 from octopus.datasplit import OuterSplit
 from octopus.logger import get_logger
 from octopus.modules import ModuleResult, ResultType, StudyContext, Task
-from octopus.utils import calculate_feature_groups
+from octopus.utils import calculate_feature_groups, parquet_save
 
 logger = get_logger()
 
@@ -55,9 +55,9 @@ def run(self, outersplit_id: int, outersplit: OuterSplit) -> None:
         fold_dir = self.study_context.output_path / f"outersplit{outersplit_id}"
         fold_dir.mkdir(parents=True, exist_ok=True)
         train_path = fold_dir / "data_traindev.parquet"
-        outersplit.traindev.to_parquet(str(train_path), storage_options=train_path.storage_options, engine="pyarrow")
+        parquet_save(outersplit.traindev, train_path)
         test_path = fold_dir / "data_test.parquet"
-        outersplit.test.to_parquet(str(test_path), storage_options=test_path.storage_options, engine="pyarrow")
+        parquet_save(outersplit.test, test_path)
 
         # task_results: dict[task_id -> dict[ResultType, ModuleResult]]
         task_results: dict[int, dict[ResultType, ModuleResult]] = {}

octopus/manager/workflow_runner.py

@@ -31,6 +31,7 @@
 from octopus.metrics.utils import get_score_from_model
 from octopus.modules import FIDataset, FIMethod, ModuleExecution, ModuleResult, ResultType, StudyContext
 from octopus.types import MLType
+from octopus.utils import csv_save
 
 if TYPE_CHECKING:
     from octopus.modules import (
@@ -401,19 +402,13 @@ def _save_leaderboard_info(
         # Save leaderboard
         leaderboard = model.leaderboard(ag_test_data, extra_info=True)
         leaderboard_path = results_dir / "leaderboard.csv"
-        leaderboard.to_csv(
-            str(leaderboard_path),
-            storage_options=dict(leaderboard_path.storage_options),
-        )
+        csv_save(leaderboard, leaderboard_path)
 
         # Save best model results
         best_model = leaderboard.iloc[0]
         best_result_df = pd.DataFrame(best_model).transpose()
         best_result_path = results_dir / "best_model_result.csv"
-        best_result_df.to_csv(
-            str(best_result_path),
-            storage_options=dict(best_result_path.storage_options),
-        )
+        csv_save(best_result_df, best_result_path)
 
         # Save model info
         model_info = model.info()

octopus/modules/autogluon/core.py

@@ -16,7 +16,7 @@
 from octopus.models import Models
 from octopus.modules import ModuleExecution, ModuleResult, ResultType
 from octopus.modules.mrmr.core import _maxrminr, _relevance_fstats
-from octopus.utils import joblib_load
+from octopus.utils import joblib_load, parquet_save
 
 from .bag import Bag
 from .enssel import EnSel
@@ -407,11 +407,7 @@ def logging_callback(study, trial):
                     }
                 )
         dict_optuna_path = results_dir / "optuna_results.parquet"
-        pd.DataFrame(dict_optuna).to_parquet(
-            str(dict_optuna_path),
-            storage_options=dict_optuna_path.storage_options,
-            engine="pyarrow",
-        )
+        parquet_save(pd.DataFrame(dict_optuna), dict_optuna_path)
 
         # display results
         logger.set_log_group(LogGroup.SCORES, f"OUTER {outersplit_id} SQE TBD")

octopus/modules/octo/core.py

@@ -433,6 +433,13 @@ def calculate_fi_constant(self):
 
     def calculate_fi_internal(self):
         """Sklearn-provided internal feature importance (based on train dataset)."""
+        # Handle unsupported "timetoevent" case as in your original code
+        if getattr(self, "ml_type", None) == MLType.TIMETOEVENT:
+            fi_df = pd.DataFrame(columns=["feature", "importance"])
+            logger.warning("Internal features importances not available for timetoevent.")
+            self.feature_importances["internal"] = fi_df
+            return
+
         # 1) Tree-based models exposing feature_importances_
         if hasattr(self.model, "feature_importances_"):
             fi = np.asarray(self.model.feature_importances_)