Move eval logic to fasteval project, make codebase a monorepo (#240)

* Move eval logic to fasteval project

* minor update pyproject.toml

* minor update pyproject.toml

* minor update pyproject.toml

* Avoid root internal imports in TabArena

* Update install instructions

* Remove special root import test

* Move tabarena src out of the root directory

* Fix CI

* Rename fasteval -> bencheval

Author: Innixma

.github/workflows/pytest-pytest.yml

@@ -48,7 +48,7 @@ jobs:
 
           # Allow prerelease versions of AutoGluon
           # Install your package in editable/develop mode
-          uv pip install --system --prerelease=allow -e .
+          uv pip install --system --prerelease=allow -e ./tabarena
 
           # Optional tools
           uv pip install --system flake8 pytest

README.md

@@ -50,25 +50,56 @@ TabArena code is currently being polished. Detailed Documentation for TabArena w
 
 To install TabArena, ensure you are using Python 3.9-3.12. Then, run the following:
 
+### Install UV
+
+Ensure [UV is installed](https://docs.astral.sh/uv/getting-started/installation/) for the most stable install.
+
+```
+pip install uv  # if pip is available
+```
+
+### Install AutoGluon
+
+In future AutoGluon installation will occur automatically, but due to changes yet to be released, we need to install AutoGluon from source.
+
+```
+git clone https://github.com/autogluon/autogluon.git
+./autogluon/full_install.sh
+```
+
+### Clone the repository
+
+```
+git clone https://github.com/autogluon/tabarena.git
+cd tabarena  # ensure the working directory is the project root, otherwise the below commands won't work
+```
+
 ### Evaluation (Leaderboard / Metrics)
 
 If you don't intend to fit models, this is the simplest installation.
 
+#### UV Install (recommended)
+
 ```
-git clone https://github.com/autogluon/tabarena.git
-pip install -e tabarena/
+uv pip install --prerelease=allow -e ./tabarena
+```
+
+#### PIP Install (not recommended)
+
+```
+pip install -e ./bencheval
+pip install -e ./tabarena
 ```
 
 ### Benchmark (Fitting Models)
 
 If you intend to fit models, this is required.
 
 ```
-git clone https://github.com/autogluon/tabarena.git
-pip install -e tabarena/[benchmark]
+uv pip install --prerelease=allow -e ./tabarena[benchmark]
 
 # use GIT_LFS_SKIP_SMUDGE=1 in front of the command if installing TabDPT fails due to a broken LFS/pip setup
-# GIT_LFS_SKIP_SMUDGE=1 uv pip install -e tabarena/[benchmark]
+# GIT_LFS_SKIP_SMUDGE=1 uv pip install --prerelease=allow -e ./tabarena/[benchmark]
 ```
 
 ### Developer Install
@@ -80,7 +111,7 @@ git clone https://github.com/autogluon/autogluon.git
 ./autogluon/full_install.sh
 
 git clone https://github.com/autogluon/tabarena.git
-pip install -e tabarena/[benchmark]
+uv pip install --prerelease=allow -e ./tabarena[benchmark]
 ```
 
 ### Example Install + Run
@@ -89,9 +120,12 @@ pip install -e tabarena/[benchmark]
 pip install uv
 uv init -p 3.11
 uv sync
+git clone https://github.com/autogluon/autogluon.git
+./autogluon/full_install.sh
 git clone https://github.com/autogluon/tabarena.git
-uv pip install -e tabarena/[benchmark]
-cd examples/benchmarking/ 
+cd tabarena
+uv pip install --prerelease=allow -e ./tabarena[benchmark]
+cd examples/benchmarking
 python run_quickstart_tabarena.py 
 ```
 

tabarena/benchmark/__init__.py bencheval/bencheval/__init__.pytabarena/benchmark/__init__.py renamed to bencheval/bencheval/__init__.py

@@ -10,9 +10,9 @@
 from pandas.api.types import is_numeric_dtype
 from scipy.stats import gmean
 
-from tabarena.tabarena.elo_utils import EloHelper
-from tabarena.tabarena.mean_utils import compute_weighted_mean_by_task
-from tabarena.tabarena.winrate_utils import compute_winrate, compute_winrate_matrix
+from .elo_utils import EloHelper
+from .mean_utils import compute_weighted_mean_by_task
+from .winrate_utils import compute_winrate, compute_winrate_matrix
 
 RANK = "rank"
 IMPROVABILITY = "improvability"

tabarena/tabarena/elo_utils.py bencheval/bencheval/elo_utils.pytabarena/tabarena/elo_utils.py renamed to bencheval/bencheval/elo_utils.py

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["setuptools>=69", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "bencheval"
+version = "0.0.1"
+requires-python = ">=3.10"
+license = { text = "Apache-2.0" }
+authors = [{ name = "AutoGluon Community" }]
+dependencies = [
+    "scipy",
+    "numpy",
+    "pandas",
+    "tqdm",
+    "typing-extensions>=4.11,<5",  # used for `Self` type hint
+    "scikit-learn",
+    "seaborn==0.13.2",
+    "matplotlib==3.10.1",
+    "plotly>=6.3.0",
+    "kaleido>=0.2.1,<1",  # for winrate matrix
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["bencheval*"]
+
+#[tool.setuptools]
+#package-dir = {}
+#packages = ["bencheval"]
+
+[project.urls]
+Homepage = "https://github.com/autogluon/tabarena"

tabarena/tabarena/mean_utils.py bencheval/bencheval/mean_utils.pytabarena/tabarena/mean_utils.py renamed to bencheval/bencheval/mean_utils.py

@@ -119,7 +119,7 @@ def is_in(dataset: str, fold: int) -> bool:
 
     calibration_framework = "RF (default)"
 
-    from tabarena.tabarena.tabarena import TabArena
+    from bencheval.tabarena import TabArena
     tabarena = TabArena(
         method_col="method",
         task_col="dataset",