fix: pin pyarrow version to resolve datasets compatibility

- Add pyarrow >= 8.0.0, < 21.0.0 constraint to pyproject.toml and requirements-dev.txt
- Fixes 'PyExtensionType' AttributeError when importing datasets
- PyArrow 21.0.0 removed PyExtensionType, breaking datasets==2.14.5 compatibility
- Ensures stable test environment and prevents import errors

Author: glennko

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
     "pytorch-lightning",
     "transformers==4.39.3",
     "datasets==2.14.5",
+    "pyarrow >= 8.0.0, < 21.0.0",
     "evaluate==0.4.0",
     "bitsandbytes==0.41.1",
     "sentencepiece",

requirements-dev.txt

@@ -2,3 +2,4 @@ pre-commit
 pytest
 autoflake
 absolufy-imports
+pyarrow >= 8.0.0, < 21.0.0