Make it possible to exclude pyarrow dep

pyproject.toml

@@ -37,13 +37,16 @@ aiohttp = "^3.9.3"
 filelock = "^3.13.1"
 eval-type-backport = ">=0.1.3,<0.3.0"
 click = "^8.1.7"
-pyarrow = ">=10.0.1"
+pyarrow = { version = ">=10.0.1", optional = true }
 numpy = [
     { version = ">=1.23.5", python = "<3.12" },
     { version = ">=1.26.0", python = ">=3.12" },
 ]
 pillow = "^11.1.0"
 
+[tool.poetry.extras]
+pyarrow = ["pyarrow"]
+
 [tool.poetry.group.quality]
 optional = true
 

src/together/utils/files.py

@@ -6,7 +6,6 @@
 from traceback import format_exc
 from typing import Any, Dict, List
 
-from pyarrow import ArrowInvalid, parquet
 
 from together.constants import (
     MAX_FILE_SIZE_GB,
@@ -372,6 +371,12 @@ def _check_jsonl(file: Path) -> Dict[str, Any]:
 
 
 def _check_parquet(file: Path) -> Dict[str, Any]:
+    try:
+        # Pyarrow is optional as it's large (~80MB) and isn't compatible with older systems.
+        from pyarrow import ArrowInvalid, parquet
+    except ImportError:
+        raise ImportError("pyarrow is not installed and is required to use parquet files. Please install it via `pip install together[pyarrow]`")
+
     report_dict: Dict[str, Any] = {}
 
     try: