Updated dependencies, makefile, and developer docs

Author: ghanse

CONTRIBUTING.md

@@ -29,7 +29,6 @@ make lint                # Check code quality
 
 # Run tests
 make test                # Run tests
-make coverage            # Run tests with coverage report
 
 # Build package
 make build               # Build with modern build system
@@ -89,10 +88,6 @@ The name used to flag the spark instance should be the test module or test class
 # Run all tests
 make test
 
-# Run tests with coverage report (generates htmlcov/index.html)
-make coverage
-```
-
 If using an environment with multiple Python versions, make sure to use virtual env or similar to pick up correct python versions.
 
 If necessary, set `PYSPARK_PYTHON` and `PYSPARK_DRIVER_PYTHON` to point to correct versions of Python.

makefile

@@ -1,6 +1,6 @@
-.PHONY: dev test coverage lint fmt clean build docs
+.PHONY: dev test lint fmt clean build docs
 
-all: clean dev lint fmt test coverage
+all: clean dev lint fmt test
 
 clean:
 	rm -fr .venv clean htmlcov .mypy_cache .pytest_cache .ruff_cache .coverage coverage.xml
@@ -22,9 +22,6 @@ fmt:
 test:
 	hatch run test
 
-coverage:
-	hatch run coverage
-
 build:
 	hatch build
 

pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
     "numpy>=1.22.0",
     "pandas>=1.3.4",
     "pyarrow>=7.0.0",
-    "pyspark>=3.3.0",
+    "pyspark[sql]>=3.3.0",
     "python-dateutil>=2.8.2",
     "six>=1.16.0",
     "pyparsing>=3.0.4",
@@ -110,7 +110,7 @@ dependencies = [
     "ruff~=0.3.4",
     "types-PyYAML~=6.0.12",
     "types-requests~=2.31.0",
-    "pyspark~=3.5.0"
+    "pyspark[sql]~=3.5.0"
 ]
 
 python="3.10"
@@ -119,8 +119,7 @@ python="3.10"
 path = ".venv"
 
 [tool.hatch.envs.default.scripts]
-test        = "pytest tests/ -n 10 --cov --cov-report=xml:coverage-unit.xml --timeout 600 --durations 20"
-coverage    = "pytest tests/ -n 10 --cov --cov-report=html --timeout 600 --durations 20"
+test    = "pytest tests/ -n 10 --cov --cov-report=html --timeout 600 --durations 20"
 fmt         = ["ruff check . --fix",
                "mypy .",
                "pylint --output-format=colorized -j 0 dbldatagen tests"]