feat: Session refactoring

.github/workflows/pr.yaml

@@ -39,7 +39,7 @@ jobs:
       - name: Check dependencies
         run: uv run deptry .
       - name: Run tests
-        run: uv run pytest
+        run: make test
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
         with:

Makefile

@@ -38,9 +38,22 @@ check: ## Lint and format code
 	@uv run pydoclint --config=pyproject.toml src
 	@uv run pydoclint --config=pyproject.toml --skip-checking-short-docstrings=true tests
 
-test: ## Run tests
-	@echo "Running Tests..."
-	@uv run pytest
+test-no-shared-spark-session: ## Run tests that can not rely on shared SparkSession.
+	@echo "Running tests that can not rely on shared SparkSession fixture..."
+	@COVERAGE_FILE=.coverage.no_shared_spark uv run pytest -m "no_shared_spark and not download_jars_from_web" -n0 --cov-report=
+
+test-shared-spark-session: ## Run tests that can use shared SparkSession fixture.
+	@echo "Running tests that can share SparkSession fixture..."
+	@COVERAGE_FILE=.coverage.shared_spark uv run pytest --cov-report=
+
+test-no-shared-spark-session-web-dependencies: ## Run tests that require to download spark dependency jars from the web (not run by default).
+	@echo "Running tests that can not rely on shared SparkSession and require downloading jar dependencies from web..."
+	@COVERAGE_FILE=.coverage.no_shared_spark_web_deps uv run pytest -m "download_jars_from_web" --cov-report=
+
+test: test-no-shared-spark-session test-shared-spark-session ## Run default test suite
+	@uv run coverage combine .coverage.shared_spark .coverage.no_shared_spark
+	@uv run coverage xml
+	@rm -f .coverage.shared_spark .coverage.no_shared_spark
 
 build-documentation: ## Create local server with documentation
 	@echo "Building Documentation..."

docs/python_api/common/session.md

@@ -5,4 +5,4 @@ title: session
 ## Spark Session wrapper for gentropy
 
 :::gentropy.common.session.Session
-:::gentropy.common.session.Log4j
+:::gentropy.common.session.SparkWriteMode

pyproject.toml

@@ -130,13 +130,12 @@ color = true
 exclude = ["dist"]
 
 [tool.pytest.ini_options]
-addopts = "-n auto --doctest-modules --cov=src/ --cov-report=xml --cache-clear"
+addopts = "-n auto --doctest-modules --cov=src/ --cov-report=xml --cache-clear -m 'not download_jars_from_web and not no_shared_spark'"
 pythonpath = ["."]
 testpaths = ["tests/gentropy", "src/gentropy"]
-markers = ["step_test", "long_test"]
+markers = ["step_test", "download_jars_from_web", "no_shared_spark"]
 filterwarnings = [
   "ignore:.*it is preferred to specify type hints for pandas UDF.*:UserWarning"
-
 ]
 
 # Semi-strict mode for mypy

src/gentropy/__init__.py

@@ -4,12 +4,18 @@
 
 import warnings
 
-# NOTE: Suppress DeprecationWarnings from pyspark related to pandas API on Spark due to LooseVersion being deprecated in Python 3.12+
+# NOTE: Suppress DeprecationWarnings and UserWarnings from pyspark related to pandas API on Spark due to LooseVersion being deprecated in Python 3.12+
 warnings.filterwarnings(
     "ignore",
     category=DeprecationWarning,
     module="pyspark.sql.pandas.utils",
 )
+warnings.filterwarnings(
+    "ignore",
+    category=UserWarning,
+    module="pyspark.sql.pandas.functions",
+)
+
 
 from gentropy.common.session import Session
 from gentropy.dataset.biosample_index import BiosampleIndex

src/gentropy/assets/log4j.properties

@@ -0,0 +1,12 @@
+# Source - https://stackoverflow.com/a/76196464
+
+# Set everything to be logged to the console
+log4j.rootCategory=ERROR, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+
+# Set the log level to ERROR for everything
+log4j.logger.org.apache=ERROR
+log4j.logger.org.apache.spark=ERROR