Skip to content

fix: Update Python version mapping for Dataproc runtime 3.0 to Python 3.12 #155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
medb merged 2 commits into from
Oct 21, 2025
Merged

fix: Update Python version mapping for Dataproc runtime 3.0 to Python 3.12 #155

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f0134ad
fix: Update Python version mapping for Dataproc runtime 3.0 to Python…
fangyh20 Oct 20, 2025
8e09657
address comments
fangyh20 Oct 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion google/cloud/dataproc_spark_connect/session.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -734,7 +734,7 @@ def _check_python_version_compatibility(self, runtime_version):

# Runtime version to server Python version mapping
RUNTIME_PYTHON_MAP = {
"3.0": (3, 11),
"3.0": (3, 12),
}

client_python = sys.version_info[:2] # (major, minor)
Expand Down
32 changes: 23 additions & 9 deletions tests/unit/test_init.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,28 +21,42 @@
class TestPythonVersionCheck(unittest.TestCase):

def test_python_version_mismatch_warning_for_runtime_30(self):
"""Test that warning is shown when client Python doesn't match runtime 3.0 (Python 3.11)"""
with mock.patch("sys.version_info", (3, 12, 0)):
"""Test that warning is shown when client Python doesn't match runtime 3.0 (Python 3.12)"""
runtime_version = "3.0"
server_py_major, server_py_minor = 3, 12
client_py_major, client_py_minor = 3, 11

with mock.patch(
"sys.version_info", (client_py_major, client_py_minor, 0)
):
with mock.patch("warnings.warn") as mock_warn:
session_builder = DataprocSparkSession.Builder()
session_builder._check_python_version_compatibility("3.0")
session_builder._check_python_version_compatibility(
runtime_version
)

expected_warning = (
"Python version mismatch detected: Client is using Python 3.12, "
"but Dataproc runtime 3.0 uses Python 3.11. "
f"Python version mismatch detected: Client is using Python {client_py_major}.{client_py_minor}, "
f"but Dataproc runtime {runtime_version} uses Python {server_py_major}.{server_py_minor}. "
"This mismatch may cause issues with Python UDF (User Defined Function) compatibility. "
"Consider using Python 3.11 for optimal UDF execution."
f"Consider using Python {server_py_major}.{server_py_minor} for optimal UDF execution."
)
mock_warn.assert_called_once_with(
expected_warning, stacklevel=3
)

def test_no_warning_when_python_versions_match_runtime_30(self):
"""Test that no warning is shown when client Python matches runtime 3.0 (Python 3.11)"""
with mock.patch("sys.version_info", (3, 11, 0)):
"""Test that no warning is shown when client Python matches runtime 3.0 (Python 3.12)"""
runtime_version = "3.0"
client_py_major, client_py_minor = 3, 12
with mock.patch(
"sys.version_info", (client_py_major, client_py_minor, 0)
):
with mock.patch("warnings.warn") as mock_warn:
session_builder = DataprocSparkSession.Builder()
session_builder._check_python_version_compatibility("3.0")
session_builder._check_python_version_compatibility(
runtime_version
)

mock_warn.assert_not_called()

Expand Down