Skip to content

fix: Do not manually set already pre-defined Spark BQ Connector configs #165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
medb merged 2 commits into from
Nov 13, 2025
Merged

fix: Do not manually set already pre-defined Spark BQ Connector configs #165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b57e6b3
fix: Do not manually set already pre-defined Spark BigQuery Connector…
medb Oct 25, 2025
fdaf52c
Fix tests
medb Nov 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions google/cloud/dataproc_spark_connect/session.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -721,8 +721,6 @@ def _get_dataproc_config(self):
# Merge default configs with existing properties,
# user configs take precedence
for k, v in {
"spark.datasource.bigquery.viewsEnabled": "true",
"spark.datasource.bigquery.writeMethod": "direct",
"spark.sql.catalog.spark_catalog": "com.google.cloud.spark.bigquery.BigQuerySparkSessionCatalog",
"spark.sql.sources.default": "bigquery",
}.items():
Expand Down
34 changes: 5 additions & 29 deletions tests/unit/test_session.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1011,7 +1011,7 @@ def test_create_session_with_default_datasource_env_var(
0
]
self.assertNotIn(
"spark.datasource.bigquery.writeMethod",
"spark.sql.sources.default",
create_session_request.session.runtime_config.properties,
)
mock_logger.warning.assert_not_called()
Expand All @@ -1034,18 +1034,6 @@ def test_create_session_with_default_datasource_env_var(
0
]
# BigQuery properties should be set
self.assertEqual(
create_session_request.session.runtime_config.properties.get(
"spark.datasource.bigquery.writeMethod"
),
"direct",
)
self.assertEqual(
create_session_request.session.runtime_config.properties.get(
"spark.datasource.bigquery.viewsEnabled"
),
"true",
)
self.assertEqual(
create_session_request.session.runtime_config.properties.get(
"spark.sql.sources.default"
Expand Down Expand Up @@ -1078,7 +1066,7 @@ def test_create_session_with_default_datasource_env_var(
0
]
self.assertNotIn(
"spark.datasource.bigquery.writeMethod",
"spark.sql.sources.default",
create_session_request.session.runtime_config.properties,
)
mock_logger.warning.assert_called_once_with(
Expand All @@ -1099,7 +1087,7 @@ def test_create_session_with_default_datasource_env_var(
dataproc_config = Session()
dataproc_config.runtime_config.version = "3.0"
dataproc_config.runtime_config.properties = {
"spark.datasource.bigquery.writeMethod": "override_method",
"spark.sql.sources.default": "override_source",
"spark.some.other.property": "some_value",
}
session = DataprocSparkSession.builder.dataprocSessionConfig(
Expand All @@ -1112,24 +1100,12 @@ def test_create_session_with_default_datasource_env_var(
]
# The BigQuery default properties should be set,
# but pre-existing properties should override defaults.
self.assertEqual(
create_session_request.session.runtime_config.properties.get(
"spark.datasource.bigquery.writeMethod"
),
"override_method",
) # Pre-existing property remains
self.assertEqual(
create_session_request.session.runtime_config.properties.get(
"spark.datasource.bigquery.viewsEnabled"
),
"true",
) # Default should still be set
self.assertEqual(
create_session_request.session.runtime_config.properties.get(
"spark.sql.sources.default"
),
"bigquery",
) # Default should still be set
"override_source",
) # Pre-existing property remains
self.assertEqual(
create_session_request.session.runtime_config.properties.get(
"spark.sql.catalog.spark_catalog"
Expand Down