From 59f41472a5fa67a822e39029e1e43858f9d00c4d Mon Sep 17 00:00:00 2001 From: David Rapan Date: Tue, 28 Oct 2025 15:04:57 +0100 Subject: [PATCH 01/16] Add support for aiomysql, aiosqlite and asyncpg Signed-off-by: David Rapan --- homeassistant/components/sql/manifest.json | 9 ++- homeassistant/components/sql/models.py | 7 +- homeassistant/components/sql/sensor.py | 67 +++++++++++-------- homeassistant/components/sql/services.py | 77 +++++++++++++--------- homeassistant/components/sql/util.py | 72 +++++++++++++++----- requirements_all.txt | 12 ++++ requirements_test_all.txt | 12 ++++ script/hassfest/requirements.py | 2 + tests/components/sql/test_sensor.py | 66 ++++++++++++++++++- 9 files changed, 248 insertions(+), 76 deletions(-) diff --git a/homeassistant/components/sql/manifest.json b/homeassistant/components/sql/manifest.json index 244334565657e..fe41ebd3d42c2 100644 --- a/homeassistant/components/sql/manifest.json +++ b/homeassistant/components/sql/manifest.json @@ -6,5 +6,12 @@ "config_flow": true, "documentation": "https://www.home-assistant.io/integrations/sql", "iot_class": "local_polling", - "requirements": ["SQLAlchemy==2.0.41", "sqlparse==0.5.0"] + "requirements": [ + "aiomysql==0.3.2", + "aiosqlite==0.21.0", + "asyncpg==0.30.0", + "greenlet==3.2.4", + "SQLAlchemy==2.0.41", + "sqlparse==0.5.0" + ] } diff --git a/homeassistant/components/sql/models.py b/homeassistant/components/sql/models.py index 872ceedde71b9..45ad238573f0a 100644 --- a/homeassistant/components/sql/models.py +++ b/homeassistant/components/sql/models.py @@ -4,7 +4,8 @@ from dataclasses import dataclass -from sqlalchemy.orm import scoped_session +from sqlalchemy.ext.asyncio import AsyncSession, async_scoped_session +from sqlalchemy.orm import Session, scoped_session from homeassistant.core import CALLBACK_TYPE @@ -14,4 +15,6 @@ class SQLData: """Data for the sql integration.""" shutdown_event_cancel: CALLBACK_TYPE - session_makers_by_db_url: dict[str, scoped_session] + session_makers_by_db_url: dict[ + str, async_scoped_session[AsyncSession] | scoped_session[Session] + ] diff --git a/homeassistant/components/sql/sensor.py b/homeassistant/components/sql/sensor.py index 508365b5c0dce..facd38dfa2905 100644 --- a/homeassistant/components/sql/sensor.py +++ b/homeassistant/components/sql/sensor.py @@ -5,11 +5,12 @@ from datetime import date import decimal import logging -from typing import Any +from typing import TYPE_CHECKING, Any from sqlalchemy.engine import Result from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy.orm import scoped_session +from sqlalchemy.ext.asyncio import AsyncSession, async_scoped_session +from sqlalchemy.orm import Session, scoped_session from homeassistant.components.recorder import CONF_DB_URL, get_instance from homeassistant.components.sensor import CONF_STATE_CLASS @@ -200,7 +201,7 @@ class SQLSensor(ManualTriggerSensorEntity): def __init__( self, trigger_entity_config: ConfigType, - sessmaker: scoped_session, + sessmaker: async_scoped_session[AsyncSession] | scoped_session[Session], query: str, column: str, value_template: ValueTemplate | None, @@ -243,31 +244,10 @@ def extra_state_attributes(self) -> dict[str, Any] | None: """Return extra attributes.""" return dict(self._attr_extra_state_attributes) - async def async_update(self) -> None: - """Retrieve sensor data from the query using the right executor.""" - if self._use_database_executor: - await get_instance(self.hass).async_add_executor_job(self._update) - else: - await self.hass.async_add_executor_job(self._update) - - def _update(self) -> None: - """Retrieve sensor data from the query.""" + def _process(self, result: Result) -> None: + """Process the SQL result.""" data = None extra_state_attributes = {} - self._attr_extra_state_attributes = {} - sess: scoped_session = self.sessionmaker() - try: - result: Result = sess.execute(self._lambda_stmt) - except SQLAlchemyError as err: - _LOGGER.error( - "Error executing query %s: %s", - self._query, - redact_credentials(str(err)), - ) - sess.rollback() - sess.close() - return - for res in result.mappings(): _LOGGER.debug("Query %s result in %s", self._query, res.items()) data = res[self._column_name] @@ -298,4 +278,37 @@ def _update(self) -> None: if data is None: _LOGGER.warning("%s returned no results", self._query) - sess.close() + def _update(self) -> None: + """Retrieve sensor data from the query.""" + self._attr_extra_state_attributes = {} + if TYPE_CHECKING: + assert isinstance(self.sessionmaker, scoped_session) + with self.sessionmaker() as session: + try: + self._process(session.execute(self._lambda_stmt)) + except SQLAlchemyError as err: + _LOGGER.error( + "Error executing query %s: %s", + self._query, + redact_credentials(str(err)), + ) + session.rollback() + + async def async_update(self) -> None: + """Retrieve sensor data from the query using the right executor.""" + if isinstance(self.sessionmaker, async_scoped_session): + self._attr_extra_state_attributes = {} + async with self.sessionmaker() as session: + try: + self._process(await session.execute(self._lambda_stmt)) + except SQLAlchemyError as err: + _LOGGER.error( + "Error executing query %s: %s", + self._query, + redact_credentials(str(err)), + ) + await session.rollback() + elif self._use_database_executor: + await get_instance(self.hass).async_add_executor_job(self._update) + else: + await self.hass.async_add_executor_job(self._update) diff --git a/homeassistant/components/sql/services.py b/homeassistant/components/sql/services.py index c7b74bd82b6ed..9b6e9e3686550 100644 --- a/homeassistant/components/sql/services.py +++ b/homeassistant/components/sql/services.py @@ -5,10 +5,12 @@ import datetime import decimal import logging +from typing import TYPE_CHECKING from sqlalchemy.engine import Result from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy.orm import Session +from sqlalchemy.ext.asyncio import async_scoped_session +from sqlalchemy.orm import scoped_session import voluptuous as vol from homeassistant.components.recorder import CONF_DB_URL, get_instance @@ -70,39 +72,54 @@ async def _async_query_service( translation_placeholders={"db_url": redact_credentials(db_url)}, ) + def _process(result: Result) -> list[JsonValueType]: + rows: list[JsonValueType] = [] + for row in result.mappings(): + processed_row: dict[str, JsonValueType] = {} + for key, value in row.items(): + if isinstance(value, decimal.Decimal): + processed_row[key] = float(value) + elif isinstance(value, datetime.date): + processed_row[key] = value.isoformat() + elif isinstance(value, (bytes, bytearray)): + processed_row[key] = f"0x{value.hex()}" + else: + processed_row[key] = value + rows.append(processed_row) + return rows + def _execute_and_convert_query() -> list[JsonValueType]: """Execute the query and return the results with converted types.""" - sess: Session = sessmaker() - try: - result: Result = sess.execute(generate_lambda_stmt(query_str)) - except SQLAlchemyError as err: - _LOGGER.debug( - "Error executing query %s: %s", - query_str, - redact_credentials(str(err)), - ) - sess.rollback() - raise - else: - rows: list[JsonValueType] = [] - for row in result.mappings(): - processed_row: dict[str, JsonValueType] = {} - for key, value in row.items(): - if isinstance(value, decimal.Decimal): - processed_row[key] = float(value) - elif isinstance(value, datetime.date): - processed_row[key] = value.isoformat() - elif isinstance(value, (bytes, bytearray)): - processed_row[key] = f"0x{value.hex()}" - else: - processed_row[key] = value - rows.append(processed_row) - return rows - finally: - sess.close() + if TYPE_CHECKING: + assert isinstance(sessmaker, scoped_session) + with sessmaker() as session: + try: + return _process(session.execute(generate_lambda_stmt(query_str))) + except SQLAlchemyError as err: + _LOGGER.debug( + "Error executing query %s: %s", + query_str, + redact_credentials(str(err)), + ) + session.rollback() + raise try: - if use_database_executor: + if isinstance(sessmaker, async_scoped_session): + async with sessmaker() as session: + try: + result = _process( + await session.execute(generate_lambda_stmt(query_str)) + ) + except SQLAlchemyError as err: + _LOGGER.debug( + "Error executing query %s: %s", + query_str, + redact_credentials(str(err)), + ) + await session.rollback() + raise + elif use_database_executor: result = await get_instance(call.hass).async_add_executor_job( _execute_and_convert_query ) diff --git a/homeassistant/components/sql/util.py b/homeassistant/components/sql/util.py index 0200a83c9e849..269a6ed695abb 100644 --- a/homeassistant/components/sql/util.py +++ b/homeassistant/components/sql/util.py @@ -2,11 +2,18 @@ from __future__ import annotations +import asyncio import logging import sqlalchemy from sqlalchemy import lambda_stmt from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.ext.asyncio import ( + AsyncSession, + async_scoped_session, + async_sessionmaker, + create_async_engine, +) from sqlalchemy.orm import Session, scoped_session, sessionmaker from sqlalchemy.sql.lambdas import StatementLambdaElement from sqlalchemy.util import LRUCache @@ -55,7 +62,9 @@ def validate_sql_select(value: str) -> str: async def async_create_sessionmaker( hass: HomeAssistant, db_url: str -) -> tuple[scoped_session | None, bool, bool]: +) -> tuple[ + async_scoped_session[AsyncSession] | scoped_session[Session] | None, bool, bool +]: """Create a session maker for the given db_url. This function gets or creates a SQLAlchemy `scoped_session` for the given @@ -83,7 +92,7 @@ async def async_create_sessionmaker( uses_recorder_db = False else: uses_recorder_db = db_url == instance.db_url - sessmaker: scoped_session | None + sessmaker: async_scoped_session[AsyncSession] | scoped_session[Session] | None sql_data = _async_get_or_init_domain_data(hass) use_database_executor = False if uses_recorder_db and instance.dialect_name == SupportedDialect.SQLITE: @@ -98,6 +107,9 @@ async def async_create_sessionmaker( # for every sensor. elif db_url in sql_data.session_makers_by_db_url: sessmaker = sql_data.session_makers_by_db_url[db_url] + elif "+aiomysql" in db_url or "+aiosqlite" in db_url or "+asyncpg" in db_url: + if sessmaker := await _async_validate_and_get_session_maker_for_db_url(db_url): + sql_data.session_makers_by_db_url[db_url] = sessmaker elif sessmaker := await hass.async_add_executor_job( _validate_and_get_session_maker_for_db_url, db_url ): @@ -169,7 +181,9 @@ def _async_get_or_init_domain_data(hass: HomeAssistant) -> SQLData: sql_data: SQLData = hass.data[DOMAIN] return sql_data - session_makers_by_db_url: dict[str, scoped_session] = {} + session_makers_by_db_url: dict[ + str, async_scoped_session[AsyncSession] | scoped_session[Session] + ] = {} # # Ensure we dispose of all engines at shutdown @@ -178,10 +192,13 @@ def _async_get_or_init_domain_data(hass: HomeAssistant) -> SQLData: # Shutdown all sessions in the executor since they will # do blocking I/O # - def _shutdown_db_engines(event: Event) -> None: + async def _shutdown_db_engines(_: Event) -> None: """Shutdown all database engines.""" for sessmaker in session_makers_by_db_url.values(): - sessmaker.connection().engine.dispose() + if isinstance(sessmaker, async_scoped_session): + await (await sessmaker.connection()).engine.dispose() + else: + sessmaker.connection().engine.dispose() cancel_shutdown = hass.bus.async_listen_once( EVENT_HOMEASSISTANT_STOP, _shutdown_db_engines @@ -192,18 +209,46 @@ def _shutdown_db_engines(event: Event) -> None: return sql_data -def _validate_and_get_session_maker_for_db_url(db_url: str) -> scoped_session | None: +async def _async_validate_and_get_session_maker_for_db_url( + db_url: str, +) -> async_scoped_session[AsyncSession] | None: + """Validate the db_url and return a async session maker.""" + try: + maker = async_scoped_session( + async_sessionmaker(bind=create_async_engine(db_url)), + scopefunc=asyncio.current_task, + ) + # Run a dummy query just to test the db_url + async with maker() as session: + await session.execute(sqlalchemy.text("SELECT 1;")) + + except SQLAlchemyError as err: + _LOGGER.error( + "Couldn't connect using %s DB_URL: %s", + redact_credentials(db_url), + redact_credentials(str(err)), + ) + return None + else: + return maker + + +def _validate_and_get_session_maker_for_db_url( + db_url: str, +) -> scoped_session[Session] | None: """Validate the db_url and return a session maker. This does I/O and should be run in the executor. """ - sess: Session | None = None try: - engine = sqlalchemy.create_engine(db_url, future=True) - sessmaker = scoped_session(sessionmaker(bind=engine, future=True)) + maker = scoped_session( + sessionmaker( + bind=sqlalchemy.create_engine(db_url, future=True), future=True + ) + ) # Run a dummy query just to test the db_url - sess = sessmaker() - sess.execute(sqlalchemy.text("SELECT 1;")) + with maker() as session: + session.execute(sqlalchemy.text("SELECT 1;")) except SQLAlchemyError as err: _LOGGER.error( @@ -213,10 +258,7 @@ def _validate_and_get_session_maker_for_db_url(db_url: str) -> scoped_session | ) return None else: - return sessmaker - finally: - if sess: - sess.close() + return maker def generate_lambda_stmt(query: str) -> StatementLambdaElement: diff --git a/requirements_all.txt b/requirements_all.txt index f558d76a0e5d5..6cefb2aae9ba3 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -327,6 +327,9 @@ aiomodernforms==0.1.8 # homeassistant.components.yamaha_musiccast aiomusiccast==0.14.8 +# homeassistant.components.sql +aiomysql==0.3.2 + # homeassistant.components.nanoleaf aionanoleaf==0.2.1 @@ -404,6 +407,9 @@ aioslimproto==3.0.0 # homeassistant.components.solaredge aiosolaredge==0.2.0 +# homeassistant.components.sql +aiosqlite==0.21.0 + # homeassistant.components.steamist aiosteamist==1.0.1 @@ -553,6 +559,9 @@ asyncarve==0.1.1 # homeassistant.components.keyboard_remote asyncinotify==4.2.0 +# homeassistant.components.sql +asyncpg==0.30.0 + # homeassistant.components.supla asyncpysupla==0.0.5 @@ -1120,6 +1129,9 @@ greeclimate==2.1.0 # homeassistant.components.greeneye_monitor greeneye_monitor==3.0.3 +# homeassistant.components.sql +greenlet==3.2.4 + # homeassistant.components.greenwave greenwavereality==0.5.1 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index a0d00fe70361e..43e2c1712813e 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -309,6 +309,9 @@ aiomodernforms==0.1.8 # homeassistant.components.yamaha_musiccast aiomusiccast==0.14.8 +# homeassistant.components.sql +aiomysql==0.3.2 + # homeassistant.components.nanoleaf aionanoleaf==0.2.1 @@ -386,6 +389,9 @@ aioslimproto==3.0.0 # homeassistant.components.solaredge aiosolaredge==0.2.0 +# homeassistant.components.sql +aiosqlite==0.21.0 + # homeassistant.components.steamist aiosteamist==1.0.1 @@ -514,6 +520,9 @@ async-upnp-client==0.45.0 # homeassistant.components.arve asyncarve==0.1.1 +# homeassistant.components.sql +asyncpg==0.30.0 + # homeassistant.components.sleepiq asyncsleepiq==1.6.0 @@ -984,6 +993,9 @@ greeclimate==2.1.0 # homeassistant.components.greeneye_monitor greeneye_monitor==3.0.3 +# homeassistant.components.sql +greenlet==3.2.4 + # homeassistant.components.pure_energie gridnet==5.0.1 diff --git a/script/hassfest/requirements.py b/script/hassfest/requirements.py index f1048b866e22b..6be1162049ca1 100644 --- a/script/hassfest/requirements.py +++ b/script/hassfest/requirements.py @@ -324,6 +324,8 @@ }, # https://github.com/smappee/pysmappee "smappee": {"homeassistant": {"pysmappee"}}, + # https://github.com/aio-libs/aiomysql + "sql": {"homeassistant": {"aiomysql"}}, # https://github.com/watergate-ai/watergate-local-api-python "watergate": {"homeassistant": {"watergate-local-api"}}, # https://github.com/markusressel/xs1-api-client diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index 73879065999f6..bee480a5a795d 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -2,10 +2,12 @@ from __future__ import annotations +import copy from datetime import timedelta from pathlib import Path import sqlite3 -from typing import Any +import types +from typing import Any, Self from unittest.mock import patch from freezegun.api import FrozenDateTimeFactory @@ -255,6 +257,17 @@ async def test_invalid_url_on_update( class MockSession: """Mock session.""" + def __enter__(self) -> Self: + return self + + def __exit__( + self, + type: type[BaseException] | None, + value: BaseException | None, + traceback: types.TracebackType | None, + ) -> None: + pass + def execute(self, query: Any) -> None: """Execute the query.""" raise SQLAlchemyError("sqlite://homeassistant:hunter2@homeassistant.local") @@ -283,6 +296,21 @@ async def test_query_from_yaml(recorder_mock: Recorder, hass: HomeAssistant) -> assert state.state == "5" +async def test_async_query_from_yaml( + recorder_mock: Recorder, hass: HomeAssistant +) -> None: + """Test the SQL sensor from yaml config using async driver.""" + + config = copy.deepcopy(YAML_CONFIG) + config["sql"][CONF_DB_URL] = "sqlite+aiosqlite://" + + assert await async_setup_component(hass, DOMAIN, config) + await hass.async_block_till_done() + + state = hass.states.get("sensor.get_value") + assert state.state == "5" + + async def test_templates_with_yaml( recorder_mock: Recorder, hass: HomeAssistant ) -> None: @@ -628,6 +656,42 @@ async def test_attributes_from_entry_config( assert CONF_STATE_CLASS not in state.attributes +async def test_session_rollback_on_error( + recorder_mock: Recorder, + hass: HomeAssistant, + freezer: FrozenDateTimeFactory, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test the SQL sensor.""" + options = { + CONF_QUERY: "SELECT 5 as value", + CONF_COLUMN_NAME: "value", + CONF_UNIQUE_ID: "very_unique_id", + } + await init_integration(hass, title="Select value SQL query", options=options) + platforms = async_get_platforms(hass, "sql") + sql_entity = platforms[0].entities["sensor.select_value_sql_query"] + + state = hass.states.get("sensor.select_value_sql_query") + assert state.state == "5" + assert state.attributes["value"] == 5 + + with ( + patch.object( + sql_entity, + "_lambda_stmt", + generate_lambda_stmt("Faulty syntax create operational issue"), + ), + patch("sqlalchemy.orm.session.Session.rollback") as mock_session_rollback, + ): + freezer.tick(timedelta(minutes=1)) + async_fire_time_changed(hass) + await hass.async_block_till_done(wait_background_tasks=True) + assert "sqlite3.OperationalError" in caplog.text + + assert mock_session_rollback.call_count == 1 + + async def test_query_recover_from_rollback( recorder_mock: Recorder, hass: HomeAssistant, From 9bc949c489c7528efd444f32150f18cc616e6899 Mon Sep 17 00:00:00 2001 From: David Rapan Date: Tue, 28 Oct 2025 16:48:50 +0100 Subject: [PATCH 02/16] Remove aiomysql from hassfest/requirements.py Signed-off-by: David Rapan --- script/hassfest/requirements.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/script/hassfest/requirements.py b/script/hassfest/requirements.py index 6be1162049ca1..f1048b866e22b 100644 --- a/script/hassfest/requirements.py +++ b/script/hassfest/requirements.py @@ -324,8 +324,6 @@ }, # https://github.com/smappee/pysmappee "smappee": {"homeassistant": {"pysmappee"}}, - # https://github.com/aio-libs/aiomysql - "sql": {"homeassistant": {"aiomysql"}}, # https://github.com/watergate-ai/watergate-local-api-python "watergate": {"homeassistant": {"watergate-local-api"}}, # https://github.com/markusressel/xs1-api-client From f772f6c02641657694890c6715482e8c9d620c6f Mon Sep 17 00:00:00 2001 From: David Rapan Date: Tue, 28 Oct 2025 16:49:25 +0100 Subject: [PATCH 03/16] Add aiomysql to hassfest/requirements.py Signed-off-by: David Rapan --- script/hassfest/requirements.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/hassfest/requirements.py b/script/hassfest/requirements.py index f1048b866e22b..6be1162049ca1 100644 --- a/script/hassfest/requirements.py +++ b/script/hassfest/requirements.py @@ -324,6 +324,8 @@ }, # https://github.com/smappee/pysmappee "smappee": {"homeassistant": {"pysmappee"}}, + # https://github.com/aio-libs/aiomysql + "sql": {"homeassistant": {"aiomysql"}}, # https://github.com/watergate-ai/watergate-local-api-python "watergate": {"homeassistant": {"watergate-local-api"}}, # https://github.com/markusressel/xs1-api-client From 56b7d17ae94a9cf00bfd8f768467cddeb865f179 Mon Sep 17 00:00:00 2001 From: David Rapan Date: Tue, 28 Oct 2025 17:50:54 +0100 Subject: [PATCH 04/16] Add test_sensor.test_async_session_rollback_on_error Signed-off-by: David Rapan --- tests/components/sql/test_sensor.py | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index bee480a5a795d..eb47342d8b1c2 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -692,6 +692,47 @@ async def test_session_rollback_on_error( assert mock_session_rollback.call_count == 1 +async def test_async_session_rollback_on_error( + recorder_mock: Recorder, + hass: HomeAssistant, + freezer: FrozenDateTimeFactory, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test the SQL sensor.""" + config = {CONF_DB_URL: "sqlite+aiosqlite:///"} + options = { + CONF_QUERY: "SELECT 5 as value", + CONF_COLUMN_NAME: "value", + CONF_UNIQUE_ID: "very_unique_id", + } + await init_integration( + hass, title="Select value SQL query", config=config, options=options + ) + platforms = async_get_platforms(hass, "sql") + sql_entity = platforms[0].entities["sensor.select_value_sql_query"] + + state = hass.states.get("sensor.select_value_sql_query") + assert state.state == "5" + assert state.attributes["value"] == 5 + + with ( + patch.object( + sql_entity, + "_lambda_stmt", + generate_lambda_stmt("Faulty syntax create operational issue"), + ), + patch( + "sqlalchemy.ext.asyncio.session.AsyncSession.rollback" + ) as mock_async_session_rollback, + ): + freezer.tick(timedelta(minutes=1)) + async_fire_time_changed(hass) + await hass.async_block_till_done(wait_background_tasks=True) + assert "sqlite3.OperationalError" in caplog.text + + assert mock_async_session_rollback.call_count == 1 + + async def test_query_recover_from_rollback( recorder_mock: Recorder, hass: HomeAssistant, From cc11b8595874413ad8fc3e98de0bdb128f7d7464 Mon Sep 17 00:00:00 2001 From: David Rapan Date: Tue, 28 Oct 2025 18:48:16 +0100 Subject: [PATCH 05/16] Add test_sensor.test_async_invalid_url_setup Signed-off-by: David Rapan --- tests/components/sql/test_sensor.py | 53 +++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index eb47342d8b1c2..d007d35dca3bd 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -243,6 +243,59 @@ async def test_invalid_url_setup( assert pattern in caplog.text +@pytest.mark.parametrize( + ("url", "expected_patterns", "not_expected_patterns"), + [ + ( + "sqlite+aiosqlite://homeassistant:hunter2@homeassistant.local", + ["sqlite+aiosqlite://****:****@homeassistant.local"], + ["sqlite+aiosqlite://homeassistant:hunter2@homeassistant.local"], + ), + ( + "sqlite+aiosqlite://homeassistant.local", + ["sqlite+aiosqlite://homeassistant.local"], + [], + ), + ], +) +async def test_async_invalid_url_setup( + recorder_mock: Recorder, + hass: HomeAssistant, + caplog: pytest.LogCaptureFixture, + url: str, + expected_patterns: str, + not_expected_patterns: str, +) -> None: + """Test invalid db url with redacted credentials.""" + config = { + CONF_QUERY: "SELECT 5 as value", + CONF_COLUMN_NAME: "value", + } + entry = MockConfigEntry( + title="count_tables", + domain=DOMAIN, + source=SOURCE_USER, + data={CONF_DB_URL: url}, + options=config, + entry_id="1", + version=2, + ) + + entry.add_to_hass(hass) + + with patch( + "homeassistant.components.sql.util.create_async_engine", + side_effect=SQLAlchemyError(url), + ): + await hass.config_entries.async_setup(entry.entry_id) + await hass.async_block_till_done() + + for pattern in not_expected_patterns: + assert pattern not in caplog.text + for pattern in expected_patterns: + assert pattern in caplog.text + + async def test_invalid_url_on_update( recorder_mock: Recorder, hass: HomeAssistant, From bb93d46407b8ff7f410aa7b9406465db2aa2d7a1 Mon Sep 17 00:00:00 2001 From: David Rapan Date: Wed, 29 Oct 2025 12:09:24 +0100 Subject: [PATCH 06/16] Add test_sensor.test_multiple_sensors_using_same_external_db Signed-off-by: David Rapan --- homeassistant/components/sql/sensor.py | 8 +++-- homeassistant/components/sql/util.py | 4 ++- tests/components/sql/test_sensor.py | 47 ++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/sql/sensor.py b/homeassistant/components/sql/sensor.py index facd38dfa2905..9ea11069d7c0e 100644 --- a/homeassistant/components/sql/sensor.py +++ b/homeassistant/components/sql/sensor.py @@ -279,8 +279,10 @@ def _process(self, result: Result) -> None: _LOGGER.warning("%s returned no results", self._query) def _update(self) -> None: - """Retrieve sensor data from the query.""" - self._attr_extra_state_attributes = {} + """Retrieve sensor data from the query. + + This does I/O and should be run in the executor. + """ if TYPE_CHECKING: assert isinstance(self.sessionmaker, scoped_session) with self.sessionmaker() as session: @@ -296,8 +298,8 @@ def _update(self) -> None: async def async_update(self) -> None: """Retrieve sensor data from the query using the right executor.""" + self._attr_extra_state_attributes = {} if isinstance(self.sessionmaker, async_scoped_session): - self._attr_extra_state_attributes = {} async with self.sessionmaker() as session: try: self._process(await session.execute(self._lambda_stmt)) diff --git a/homeassistant/components/sql/util.py b/homeassistant/components/sql/util.py index 269a6ed695abb..d70b251012720 100644 --- a/homeassistant/components/sql/util.py +++ b/homeassistant/components/sql/util.py @@ -215,7 +215,9 @@ async def _async_validate_and_get_session_maker_for_db_url( """Validate the db_url and return a async session maker.""" try: maker = async_scoped_session( - async_sessionmaker(bind=create_async_engine(db_url)), + async_sessionmaker( + bind=create_async_engine(db_url, future=True), future=True + ), scopefunc=asyncio.current_task, ) # Run a dummy query just to test the db_url diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index d007d35dca3bd..daa9a4eebc419 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -638,6 +638,53 @@ async def test_multiple_sensors_using_same_db( await hass.async_stop() +async def test_multiple_sensors_using_same_external_db( + recorder_mock: Recorder, hass: HomeAssistant, tmp_path: Path +) -> None: + """Test multiple sensors using the same external db.""" + db_path = tmp_path / "test.db" + + # Create and populate the external database + conn = sqlite3.connect(db_path) + conn.execute("CREATE TABLE users (name TEXT, age INTEGER)") + conn.execute("INSERT INTO users (name, age) VALUES ('Alice', 30), ('Bob', 25)") + conn.commit() + conn.close() + + config = {CONF_DB_URL: f"sqlite:///{db_path}"} + config2 = {CONF_DB_URL: f"sqlite:///{db_path}"} + options = { + CONF_QUERY: "SELECT name FROM users ORDER BY age LIMIT 1", + CONF_COLUMN_NAME: "name", + } + options2 = { + CONF_QUERY: "SELECT name FROM users ORDER BY age DESC LIMIT 1", + CONF_COLUMN_NAME: "name", + } + + await init_integration( + hass, title="Select name SQL query", config=config, options=options + ) + + assert hass.data["sql"] + assert len(hass.data["sql"].session_makers_by_db_url) == 1 + assert hass.states.get("sensor.select_name_sql_query").state == "Bob" + + await init_integration( + hass, + title="Select name SQL query 2", + config=config2, + options=options2, + entry_id="2", + ) + + assert len(hass.data["sql"].session_makers_by_db_url) == 1 + assert hass.states.get("sensor.select_name_sql_query_2").state == "Alice" + + with patch("sqlalchemy.engine.base.Engine.dispose"): + await hass.async_stop() + + async def test_engine_is_disposed_at_stop( recorder_mock: Recorder, hass: HomeAssistant ) -> None: From 46e00cec686add6395a0e77eb04706949a226f34 Mon Sep 17 00:00:00 2001 From: David Rapan Date: Wed, 29 Oct 2025 12:18:06 +0100 Subject: [PATCH 07/16] Add test_services.test_query_service_rollback_on_error Signed-off-by: David Rapan --- tests/components/sql/test_services.py | 41 +++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/components/sql/test_services.py b/tests/components/sql/test_services.py index ad1fa20215342..17ea68717077c 100644 --- a/tests/components/sql/test_services.py +++ b/tests/components/sql/test_services.py @@ -7,6 +7,7 @@ from unittest.mock import patch import pytest +from sqlalchemy.exc import SQLAlchemyError import voluptuous as vol from voluptuous import MultipleInvalid @@ -86,6 +87,46 @@ async def test_query_service_external_db(hass: HomeAssistant, tmp_path: Path) -> } +async def test_query_service_rollback_on_error( + hass: HomeAssistant, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test the query service.""" + db_path = tmp_path / "test.db" + db_url = f"sqlite:///{db_path}" + + # Create and populate the external database + conn = sqlite3.connect(db_path) + conn.execute("CREATE TABLE users (name TEXT, age INTEGER)") + conn.execute("INSERT INTO users (name, age) VALUES ('Alice', 30), ('Bob', 25)") + conn.commit() + conn.close() + + await async_setup_component(hass, DOMAIN, {}) + await hass.async_block_till_done() + + with ( + patch( + "homeassistant.components.sql.services.generate_lambda_stmt", + side_effect=SQLAlchemyError("Error executing query"), + ), + pytest.raises( + ServiceValidationError, match="An error occurred when executing the query" + ), + patch("sqlalchemy.orm.session.Session.rollback") as mock_session_rollback, + ): + await hass.services.async_call( + DOMAIN, + SERVICE_QUERY, + {"query": "SELECT name, age FROM users ORDER BY age", "db_url": db_url}, + blocking=True, + return_response=True, + ) + + assert mock_session_rollback.call_count == 1 + + async def test_query_service_data_conversion( hass: HomeAssistant, tmp_path: Path ) -> None: From b5f875826fa6ceb1534004d8ae2cfe7c4d3f9d31 Mon Sep 17 00:00:00 2001 From: David Rapan Date: Wed, 29 Oct 2025 15:38:09 +0100 Subject: [PATCH 08/16] Merge util._async_validate_and_get_session_maker_for_db_url Signed-off-by: David Rapan --- homeassistant/components/sql/util.py | 79 +++++++++++---------------- tests/components/sql/test_sensor.py | 3 + tests/components/sql/test_services.py | 2 +- 3 files changed, 37 insertions(+), 47 deletions(-) diff --git a/homeassistant/components/sql/util.py b/homeassistant/components/sql/util.py index d70b251012720..59cd979ccce35 100644 --- a/homeassistant/components/sql/util.py +++ b/homeassistant/components/sql/util.py @@ -30,6 +30,7 @@ _LOGGER = logging.getLogger(__name__) +_SQL_SELECT = sqlalchemy.text("SELECT 1;") _SQL_LAMBDA_CACHE: LRUCache = LRUCache(1000) @@ -107,11 +108,8 @@ async def async_create_sessionmaker( # for every sensor. elif db_url in sql_data.session_makers_by_db_url: sessmaker = sql_data.session_makers_by_db_url[db_url] - elif "+aiomysql" in db_url or "+aiosqlite" in db_url or "+asyncpg" in db_url: - if sessmaker := await _async_validate_and_get_session_maker_for_db_url(db_url): - sql_data.session_makers_by_db_url[db_url] = sessmaker - elif sessmaker := await hass.async_add_executor_job( - _validate_and_get_session_maker_for_db_url, db_url + elif sessmaker := await _async_validate_and_get_session_maker_for_db_url( + hass, db_url ): sql_data.session_makers_by_db_url[db_url] = sessmaker else: @@ -210,57 +208,46 @@ async def _shutdown_db_engines(_: Event) -> None: async def _async_validate_and_get_session_maker_for_db_url( - db_url: str, -) -> async_scoped_session[AsyncSession] | None: + hass: HomeAssistant, db_url: str +) -> async_scoped_session[AsyncSession] | scoped_session[Session] | None: """Validate the db_url and return a async session maker.""" try: - maker = async_scoped_session( - async_sessionmaker( - bind=create_async_engine(db_url, future=True), future=True - ), - scopefunc=asyncio.current_task, - ) - # Run a dummy query just to test the db_url - async with maker() as session: - await session.execute(sqlalchemy.text("SELECT 1;")) - - except SQLAlchemyError as err: - _LOGGER.error( - "Couldn't connect using %s DB_URL: %s", - redact_credentials(db_url), - redact_credentials(str(err)), - ) - return None - else: - return maker - - -def _validate_and_get_session_maker_for_db_url( - db_url: str, -) -> scoped_session[Session] | None: - """Validate the db_url and return a session maker. - - This does I/O and should be run in the executor. - """ - try: - maker = scoped_session( - sessionmaker( - bind=sqlalchemy.create_engine(db_url, future=True), future=True + if "+aiomysql" in db_url or "+aiosqlite" in db_url or "+asyncpg" in db_url: + maker = async_scoped_session( + async_sessionmaker( + bind=create_async_engine(db_url, future=True), future=True + ), + scopefunc=asyncio.current_task, ) - ) - # Run a dummy query just to test the db_url - with maker() as session: - session.execute(sqlalchemy.text("SELECT 1;")) + # Run a dummy query just to test the db_url + async with maker() as session: + await session.execute(_SQL_SELECT) + return maker + + def _get_session_maker_for_db_url() -> scoped_session[Session] | None: + """Validate the db_url and return a session maker. + + This does I/O and should be run in the executor. + """ + maker = scoped_session( + sessionmaker( + bind=sqlalchemy.create_engine(db_url, future=True), future=True + ) + ) + # Run a dummy query just to test the db_url + with maker() as session: + session.execute(_SQL_SELECT) + return maker + return await hass.async_add_executor_job(_get_session_maker_for_db_url) except SQLAlchemyError as err: _LOGGER.error( "Couldn't connect using %s DB_URL: %s", redact_credentials(db_url), redact_credentials(str(err)), ) - return None - else: - return maker + + return None def generate_lambda_stmt(query: str) -> StatementLambdaElement: diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index daa9a4eebc419..3fdfd9336cfed 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -325,6 +325,9 @@ def execute(self, query: Any) -> None: """Execute the query.""" raise SQLAlchemyError("sqlite://homeassistant:hunter2@homeassistant.local") + def rollback(self) -> None: + pass + with patch( "homeassistant.components.sql.util.scoped_session", return_value=MockSession, diff --git a/tests/components/sql/test_services.py b/tests/components/sql/test_services.py index 17ea68717077c..8c79b1363ea53 100644 --- a/tests/components/sql/test_services.py +++ b/tests/components/sql/test_services.py @@ -230,7 +230,7 @@ async def test_query_service_invalid_db_url(hass: HomeAssistant) -> None: with ( patch( - "homeassistant.components.sql.util._validate_and_get_session_maker_for_db_url", + "homeassistant.components.sql.util._async_validate_and_get_session_maker_for_db_url", return_value=None, ), pytest.raises( From 865c0e972083fd512d2c3966b18aa84e2bd12c95 Mon Sep 17 00:00:00 2001 From: David Rapan Date: Wed, 29 Oct 2025 15:44:43 +0100 Subject: [PATCH 09/16] Merge test_sensor.test_session_rollback_on_error Signed-off-by: David Rapan --- tests/components/sql/test_sensor.py | 58 +++++++++-------------------- 1 file changed, 17 insertions(+), 41 deletions(-) diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index 3fdfd9336cfed..ad02b304d1464 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -759,50 +759,28 @@ async def test_attributes_from_entry_config( assert CONF_STATE_CLASS not in state.attributes -async def test_session_rollback_on_error( - recorder_mock: Recorder, - hass: HomeAssistant, - freezer: FrozenDateTimeFactory, - caplog: pytest.LogCaptureFixture, -) -> None: - """Test the SQL sensor.""" - options = { - CONF_QUERY: "SELECT 5 as value", - CONF_COLUMN_NAME: "value", - CONF_UNIQUE_ID: "very_unique_id", - } - await init_integration(hass, title="Select value SQL query", options=options) - platforms = async_get_platforms(hass, "sql") - sql_entity = platforms[0].entities["sensor.select_value_sql_query"] - - state = hass.states.get("sensor.select_value_sql_query") - assert state.state == "5" - assert state.attributes["value"] == 5 - - with ( - patch.object( - sql_entity, - "_lambda_stmt", - generate_lambda_stmt("Faulty syntax create operational issue"), +@pytest.mark.parametrize( + ("config", "patch_rollback"), + [ + ( + {}, + "sqlalchemy.orm.session.Session.rollback", ), - patch("sqlalchemy.orm.session.Session.rollback") as mock_session_rollback, - ): - freezer.tick(timedelta(minutes=1)) - async_fire_time_changed(hass) - await hass.async_block_till_done(wait_background_tasks=True) - assert "sqlite3.OperationalError" in caplog.text - - assert mock_session_rollback.call_count == 1 - - -async def test_async_session_rollback_on_error( + ( + {CONF_DB_URL: "sqlite+aiosqlite:///"}, + "sqlalchemy.ext.asyncio.session.AsyncSession.rollback", + ), + ], +) +async def test_session_rollback_on_error( recorder_mock: Recorder, hass: HomeAssistant, freezer: FrozenDateTimeFactory, caplog: pytest.LogCaptureFixture, + config: dict[str, Any], + patch_rollback: str, ) -> None: """Test the SQL sensor.""" - config = {CONF_DB_URL: "sqlite+aiosqlite:///"} options = { CONF_QUERY: "SELECT 5 as value", CONF_COLUMN_NAME: "value", @@ -824,16 +802,14 @@ async def test_async_session_rollback_on_error( "_lambda_stmt", generate_lambda_stmt("Faulty syntax create operational issue"), ), - patch( - "sqlalchemy.ext.asyncio.session.AsyncSession.rollback" - ) as mock_async_session_rollback, + patch(patch_rollback) as mock_rollback, ): freezer.tick(timedelta(minutes=1)) async_fire_time_changed(hass) await hass.async_block_till_done(wait_background_tasks=True) assert "sqlite3.OperationalError" in caplog.text - assert mock_async_session_rollback.call_count == 1 + assert mock_rollback.call_count == 1 async def test_query_recover_from_rollback( From a1732931db33aa7c865638faa9ff68800bcb5a16 Mon Sep 17 00:00:00 2001 From: David Rapan Date: Wed, 29 Oct 2025 15:51:06 +0100 Subject: [PATCH 10/16] Merge test_sensor.test_invalid_url_setup Signed-off-by: David Rapan --- tests/components/sql/test_sensor.py | 57 ++++------------------------- 1 file changed, 8 insertions(+), 49 deletions(-) diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index ad02b304d1464..9bdf536da1ace 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -191,77 +191,39 @@ def make_test_db(): @pytest.mark.parametrize( - ("url", "expected_patterns", "not_expected_patterns"), + ("patch_create", "url", "expected_patterns", "not_expected_patterns"), [ ( + "homeassistant.components.sql.util.sqlalchemy.create_engine", "sqlite://homeassistant:hunter2@homeassistant.local", ["sqlite://****:****@homeassistant.local"], ["sqlite://homeassistant:hunter2@homeassistant.local"], ), ( + "homeassistant.components.sql.util.sqlalchemy.create_engine", "sqlite://homeassistant.local", ["sqlite://homeassistant.local"], [], ), - ], -) -async def test_invalid_url_setup( - recorder_mock: Recorder, - hass: HomeAssistant, - caplog: pytest.LogCaptureFixture, - url: str, - expected_patterns: str, - not_expected_patterns: str, -) -> None: - """Test invalid db url with redacted credentials.""" - config = { - CONF_QUERY: "SELECT 5 as value", - CONF_COLUMN_NAME: "value", - } - entry = MockConfigEntry( - title="count_tables", - domain=DOMAIN, - source=SOURCE_USER, - data={CONF_DB_URL: url}, - options=config, - entry_id="1", - version=2, - ) - - entry.add_to_hass(hass) - - with patch( - "homeassistant.components.sql.util.sqlalchemy.create_engine", - side_effect=SQLAlchemyError(url), - ): - await hass.config_entries.async_setup(entry.entry_id) - await hass.async_block_till_done() - - for pattern in not_expected_patterns: - assert pattern not in caplog.text - for pattern in expected_patterns: - assert pattern in caplog.text - - -@pytest.mark.parametrize( - ("url", "expected_patterns", "not_expected_patterns"), - [ ( + "homeassistant.components.sql.util.create_async_engine", "sqlite+aiosqlite://homeassistant:hunter2@homeassistant.local", ["sqlite+aiosqlite://****:****@homeassistant.local"], ["sqlite+aiosqlite://homeassistant:hunter2@homeassistant.local"], ), ( + "homeassistant.components.sql.util.create_async_engine", "sqlite+aiosqlite://homeassistant.local", ["sqlite+aiosqlite://homeassistant.local"], [], ), ], ) -async def test_async_invalid_url_setup( +async def test_invalid_url_setup( recorder_mock: Recorder, hass: HomeAssistant, caplog: pytest.LogCaptureFixture, + patch_create: str, url: str, expected_patterns: str, not_expected_patterns: str, @@ -283,10 +245,7 @@ async def test_async_invalid_url_setup( entry.add_to_hass(hass) - with patch( - "homeassistant.components.sql.util.create_async_engine", - side_effect=SQLAlchemyError(url), - ): + with patch(patch_create, side_effect=SQLAlchemyError(url)): await hass.config_entries.async_setup(entry.entry_id) await hass.async_block_till_done() From 35c6078403b6fd007dfa2a8f78cde4578dd73efa Mon Sep 17 00:00:00 2001 From: David Rapan Date: Wed, 29 Oct 2025 15:56:26 +0100 Subject: [PATCH 11/16] Merge test_sensor.test_query_from_yaml Signed-off-by: David Rapan --- tests/components/sql/test_sensor.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index 9bdf536da1ace..9b82bb0a1c3db 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -301,23 +301,17 @@ def rollback(self) -> None: assert "sqlite://****:****@homeassistant.local" in caplog.text -async def test_query_from_yaml(recorder_mock: Recorder, hass: HomeAssistant) -> None: - """Test the SQL sensor from yaml config.""" - - assert await async_setup_component(hass, DOMAIN, YAML_CONFIG) - await hass.async_block_till_done() - - state = hass.states.get("sensor.get_value") - assert state.state == "5" - - -async def test_async_query_from_yaml( - recorder_mock: Recorder, hass: HomeAssistant +@pytest.mark.parametrize("async_driver", [False, True]) +async def test_query_from_yaml( + recorder_mock: Recorder, hass: HomeAssistant, async_driver: bool ) -> None: """Test the SQL sensor from yaml config using async driver.""" - config = copy.deepcopy(YAML_CONFIG) - config["sql"][CONF_DB_URL] = "sqlite+aiosqlite://" + config = YAML_CONFIG + + if async_driver: + config = copy.deepcopy(YAML_CONFIG) + config["sql"][CONF_DB_URL] = "sqlite+aiosqlite://" assert await async_setup_component(hass, DOMAIN, config) await hass.async_block_till_done() From 34f80a1ddf6095a2a9a24f55576e3c7a95f44c04 Mon Sep 17 00:00:00 2001 From: David Rapan Date: Wed, 29 Oct 2025 16:03:20 +0100 Subject: [PATCH 12/16] Extend test_sensor.test_invalid_url_setup_from_yaml Signed-off-by: David Rapan --- tests/components/sql/test_sensor.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index 9b82bb0a1c3db..89deb6661603b 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -409,24 +409,39 @@ async def test_config_from_old_yaml( @pytest.mark.parametrize( - ("url", "expected_patterns", "not_expected_patterns"), + ("patch_create", "url", "expected_patterns", "not_expected_patterns"), [ ( + "homeassistant.components.sql.util.sqlalchemy.create_engine", "sqlite://homeassistant:hunter2@homeassistant.local", ["sqlite://****:****@homeassistant.local"], ["sqlite://homeassistant:hunter2@homeassistant.local"], ), ( + "homeassistant.components.sql.util.sqlalchemy.create_engine", "sqlite://homeassistant.local", ["sqlite://homeassistant.local"], [], ), + ( + "homeassistant.components.sql.util.create_async_engine", + "sqlite+aiosqlite://homeassistant:hunter2@homeassistant.local", + ["sqlite+aiosqlite://****:****@homeassistant.local"], + ["sqlite+aiosqlite://homeassistant:hunter2@homeassistant.local"], + ), + ( + "homeassistant.components.sql.util.create_async_engine", + "sqlite+aiosqlite://homeassistant.local", + ["sqlite+aiosqlite://homeassistant.local"], + [], + ), ], ) async def test_invalid_url_setup_from_yaml( recorder_mock: Recorder, hass: HomeAssistant, caplog: pytest.LogCaptureFixture, + patch_create: str, url: str, expected_patterns: str, not_expected_patterns: str, @@ -441,11 +456,9 @@ async def test_invalid_url_setup_from_yaml( } } - with patch( - "homeassistant.components.sql.util.sqlalchemy.create_engine", - side_effect=SQLAlchemyError(url), - ): + with patch(patch_create, side_effect=SQLAlchemyError(url)): assert await async_setup_component(hass, DOMAIN, config) + await hass.async_block_till_done() for pattern in not_expected_patterns: From 9c347b7d0ec780e3781ea7931ca72ac3e099d0d6 Mon Sep 17 00:00:00 2001 From: David Rapan Date: Wed, 29 Oct 2025 19:36:12 +0100 Subject: [PATCH 13/16] hm Signed-off-by: David Rapan --- homeassistant/components/sql/manifest.json | 3 +-- requirements_all.txt | 7 +++---- requirements_test_all.txt | 7 +++---- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/homeassistant/components/sql/manifest.json b/homeassistant/components/sql/manifest.json index fe41ebd3d42c2..9517bd2f3e487 100644 --- a/homeassistant/components/sql/manifest.json +++ b/homeassistant/components/sql/manifest.json @@ -10,8 +10,7 @@ "aiomysql==0.3.2", "aiosqlite==0.21.0", "asyncpg==0.30.0", - "greenlet==3.2.4", - "SQLAlchemy==2.0.41", + "SQLAlchemy[asyncio]==2.0.41", "sqlparse==0.5.0" ] } diff --git a/requirements_all.txt b/requirements_all.txt index 6cefb2aae9ba3..2bd65b88ce314 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -118,9 +118,11 @@ RestrictedPython==8.1 RtmAPI==0.7.2 # homeassistant.components.recorder -# homeassistant.components.sql SQLAlchemy==2.0.41 +# homeassistant.components.sql +SQLAlchemy[asyncio]==2.0.41 + # homeassistant.components.tami4 Tami4EdgeAPI==3.0 @@ -1129,9 +1131,6 @@ greeclimate==2.1.0 # homeassistant.components.greeneye_monitor greeneye_monitor==3.0.3 -# homeassistant.components.sql -greenlet==3.2.4 - # homeassistant.components.greenwave greenwavereality==0.5.1 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 43e2c1712813e..3f010afda634e 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -112,9 +112,11 @@ RestrictedPython==8.1 RtmAPI==0.7.2 # homeassistant.components.recorder -# homeassistant.components.sql SQLAlchemy==2.0.41 +# homeassistant.components.sql +SQLAlchemy[asyncio]==2.0.41 + # homeassistant.components.tami4 Tami4EdgeAPI==3.0 @@ -993,9 +995,6 @@ greeclimate==2.1.0 # homeassistant.components.greeneye_monitor greeneye_monitor==3.0.3 -# homeassistant.components.sql -greenlet==3.2.4 - # homeassistant.components.pure_energie gridnet==5.0.1 From 6c56fb0a7bbfa3380c7e331331db4f293cf9a1ef Mon Sep 17 00:00:00 2001 From: David Rapan Date: Wed, 29 Oct 2025 23:39:26 +0100 Subject: [PATCH 14/16] Reuse conversion to serializable Signed-off-by: David Rapan --- homeassistant/components/sql/sensor.py | 23 ++++++++--------------- homeassistant/components/sql/services.py | 12 ++---------- homeassistant/components/sql/util.py | 18 +++++++++++++++++- tests/components/sql/test_sensor.py | 2 +- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/homeassistant/components/sql/sensor.py b/homeassistant/components/sql/sensor.py index 9ea11069d7c0e..42d311c985c76 100644 --- a/homeassistant/components/sql/sensor.py +++ b/homeassistant/components/sql/sensor.py @@ -2,8 +2,6 @@ from __future__ import annotations -from datetime import date -import decimal import logging from typing import TYPE_CHECKING, Any @@ -44,6 +42,7 @@ from .const import CONF_ADVANCED_OPTIONS, CONF_COLUMN_NAME, CONF_QUERY, DOMAIN from .util import ( async_create_sessionmaker, + ensure_serializable, generate_lambda_stmt, redact_credentials, resolve_db_url, @@ -247,19 +246,13 @@ def extra_state_attributes(self) -> dict[str, Any] | None: def _process(self, result: Result) -> None: """Process the SQL result.""" data = None - extra_state_attributes = {} - for res in result.mappings(): - _LOGGER.debug("Query %s result in %s", self._query, res.items()) - data = res[self._column_name] - for key, value in res.items(): - if isinstance(value, decimal.Decimal): - value = float(value) - elif isinstance(value, date): - value = value.isoformat() - elif isinstance(value, (bytes, bytearray)): - value = f"0x{value.hex()}" - extra_state_attributes[key] = value - self._attr_extra_state_attributes[key] = value + + for row in result.mappings(): + row_items = row.items() + _LOGGER.debug("Query %s result in %s", self._query, row_items) + data = row[self._column_name] + for key, value in row_items: + self._attr_extra_state_attributes[key] = ensure_serializable(value) if data is not None and isinstance(data, (bytes, bytearray)): data = f"0x{data.hex()}" diff --git a/homeassistant/components/sql/services.py b/homeassistant/components/sql/services.py index 9b6e9e3686550..9fd6dd8f66948 100644 --- a/homeassistant/components/sql/services.py +++ b/homeassistant/components/sql/services.py @@ -2,8 +2,6 @@ from __future__ import annotations -import datetime -import decimal import logging from typing import TYPE_CHECKING @@ -28,6 +26,7 @@ from .const import CONF_QUERY, DOMAIN from .util import ( async_create_sessionmaker, + ensure_serializable, generate_lambda_stmt, redact_credentials, resolve_db_url, @@ -77,14 +76,7 @@ def _process(result: Result) -> list[JsonValueType]: for row in result.mappings(): processed_row: dict[str, JsonValueType] = {} for key, value in row.items(): - if isinstance(value, decimal.Decimal): - processed_row[key] = float(value) - elif isinstance(value, datetime.date): - processed_row[key] = value.isoformat() - elif isinstance(value, (bytes, bytearray)): - processed_row[key] = f"0x{value.hex()}" - else: - processed_row[key] = value + processed_row[key] = ensure_serializable(value) rows.append(processed_row) return rows diff --git a/homeassistant/components/sql/util.py b/homeassistant/components/sql/util.py index 59cd979ccce35..d6618922e070c 100644 --- a/homeassistant/components/sql/util.py +++ b/homeassistant/components/sql/util.py @@ -3,7 +3,10 @@ from __future__ import annotations import asyncio +from datetime import date +from decimal import Decimal import logging +from typing import Any import sqlalchemy from sqlalchemy import lambda_stmt @@ -210,7 +213,7 @@ async def _shutdown_db_engines(_: Event) -> None: async def _async_validate_and_get_session_maker_for_db_url( hass: HomeAssistant, db_url: str ) -> async_scoped_session[AsyncSession] | scoped_session[Session] | None: - """Validate the db_url and return a async session maker.""" + """Validate the db_url and return a session maker.""" try: if "+aiomysql" in db_url or "+aiosqlite" in db_url or "+asyncpg" in db_url: maker = async_scoped_session( @@ -254,3 +257,16 @@ def generate_lambda_stmt(query: str) -> StatementLambdaElement: """Generate the lambda statement.""" text = sqlalchemy.text(query) return lambda_stmt(lambda: text, lambda_cache=_SQL_LAMBDA_CACHE) + + +def ensure_serializable(value: Any) -> Any: + """Ensure value is serializable.""" + match value: + case Decimal(): + return float(value) + case date(): + return value.isoformat() + case bytes() | bytearray(): + return f"0x{value.hex()}" + case _: + return value diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index 89deb6661603b..7292ac4d4a8cb 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -305,7 +305,7 @@ def rollback(self) -> None: async def test_query_from_yaml( recorder_mock: Recorder, hass: HomeAssistant, async_driver: bool ) -> None: - """Test the SQL sensor from yaml config using async driver.""" + """Test the SQL sensor from yaml config.""" config = YAML_CONFIG From a6c3fb69d4b1b5df4407080b7c244e2c7df20e14 Mon Sep 17 00:00:00 2001 From: David Rapan Date: Thu, 30 Oct 2025 13:52:05 +0100 Subject: [PATCH 15/16] Add test_util.test_data_conversion Signed-off-by: David Rapan --- tests/components/sql/test_util.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/components/sql/test_util.py b/tests/components/sql/test_util.py index 737a5e4a41baa..f63626df6923b 100644 --- a/tests/components/sql/test_util.py +++ b/tests/components/sql/test_util.py @@ -1,10 +1,17 @@ """Test the sql utils.""" +from datetime import date +from decimal import Decimal + import pytest import voluptuous as vol from homeassistant.components.recorder import Recorder, get_instance -from homeassistant.components.sql.util import resolve_db_url, validate_sql_select +from homeassistant.components.sql.util import ( + ensure_serializable, + resolve_db_url, + validate_sql_select, +) from homeassistant.core import HomeAssistant @@ -64,3 +71,22 @@ async def test_invalid_sql_queries( """Test that various invalid or disallowed SQL queries raise the correct exception.""" with pytest.raises(vol.Invalid, match=expected_error_message): validate_sql_select(sql_query) + + +@pytest.mark.parametrize( + ("input", "expected_output"), + [ + (Decimal("199.99"), 199.99), + (date(2023, 1, 15), "2023-01-15"), + (b"\xde\xad\xbe\xef", "0xdeadbeef"), + ("deadbeef", "deadbeef"), + (199.99, 199.99), + (69, 69), + ], +) +async def test_data_conversion( + input: Decimal | date | bytes | str | float, + expected_output: str | float, +) -> None: + """Test data conversion to serializable type.""" + assert ensure_serializable(input) == expected_output From 1ec8b6d4362a3dc3c7622bb49c587c95026c0499 Mon Sep 17 00:00:00 2001 From: David Rapan Date: Thu, 30 Oct 2025 19:30:06 +0100 Subject: [PATCH 16/16] Extend test_services.test_query_service_rollback_on_error Signed-off-by: David Rapan --- homeassistant/components/sql/util.py | 6 ++- tests/components/sql/test_sensor.py | 60 +++++++++++++-------------- tests/components/sql/test_services.py | 26 ++++++++++-- 3 files changed, 56 insertions(+), 36 deletions(-) diff --git a/homeassistant/components/sql/util.py b/homeassistant/components/sql/util.py index d6618922e070c..f99c7aae4a83b 100644 --- a/homeassistant/components/sql/util.py +++ b/homeassistant/components/sql/util.py @@ -197,9 +197,11 @@ async def _shutdown_db_engines(_: Event) -> None: """Shutdown all database engines.""" for sessmaker in session_makers_by_db_url.values(): if isinstance(sessmaker, async_scoped_session): + _LOGGER.error("Disposed async engine for shutdown 1") await (await sessmaker.connection()).engine.dispose() - else: - sessmaker.connection().engine.dispose() + _LOGGER.error("Disposed async engine for shutdown 2") + raise SQLAlchemyError("Disposed async engine for shutdown") + sessmaker.connection().engine.dispose() cancel_shutdown = hass.bus.async_listen_once( EVENT_HOMEASSISTANT_STOP, _shutdown_db_engines diff --git a/tests/components/sql/test_sensor.py b/tests/components/sql/test_sensor.py index 7292ac4d4a8cb..d66ac28f6b12f 100644 --- a/tests/components/sql/test_sensor.py +++ b/tests/components/sql/test_sensor.py @@ -193,18 +193,6 @@ def make_test_db(): @pytest.mark.parametrize( ("patch_create", "url", "expected_patterns", "not_expected_patterns"), [ - ( - "homeassistant.components.sql.util.sqlalchemy.create_engine", - "sqlite://homeassistant:hunter2@homeassistant.local", - ["sqlite://****:****@homeassistant.local"], - ["sqlite://homeassistant:hunter2@homeassistant.local"], - ), - ( - "homeassistant.components.sql.util.sqlalchemy.create_engine", - "sqlite://homeassistant.local", - ["sqlite://homeassistant.local"], - [], - ), ( "homeassistant.components.sql.util.create_async_engine", "sqlite+aiosqlite://homeassistant:hunter2@homeassistant.local", @@ -217,6 +205,18 @@ def make_test_db(): ["sqlite+aiosqlite://homeassistant.local"], [], ), + ( + "homeassistant.components.sql.util.sqlalchemy.create_engine", + "sqlite://homeassistant:hunter2@homeassistant.local", + ["sqlite://****:****@homeassistant.local"], + ["sqlite://homeassistant:hunter2@homeassistant.local"], + ), + ( + "homeassistant.components.sql.util.sqlalchemy.create_engine", + "sqlite://homeassistant.local", + ["sqlite://homeassistant.local"], + [], + ), ], ) async def test_invalid_url_setup( @@ -301,7 +301,7 @@ def rollback(self) -> None: assert "sqlite://****:****@homeassistant.local" in caplog.text -@pytest.mark.parametrize("async_driver", [False, True]) +@pytest.mark.parametrize("async_driver", [True, False]) async def test_query_from_yaml( recorder_mock: Recorder, hass: HomeAssistant, async_driver: bool ) -> None: @@ -411,18 +411,6 @@ async def test_config_from_old_yaml( @pytest.mark.parametrize( ("patch_create", "url", "expected_patterns", "not_expected_patterns"), [ - ( - "homeassistant.components.sql.util.sqlalchemy.create_engine", - "sqlite://homeassistant:hunter2@homeassistant.local", - ["sqlite://****:****@homeassistant.local"], - ["sqlite://homeassistant:hunter2@homeassistant.local"], - ), - ( - "homeassistant.components.sql.util.sqlalchemy.create_engine", - "sqlite://homeassistant.local", - ["sqlite://homeassistant.local"], - [], - ), ( "homeassistant.components.sql.util.create_async_engine", "sqlite+aiosqlite://homeassistant:hunter2@homeassistant.local", @@ -435,6 +423,18 @@ async def test_config_from_old_yaml( ["sqlite+aiosqlite://homeassistant.local"], [], ), + ( + "homeassistant.components.sql.util.sqlalchemy.create_engine", + "sqlite://homeassistant:hunter2@homeassistant.local", + ["sqlite://****:****@homeassistant.local"], + ["sqlite://homeassistant:hunter2@homeassistant.local"], + ), + ( + "homeassistant.components.sql.util.sqlalchemy.create_engine", + "sqlite://homeassistant.local", + ["sqlite://homeassistant.local"], + [], + ), ], ) async def test_invalid_url_setup_from_yaml( @@ -728,17 +728,17 @@ async def test_attributes_from_entry_config( @pytest.mark.parametrize( ("config", "patch_rollback"), [ - ( - {}, - "sqlalchemy.orm.session.Session.rollback", - ), ( {CONF_DB_URL: "sqlite+aiosqlite:///"}, "sqlalchemy.ext.asyncio.session.AsyncSession.rollback", ), + ( + {}, + "sqlalchemy.orm.session.Session.rollback", + ), ], ) -async def test_session_rollback_on_error( +async def test_query_rollback_on_error( recorder_mock: Recorder, hass: HomeAssistant, freezer: FrozenDateTimeFactory, diff --git a/tests/components/sql/test_services.py b/tests/components/sql/test_services.py index 8c79b1363ea53..39c01259f75ba 100644 --- a/tests/components/sql/test_services.py +++ b/tests/components/sql/test_services.py @@ -7,7 +7,7 @@ from unittest.mock import patch import pytest -from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy import text import voluptuous as vol from voluptuous import MultipleInvalid @@ -87,14 +87,29 @@ async def test_query_service_external_db(hass: HomeAssistant, tmp_path: Path) -> } +@pytest.mark.parametrize( + ("async_driver", "patch_rollback"), + [ + ( + True, + "sqlalchemy.ext.asyncio.session.AsyncSession.rollback", + ), + ( + False, + "sqlalchemy.orm.session.Session.rollback", + ), + ], +) async def test_query_service_rollback_on_error( hass: HomeAssistant, tmp_path: Path, caplog: pytest.LogCaptureFixture, + async_driver: bool, + patch_rollback: str, ) -> None: """Test the query service.""" db_path = tmp_path / "test.db" - db_url = f"sqlite:///{db_path}" + db_url = f"sqlite{'+aiosqlite' if async_driver else ''}:///{db_path}" # Create and populate the external database conn = sqlite3.connect(db_path) @@ -109,12 +124,12 @@ async def test_query_service_rollback_on_error( with ( patch( "homeassistant.components.sql.services.generate_lambda_stmt", - side_effect=SQLAlchemyError("Error executing query"), + return_value=text("Faulty syntax create operational issue"), ), pytest.raises( ServiceValidationError, match="An error occurred when executing the query" ), - patch("sqlalchemy.orm.session.Session.rollback") as mock_session_rollback, + patch(patch_rollback) as mock_session_rollback, ): await hass.services.async_call( DOMAIN, @@ -124,8 +139,11 @@ async def test_query_service_rollback_on_error( return_response=True, ) + assert "sqlite3.OperationalError" in caplog.text assert mock_session_rollback.call_count == 1 + await hass.async_stop() + async def test_query_service_data_conversion( hass: HomeAssistant, tmp_path: Path