From 3afae533722026d878433478f764b6f93e20e490 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 11 Nov 2025 12:31:21 +0100 Subject: [PATCH 01/12] Change marking for tests added for new features which are not yet implemented in sqlalchemy-exasol --- test/integration/sqlalchemy/test_suite.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/integration/sqlalchemy/test_suite.py b/test/integration/sqlalchemy/test_suite.py index 5df82539..9cd8b823 100644 --- a/test/integration/sqlalchemy/test_suite.py +++ b/test/integration/sqlalchemy/test_suite.py @@ -47,6 +47,10 @@ class XfailRationale(str, Enum): Manual indexes are not recommended within the Exasol DB. """ ) + NEW_FEATURE_2x = cleandoc( + """In migrating to 2.x, new features were introduced by SQLAlchemy that + have not yet been implemented in sqlalchemy-exasol.""" + ) QUOTING = cleandoc( """This suite was added to SQLAlchemy 1.3.19 on July 2020 to address issues in other dialects related to object names that contain quotes @@ -338,16 +342,19 @@ class sqlalchemy.testing.suite.ComponentReflectionTest def test_get_indexes(self, connection, use_schema): super().test_get_indexes() - @pytest.mark.xfail(reason=BREAKING_CHANGES_SQL_ALCHEMY_2x, strict=True) + @pytest.mark.xfail(reason=XfailRationale.NEW_FEATURE_2x.value, strict=True) def test_get_multi_columns(self): + # See https://docs.sqlalchemy.org/en/20/core/reflection.html#sqlalchemy.engine.reflection.Inspector.get_multi_columns super().test_get_multi_columns() - @pytest.mark.xfail(reason=BREAKING_CHANGES_SQL_ALCHEMY_2x, strict=True) + @pytest.mark.xfail(reason=XfailRationale.NEW_FEATURE_2x.value, strict=True) def test_get_multi_foreign_keys(self): + # See https://docs.sqlalchemy.org/en/20/core/reflection.html#sqlalchemy.engine.reflection.Inspector.get_multi_foreign_keys super().test_get_multi_foreign_keys() - @pytest.mark.xfail(reason=BREAKING_CHANGES_SQL_ALCHEMY_2x, strict=True) + @pytest.mark.xfail(reason=XfailRationale.NEW_FEATURE_2x.value, strict=True) def test_get_multi_pk_constraint(self): + # See https://docs.sqlalchemy.org/en/20/core/reflection.html#sqlalchemy.engine.reflection.Inspector.get_multi_pk_constraint super().test_get_multi_pk_constraint() @pytest.mark.xfail(reason=BREAKING_CHANGES_SQL_ALCHEMY_2x, strict=True) From c7ade00d5a738a7b643be963105a77609145b34a Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 11 Nov 2025 12:50:52 +0100 Subject: [PATCH 02/12] Upon checking what needed to be done to resolve these, it was found that test_get_multi_pk_constraint fully succeeds --- test/integration/sqlalchemy/test_suite.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/integration/sqlalchemy/test_suite.py b/test/integration/sqlalchemy/test_suite.py index 9cd8b823..e899722e 100644 --- a/test/integration/sqlalchemy/test_suite.py +++ b/test/integration/sqlalchemy/test_suite.py @@ -352,11 +352,6 @@ def test_get_multi_foreign_keys(self): # See https://docs.sqlalchemy.org/en/20/core/reflection.html#sqlalchemy.engine.reflection.Inspector.get_multi_foreign_keys super().test_get_multi_foreign_keys() - @pytest.mark.xfail(reason=XfailRationale.NEW_FEATURE_2x.value, strict=True) - def test_get_multi_pk_constraint(self): - # See https://docs.sqlalchemy.org/en/20/core/reflection.html#sqlalchemy.engine.reflection.Inspector.get_multi_pk_constraint - super().test_get_multi_pk_constraint() - @pytest.mark.xfail(reason=BREAKING_CHANGES_SQL_ALCHEMY_2x, strict=True) def test_get_view_definition_does_not_exist(self): super().test_get_view_definition_does_not_exist() From ad5c6b44073af55fa4509b8c74d72a8e9ee9bf04 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 11 Nov 2025 13:29:30 +0100 Subject: [PATCH 03/12] Resolve test_get_view_definition_does_not_exist by raising error when view does not exist --- sqlalchemy_exasol/base.py | 17 ++++++++++++----- test/integration/sqlalchemy/test_suite.py | 4 ---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/sqlalchemy_exasol/base.py b/sqlalchemy_exasol/base.py index 62ac1963..78178a2d 100644 --- a/sqlalchemy_exasol/base.py +++ b/sqlalchemy_exasol/base.py @@ -934,7 +934,10 @@ def get_table_names(self, connection, schema, **kw): tables = [self.normalize_name(row[0]) for row in result] return tables - def has_table(self, connection, table_name, schema=None, **kw): + @reflection.cache + def has_table(self, connection, table_name, schema=None, **kw) -> bool: + self._ensure_has_table_connection(connection) + schema = self._get_schema_for_input(connection, schema) sql_statement = ( "SELECT OBJECT_NAME FROM SYS.EXA_ALL_OBJECTS " @@ -970,9 +973,9 @@ def get_view_names(self, connection, schema=None, **kw): @reflection.cache def get_view_definition(self, connection, view_name, schema=None, **kw): - schema = self._get_schema_for_input(connection, schema) + schema_name = self._get_schema_for_input(connection, schema) sql_stmnt = "SELECT view_text FROM sys.exa_all_views WHERE view_name = :view_name AND view_schema = " - if schema is None: + if schema_name is None: sql_stmnt += "CURRENT_SCHEMA" else: sql_stmnt += ":schema" @@ -980,10 +983,14 @@ def get_view_definition(self, connection, view_name, schema=None, **kw): sql.text(sql_stmnt), { "view_name": self.denormalize_name(view_name), - "schema": self.denormalize_name(schema), + "schema": self.denormalize_name(schema_name), }, ).scalar() - return result if result else None + if result: + return result + raise sqlalchemy.exc.NoSuchTableError( + f"{schema_name}.{view_name}" if schema_name else view_name + ) @staticmethod def quote_string_value(string_value): diff --git a/test/integration/sqlalchemy/test_suite.py b/test/integration/sqlalchemy/test_suite.py index e899722e..0aebc74b 100644 --- a/test/integration/sqlalchemy/test_suite.py +++ b/test/integration/sqlalchemy/test_suite.py @@ -352,10 +352,6 @@ def test_get_multi_foreign_keys(self): # See https://docs.sqlalchemy.org/en/20/core/reflection.html#sqlalchemy.engine.reflection.Inspector.get_multi_foreign_keys super().test_get_multi_foreign_keys() - @pytest.mark.xfail(reason=BREAKING_CHANGES_SQL_ALCHEMY_2x, strict=True) - def test_get_view_definition_does_not_exist(self): - super().test_get_view_definition_does_not_exist() - class HasIndexTest(_HasIndexTest): @pytest.mark.xfail(reason=XfailRationale.MANUAL_INDEX.value, strict=True) From e1360a0b3f892bf7e8095ab46d2854709b933443 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 12 Nov 2025 13:07:23 +0100 Subject: [PATCH 04/12] Refactor names in base.py to be clearer --- sqlalchemy_exasol/base.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sqlalchemy_exasol/base.py b/sqlalchemy_exasol/base.py index 78178a2d..4a238162 100644 --- a/sqlalchemy_exasol/base.py +++ b/sqlalchemy_exasol/base.py @@ -959,28 +959,28 @@ def has_table(self, connection, table_name, schema=None, **kw) -> bool: @reflection.cache def get_view_names(self, connection, schema=None, **kw): - schema = self._get_schema_for_input(connection, schema) + schema_name = self._get_schema_for_input(connection, schema) sql_statement = "SELECT view_name FROM SYS.EXA_ALL_VIEWS WHERE view_schema = " - if schema is None: + if schema_name is None: sql_statement += "CURRENT_SCHEMA ORDER BY view_name" result = connection.execute(sql.text(sql_statement)) else: sql_statement += ":schema ORDER BY view_name" result = connection.execute( - sql.text(sql_statement), {"schema": self.denormalize_name(schema)} + sql.text(sql_statement), {"schema": self.denormalize_name(schema_name)} ) return [self.normalize_name(row[0]) for row in result] @reflection.cache def get_view_definition(self, connection, view_name, schema=None, **kw): schema_name = self._get_schema_for_input(connection, schema) - sql_stmnt = "SELECT view_text FROM sys.exa_all_views WHERE view_name = :view_name AND view_schema = " + sql_statement = "SELECT view_text FROM sys.exa_all_views WHERE view_name = :view_name AND view_schema = " if schema_name is None: - sql_stmnt += "CURRENT_SCHEMA" + sql_statement += "CURRENT_SCHEMA" else: - sql_stmnt += ":schema" + sql_statement += ":schema" result = connection.execute( - sql.text(sql_stmnt), + sql.text(sql_statement), { "view_name": self.denormalize_name(view_name), "schema": self.denormalize_name(schema_name), From d43f95bc93fa817e3fabb6f0b7c3a2de4ce44067 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 12 Nov 2025 13:09:23 +0100 Subject: [PATCH 05/12] Fix test_get_multi_columns as resolvable due to differing nullable expectation only --- test/integration/sqlalchemy/test_suite.py | 49 +++++++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/test/integration/sqlalchemy/test_suite.py b/test/integration/sqlalchemy/test_suite.py index 0aebc74b..ece7701d 100644 --- a/test/integration/sqlalchemy/test_suite.py +++ b/test/integration/sqlalchemy/test_suite.py @@ -5,6 +5,7 @@ import pytest import sqlalchemy as sa from pyexasol import ExaQueryError +from sqlalchemy import Inspector from sqlalchemy.schema import ( DDL, Index, @@ -23,6 +24,7 @@ from sqlalchemy.testing.suite import ReturningGuardsTest as _ReturningGuardsTest from sqlalchemy.testing.suite import RowCountTest as _RowCountTest from sqlalchemy.testing.suite import RowFetchTest as _RowFetchTest +from sqlalchemy.testing.suite.test_reflection import _multi_combination """ Here, all tests are imported from the testing suite of sqlalchemy to ensure that the @@ -338,14 +340,53 @@ class sqlalchemy.testing.suite.ComponentReflectionTest if not schema and testing.requires.temp_table_reflection.enabled: cls.define_temp_tables(metadata) + @staticmethod + def _convert_view_nullable(expected_multi_output): + """ + Convert expected nullable to None + + For view, nullable is always NULL, so the expected result needs + to be modified. For more reference, see: + https://docs.exasol.com/saas/sql_references/system_tables/metadata/exa_all_columns.htm + """ + for key, value_list in expected_multi_output.items(): + schema, table_or_view = key + if not table_or_view.endswith("_v"): + continue + for column_def in value_list: + # Replace nullable: with nullable: None + column_def["nullable"] = None + @pytest.mark.xfail(reason=XfailRationale.MANUAL_INDEX.value) def test_get_indexes(self, connection, use_schema): super().test_get_indexes() - @pytest.mark.xfail(reason=XfailRationale.NEW_FEATURE_2x.value, strict=True) - def test_get_multi_columns(self): - # See https://docs.sqlalchemy.org/en/20/core/reflection.html#sqlalchemy.engine.reflection.Inspector.get_multi_columns - super().test_get_multi_columns() + @_multi_combination + def test_get_multi_columns(self, get_multi_exp, schema, scope, kind, use_filter): + """ + The default implementation of test_get_multi_columns in + class sqlalchemy.testing.suite.ComponentReflectionTest + needs to be overridden here as Exasol always requires that nullable be NULL + for the columns of views. The code given in this overriding class + method was directly copied. See notes, marked with 'Added', highlighting + the changed place. + """ + insp, kws, exp = get_multi_exp( + schema, + scope, + kind, + use_filter, + Inspector.get_columns, + self.exp_columns, + ) + + # Added to convert nullable for columns in views + self._convert_view_nullable(exp) + + for kw in kws: + insp.clear_cache() + result = insp.get_multi_columns(**kw) + self._check_table_dict(result, exp, self._required_column_keys) @pytest.mark.xfail(reason=XfailRationale.NEW_FEATURE_2x.value, strict=True) def test_get_multi_foreign_keys(self): From fc85011fc2316515f92cfc98ba23065dc7e17dbf Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 12 Nov 2025 13:54:42 +0100 Subject: [PATCH 06/12] Fix test_get_multi_foreign_keys as resolvable with new sorting, as custom constraints cannot be used in Exasol --- test/integration/sqlalchemy/test_suite.py | 54 +++++++++++++++++++---- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/test/integration/sqlalchemy/test_suite.py b/test/integration/sqlalchemy/test_suite.py index ece7701d..b9fc7317 100644 --- a/test/integration/sqlalchemy/test_suite.py +++ b/test/integration/sqlalchemy/test_suite.py @@ -49,10 +49,6 @@ class XfailRationale(str, Enum): Manual indexes are not recommended within the Exasol DB. """ ) - NEW_FEATURE_2x = cleandoc( - """In migrating to 2.x, new features were introduced by SQLAlchemy that - have not yet been implemented in sqlalchemy-exasol.""" - ) QUOTING = cleandoc( """This suite was added to SQLAlchemy 1.3.19 on July 2020 to address issues in other dialects related to object names that contain quotes @@ -388,10 +384,52 @@ class sqlalchemy.testing.suite.ComponentReflectionTest result = insp.get_multi_columns(**kw) self._check_table_dict(result, exp, self._required_column_keys) - @pytest.mark.xfail(reason=XfailRationale.NEW_FEATURE_2x.value, strict=True) - def test_get_multi_foreign_keys(self): - # See https://docs.sqlalchemy.org/en/20/core/reflection.html#sqlalchemy.engine.reflection.Inspector.get_multi_foreign_keys - super().test_get_multi_foreign_keys() + @_multi_combination + def test_get_multi_foreign_keys( + self, get_multi_exp, schema, scope, kind, use_filter + ): + """ + The default implementation of test_get_multi_foreign_keys in + class sqlalchemy.testing.suite.ComponentReflectionTest + needs to be overridden here as Exasol does not support custom constraints. + The code given in this overriding class method was directly copied. See notes, + marked with 'Replaced', highlighting the changed place. + """ + + def sort_entries_in_place(result_set): + for key, value_list in result_set.items(): + result_set[key] = sorted(value_list, key=lambda x: x["referred_table"]) + + insp, kws, exp = get_multi_exp( + schema, + scope, + kind, + use_filter, + Inspector.get_foreign_keys, + self.exp_fks, + ) + + for kw in kws: + insp.clear_cache() + result = insp.get_multi_foreign_keys(**kw) + + # Replaced as self._adjust_sort did not work, as some constraints + # cannot be added into an Exasol DB as described in define_reflected_tables + # self._adjust_sort( + # result, exp, lambda d: tuple(d["referred_table"]) + # ) + sort_entries_in_place(exp) + sort_entries_in_place(result) + + self._check_table_dict( + result, + exp, + { + "name", + "constrained_columns", + "referred_schema", + }, + ) class HasIndexTest(_HasIndexTest): From 987f44c0567cb37afcbd35a0cdeb178a05a09e00 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 12 Nov 2025 13:59:08 +0100 Subject: [PATCH 07/12] Add changelog entry --- doc/changes/unreleased.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 3c60891a..69d612a4 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -20,3 +20,8 @@ strings should be altered to start with `exa+websocket://`. - `ComponentReflectionTest.test_not_existing_table` is used to indicate that specific `EXADialect` methods (i.e. `get_columns`) check to see if the requested table/view exists and if not, they will now toss a `NoSuchTableError` exception - #403: Dropped support for Turbodbc - #404: Dropped support for pyodbc +- #654: Reinstated `sqlalchemy` tests after minor modifications to work for Exasol: + - `ComponentReflectionTest.test_get_multi_columns` + - `ComponentReflectionTest.test_get_multi_foreign_keys` + - `ComponentReflectionTest.test_get_multi_pk_constraint` + - `ComponentReflectionTest.test_get_view_definition_does_not_exist` From 6fc3323b817e223bd5fbff1468ac9d3d09231ad5 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 12 Nov 2025 14:13:13 +0100 Subject: [PATCH 08/12] Remove unused import --- test/integration/exasol/test_deadlock.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/exasol/test_deadlock.py b/test/integration/exasol/test_deadlock.py index b340902a..674feaf0 100644 --- a/test/integration/exasol/test_deadlock.py +++ b/test/integration/exasol/test_deadlock.py @@ -2,7 +2,6 @@ from threading import Thread import pytest -import sqlalchemy.testing as testing from sqlalchemy import ( create_engine, inspect, From 9f0040eebc8eea12c839a8903b18ec7a3c6a489e Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 12 Nov 2025 14:50:50 +0100 Subject: [PATCH 09/12] Fix test as now throws exception if view does not exist --- .../exasol/test_get_metadata_functions.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/integration/exasol/test_get_metadata_functions.py b/test/integration/exasol/test_get_metadata_functions.py index fa222c73..ba394a1b 100644 --- a/test/integration/exasol/test_get_metadata_functions.py +++ b/test/integration/exasol/test_get_metadata_functions.py @@ -200,12 +200,12 @@ def test_get_view_definition(self, engine_name): def test_get_view_definition_view_name_none(self, engine_name): with self.engine_map[engine_name].begin() as c: dialect = inspect(c).dialect - view_definition = dialect.get_view_definition( - connection=c, - schema=self.schema, - view_name=None, - ) - assert view_definition is None + with pytest.raises(NoSuchTableError): + dialect.get_view_definition( + connection=c, + schema=self.schema, + view_name=None, + ) @pytest.mark.parametrize( "engine_name", From 8e26a04063959247bba7e8b294270d3d291f3ec0 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Mon, 17 Nov 2025 10:22:00 +0100 Subject: [PATCH 10/12] Switch skipif to xfail --- test/integration/exasol/test_certificate.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/test/integration/exasol/test_certificate.py b/test/integration/exasol/test_certificate.py index 9879c909..9159e181 100644 --- a/test/integration/exasol/test_certificate.py +++ b/test/integration/exasol/test_certificate.py @@ -12,10 +12,6 @@ from sqlalchemy.testing import config from sqlalchemy.testing.fixtures import TestBase -FINGERPRINT_SECURITY_RATIONALE = ( - "Only websocket supports giving a fingerprint in the connection" -) - def get_fingerprint(dsn): import websocket @@ -60,9 +56,10 @@ def remove_ssl_settings(url): pass return url.set(query=query) - @pytest.mark.skipif( + @pytest.mark.xfail( testing.db.dialect.server_version_info < (7, 1, 0), reason="DB version(s) before 7.1.0 don't enforce ssl/tls", + strict=True, ) def test_db_connection_fails_with_default_settings_for_self_signed_certificates( self, @@ -79,10 +76,6 @@ def test_db_connection_fails_with_default_settings_for_self_signed_certificates( expected_substrings = ["self-signed certificate", "self signed certificate"] assert any([e in actual_message for e in expected_substrings]) - @pytest.mark.skipif( - "websocket" not in testing.db.dialect.driver, - reason="Only websocket supports passing on connect_args like this.", - ) def test_db_skip_certification_validation_passes(self): url = self.remove_ssl_settings(config.db.url) @@ -99,10 +92,6 @@ def test_db_with_ssl_verify_none_passes(self): result = self.perform_test_query(engine) assert result == [(42,)] - @pytest.mark.skipif( - "websocket" not in testing.db.dialect.driver, - reason=FINGERPRINT_SECURITY_RATIONALE, - ) def test_db_with_fingerprint_passes(self): url = self.remove_ssl_settings(config.db.url) connect_args = url.translate_connect_args(database="schema") @@ -116,10 +105,6 @@ def test_db_with_fingerprint_passes(self): result = self.perform_test_query(engine) assert result == [(42,)] - @pytest.mark.skipif( - "websocket" not in testing.db.dialect.driver, - reason=FINGERPRINT_SECURITY_RATIONALE, - ) def test_db_with_wrong_fingerprint_fails(self): url = self.remove_ssl_settings(config.db.url) connect_args = url.translate_connect_args(database="schema") From 9618ac836b7c2e5ad74c06a9cff732e589801aea Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Mon, 17 Nov 2025 10:23:46 +0100 Subject: [PATCH 11/12] Add missing strict=True for xfail --- test/integration/sqlalchemy/test_suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/sqlalchemy/test_suite.py b/test/integration/sqlalchemy/test_suite.py index b9fc7317..766b36a0 100644 --- a/test/integration/sqlalchemy/test_suite.py +++ b/test/integration/sqlalchemy/test_suite.py @@ -353,7 +353,7 @@ def _convert_view_nullable(expected_multi_output): # Replace nullable: with nullable: None column_def["nullable"] = None - @pytest.mark.xfail(reason=XfailRationale.MANUAL_INDEX.value) + @pytest.mark.xfail(reason=XfailRationale.MANUAL_INDEX.value, strict=True) def test_get_indexes(self, connection, use_schema): super().test_get_indexes() From 9f0da2e593b2b357fa8b7b897047050ebc21d82c Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Mon, 17 Nov 2025 10:35:49 +0100 Subject: [PATCH 12/12] Make XFAILS strict by default --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 80659a52..589c25ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ filterwarnings = [ markers = [ "backend: backend tests.", ] +xfail_strict = true [tool.black] line-length = 88