Resolve params to dict for Connection.execute() & deprecation of bind and autoload parameters (#615)

* Resolve params to dict for Connection.execute()

* Alter MetaData usage to instead bind to the right object or function instead, as it no longer supports bind

* Ensure autoload parameters have been replaced with autoload_with by setting warning to toss an error

* Update errors and changelog

* Resolve autocommit with DML to use begin for explicit transaction

Author: ArBridgeman

doc/changes/unreleased.md

@@ -9,7 +9,10 @@ This allows us to use the latest dependencies, which do not have open vulnerabil
 
 ## Refactoring
 
-- #610: Altered string input into `Connection.execute()` to be handled properly
+- #610: Altered string input into `Connection.execute()` to be handled properly with `sql.text()`
+- #614: Altered params input into `Connection.execute()` to be handled properly with `dict`
+- #616: Altered usage of MetaData which was binding to a connection to instead bind in the needed object or function
+- #617: Enacted warning for the deprecation of the `autoload` parameter and requirement of `bind`
 
 ## Internal
 

pyproject.toml

@@ -94,6 +94,10 @@ addopts = "--tb native -v -r fxX"
 filterwarnings = [
 # before end of #413, switch this from a text specific issue to a general error on a warning to ensure known deprecation issues have been resolved
 "error:.*Passing a string.*",
+"error:.* a single dictionary.*",
+"error:.*The MetaData.bind argument.*",
+"error:.*The autoload parameter.*",
+"error:.*schema methods that invoke SQL against.*",
 # this is used for turbodbc and pyodbc as historically we'd like to remove them.
 # thus far, it seems like this is not a strict requirement for the migration,
 # so we will ignore them.

sqlalchemy_exasol/base.py

@@ -53,7 +53,6 @@
 from sqlalchemy import (
     event,
     schema,
-    select,
     sql,
 )
 from sqlalchemy import types as sqltypes
@@ -910,7 +909,7 @@ def get_table_names(self, connection, schema, **kw):
         else:
             sql_statement += ":schema ORDER BY table_name"
             result = connection.execute(
-                sql.text(sql_statement), schema=self.denormalize_name(schema)
+                sql.text(sql_statement), {"schema": self.denormalize_name(schema)}
             )
         tables = [self.normalize_name(row[0]) for row in result]
         return tables
@@ -944,7 +943,7 @@ def get_view_names(self, connection, schema=None, **kw):
         else:
             sql_statement += ":schema ORDER BY view_name"
             result = connection.execute(
-                sql.text(sql_statement), schema=self.denormalize_name(schema)
+                sql.text(sql_statement), {"schema": self.denormalize_name(schema)}
             )
         return [self.normalize_name(row[0]) for row in result]
 
@@ -958,8 +957,10 @@ def get_view_definition(self, connection, view_name, schema=None, **kw):
             sql_stmnt += ":schema"
         result = connection.execute(
             sql.text(sql_stmnt),
-            view_name=self.denormalize_name(view_name),
-            schema=self.denormalize_name(schema),
+            {
+                "view_name": self.denormalize_name(view_name),
+                "schema": self.denormalize_name(schema),
+            },
         ).scalar()
         return result if result else None
 
@@ -999,8 +1000,10 @@ def _get_columns(self, connection, table_name, schema=None, **kw):
         )
         result = connection.execute(
             sql.text(sql_statement),
-            schema=self.denormalize_name(schema),
-            table=self.denormalize_name(table_name),
+            {
+                "schema": self.denormalize_name(schema),
+                "table": self.denormalize_name(table_name),
+            },
         )
         return list(result)
 
@@ -1103,8 +1106,10 @@ def _get_pk_constraint(self, connection, table_name, schema, **kw):
         )
         result = connection.execute(
             sql.text(sql_statement),
-            schema=self.denormalize_name(schema),
-            table=table_name,
+            {
+                "schema": self.denormalize_name(schema),
+                "table": self.denormalize_name(table_name),
+            },
         )
         pkeys = []
         constraint_name = None
@@ -1132,8 +1137,10 @@ def _get_foreign_keys(self, connection, table_name, schema=None, **kw):
         )
         result = connection.execute(
             sql.text(sql_statement),
-            schema=self.denormalize_name(schema),
-            table=self.denormalize_name(table_name),
+            {
+                "schema": self.denormalize_name(schema),
+                "table": self.denormalize_name(table_name),
+            },
         )
         return list(result)
 

test/integration/exasol/test_exasol.py

@@ -91,7 +91,8 @@ def define_tables(cls, metadata):
 
     def test_distribute_by_constraint(self):
         try:
-            Table("t", MetaData(testing.db), autoload=True)
+            with testing.db.connect() as conn:
+                Table("t", MetaData(), autoload_with=conn)
         except:
             assert False
         # TODO: check that reflected table object is identical

test/integration/exasol/test_large_metadata.py

@@ -54,24 +54,25 @@ def test_reflect_table_object(self):
                         )
                         c.execute(sql.text(table_ddl))
 
-                meta = MetaData(bind=config.db)
-                table_name = self.get_table_name(table_count, column_count, 0)
-                start = time.time()
-                # insert into table
-                Table(table_name, meta, autoload=True, schema=self.schema)
-                end = time.time()
-                print(
-                    "table load timer: attempt: 1, table_count: %s, column_count: %s, time: %s"
-                    % (table_count, column_count, (end - start))
-                )
-                start = time.time()
-                # insert into table
-                Table(table_name, meta, autoload=True, schema=self.schema)
-                end = time.time()
-                print(
-                    "table load timer: attempt: 2, table_count: %s, column_count: %s, time: %s"
-                    % (table_count, column_count, (end - start))
-                )
+                with config.db.connect() as conn:
+                    meta = MetaData()
+                    table_name = self.get_table_name(table_count, column_count, 0)
+                    start = time.time()
+                    # insert into table
+                    Table(table_name, meta, autoload_with=conn, schema=self.schema)
+                    end = time.time()
+                    print(
+                        "table load timer: attempt: 1, table_count: %s, column_count: %s, time: %s"
+                        % (table_count, column_count, (end - start))
+                    )
+                    start = time.time()
+                    # insert into table
+                    Table(table_name, meta, autoload_with=conn, schema=self.schema)
+                    end = time.time()
+                    print(
+                        "table load timer: attempt: 2, table_count: %s, column_count: %s, time: %s"
+                        % (table_count, column_count, (end - start))
+                    )
 
     def test_reflect_metadata_object(self):
         for table_count in table_counts:
@@ -89,9 +90,10 @@ def test_reflect_metadata_object(self):
                             self.schema, table_name, column_count
                         )
                         c.execute(sql.text(table_ddl))
-                meta = MetaData(bind=config.db)
+                meta = MetaData()
                 start = time.time()
-                meta.reflect()
+                with config.db.connect() as conn:
+                    meta.reflect(bind=conn)
                 end = time.time()
                 print(
                     "all tables (MetaData.reflect) load timer: table_count: %s, column_count: %s, time: %s"

test/integration/exasol/test_regression.py

@@ -115,10 +115,11 @@ def teardown_class(cls):
         engine = config.db
 
         def _drop_tables(schema):
-            metadata = MetaData(engine, schema=schema)
-            metadata.reflect()
-            to_be_deleted = [metadata.tables[name] for name in metadata.tables]
-            metadata.drop_all(engine, to_be_deleted)
+            metadata = MetaData(schema=schema)
+            with engine.connect() as conn:
+                metadata.reflect(bind=conn)
+                to_be_deleted = [metadata.tables[name] for name in metadata.tables]
+                metadata.drop_all(engine, to_be_deleted)
 
         def _drop_views(schema, views):
             with engine.connect() as conn:

test/integration/sqlalchemy/test_suite.py

@@ -29,8 +29,8 @@ class RowFetchTest(_RowFetchTest):
     RATIONAL = cleandoc(
         """
     PyExasol currently does not support/allow duplicate names in the results set.
-    
-    See also: 
+
+    See also:
     * pyexasol.statement.ExaStatement._check_duplicate_col_names
     """
     )
@@ -66,7 +66,7 @@ def define_views(cls, metadata):
     RATIONALE = cleandoc(
         """
     The Exasol dialect does not check against views for `has_table`, see also `Inspector.has_table()`.
-    
+
     This behaviour is subject to change with sqlalchemy 2.0.
     See also:
     * https://github.com/sqlalchemy/sqlalchemy/blob/3fc6c40ea77c971d3067dab0fdf57a5b5313b69b/lib/sqlalchemy/engine/reflection.py#L415
@@ -116,7 +116,7 @@ class RowCountTest(_RowCountTest):
         the cursor result always will be set to the rowcount = -1 in this case.
         This also is a valid behaviour according to the python DBAPI specification.
         For more details see also:
-        * https://peps.python.org/pep-0249/ 
+        * https://peps.python.org/pep-0249/
         * https://peps.python.org/pep-0249/#rowcount
         * https://peps.python.org/pep-0249/#id21
         * https://peps.python.org/pep-0249/#executemany
@@ -126,7 +126,7 @@ class RowCountTest(_RowCountTest):
     TURBODBC_RATIONALE = cleandoc(
         """
         The currently used turbodbc driver returns invalid results.
-        Attention: 
+        Attention:
         * turbodbc maintenance is paused until if it is clear if there is still demand for it
         * If this tests will succeed in the future consider repining the turbodbc driver
           dependency in order to provide support for this "features".
@@ -292,9 +292,8 @@ def test_integrity_error(self):
     @requirements.duplicate_key_raises_integrity_error
     def test_integrity_error_raw_sql(self):
         insert = text("INSERT INTO MANUAL_PK VALUES (1, 'd1')")
-        with config.db.connect() as conn:
+        with config.db.begin() as conn:
             conn.execute(insert)
-
             assert_raises(exc.IntegrityError, conn.execute, insert)