Update SQLAlchemy, fix tests

* Not all tests fixed yet
* Fix all `test_aggregate_graphs`
* Fix all `test_store_performance`
* Fix all `test_type_to_term_combination`
* Update SQLAlchemy to version 2.0.23
* Refactor sqlalchemy statements
* Add docker-compose.yml
* Add .env.sample

Author: richardscholtens

.env.sample

@@ -0,0 +1,7 @@
+POSTGRES_USERNAME=username
+POSTGRES_PASSWORD=password
+POSTGRES_HOST=127.0.0.1
+POSTGRES_PORT=5432
+POSTGRES_DBNAME=name
+DB='pgsql'
+DBURI='postgresql+psycopg2://username:[email protected]/name'

.gitignore

@@ -18,3 +18,5 @@ docs/api
 
 # Test outcomes
 test/tmpdb.sqlite
+
+.env

docker-compose.yml

@@ -0,0 +1,18 @@
+---
+version: '3.8'
+services:
+  db:
+    image: postgres:14.1-alpine
+    restart: always
+    environment:
+      - POSTGRES_USER=${POSTGRES_USERNAME}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_HOST=${POSTGRES_HOST}
+      - POSTGRES_DB=${POSTGRES_DBNAME}
+    ports:
+      - "${POSTGRES_PORT}:${POSTGRES_PORT}"
+    volumes:
+      - db:/var/lib/postgresql/data
+volumes:
+  db:
+    driver: local

rdflib_sqlalchemy/sql.py

@@ -62,31 +62,28 @@ def union_select(select_components, distinct=False, select_type=TRIPLE_SELECT):
                 cols = [c.subject, c.predicate, c.object]
             else:
                 raise ValueError('Unrecognized table type {}'.format(tableType))
-            select_clause = expression.select([functions.count().label('aCount')]).select_from(
-                expression.select(cols, whereClause).distinct().select_from(table))
+            select_clause = expression.select(*[functions.count().label('aCount')]).select_from(
+                expression.select(*cols).where(whereClause).distinct().select_from(table))
         elif select_type == CONTEXT_SELECT:
-            select_clause = expression.select([table.c.context], whereClause)
+            select_clause = expression.select(table.c.context)
+            if whereClause is not None:
+                select_clause = expression.select(table.c.context).where(whereClause)
         elif tableType in FULL_TRIPLE_PARTITIONS:
-            select_clause = table.select(whereClause)
+            select_clause = table.select().where(whereClause)
         elif tableType == ASSERTED_TYPE_PARTITION:
             select_clause = expression.select(
-                [table.c.id.label("id"),
+                *[table.c.id.label("id"),
                  table.c.member.label("subject"),
                  expression.literal(text_type(RDF.type)).label("predicate"),
                  table.c.klass.label("object"),
                  table.c.context.label("context"),
                  table.c.termComb.label("termcomb"),
                  expression.literal_column("NULL").label("objlanguage"),
-                 expression.literal_column("NULL").label("objdatatype")],
+                 expression.literal_column("NULL").label("objdatatype")]).where(
                 whereClause)
         elif tableType == ASSERTED_NON_TYPE_PARTITION:
-            select_clause = expression.select(
-                [c for c in table.columns] +
-                [expression.literal_column("NULL").label("objlanguage"),
-                 expression.literal_column("NULL").label("objdatatype")],
-                whereClause,
-                from_obj=[table])
-
+            all_table_columns = [c for c in table.columns] + [expression.literal_column("NULL").label("objlanguage"), expression.literal_column("NULL").label("objdatatype")]
+            select_clause = expression.select(*all_table_columns).select_from(table).where(whereClause)
         selects.append(select_clause)
 
     order_statement = []
@@ -97,6 +94,6 @@ def union_select(select_components, distinct=False, select_type=TRIPLE_SELECT):
             expression.literal_column("object"),
         ]
     if distinct and select_type != COUNT_SELECT:
-        return expression.union(*selects, **{"order_by": order_statement})
+        return expression.union(*selects).order_by(*order_statement)
     else:
-        return expression.union_all(*selects, **{"order_by": order_statement})
+        return expression.union_all(*selects).order_by(*order_statement)

rdflib_sqlalchemy/store.py

@@ -396,7 +396,7 @@ def remove(self, triple, context):
                     if not self.STRONGLY_TYPED_TERMS or isinstance(obj, Literal):
                         # remove literal triple
                         clause = self.build_clause(literal_table, subject, predicate, obj, context)
-                        connection.execute(literal_table.delete(clause))
+                        connection.execute(literal_table.delete().where(clause))
 
                     for table in [quoted_table, asserted_table]:
                         # If asserted non rdf:type table and obj is Literal,
@@ -405,16 +405,16 @@ def remove(self, triple, context):
                             continue
                         else:
                             clause = self.build_clause(table, subject, predicate, obj, context)
-                            connection.execute(table.delete(clause))
+                            connection.execute(table.delete().where(clause))
 
                 if predicate == RDF.type or predicate is None:
                     # Need to check rdf:type and quoted partitions (in addition
                     # perhaps)
                     clause = self.build_clause(asserted_type_table, subject, RDF.type, obj, context, True)
-                    connection.execute(asserted_type_table.delete(clause))
+                    connection.execute(asserted_type_table.delete().where(clause))
 
                     clause = self.build_clause(quoted_table, subject, predicate, obj, context)
-                    connection.execute(quoted_table.delete(clause))
+                    connection.execute(quoted_table.delete().where(clause))
             except Exception:
                 _logger.exception("Removal failed.")
                 raise
@@ -654,7 +654,7 @@ def prefix(self, namespace):
         with self.engine.begin() as connection:
             nb_table = self.tables["namespace_binds"]
             namespace = text_type(namespace)
-            s = select([nb_table.c.prefix]).where(nb_table.c.uri == namespace)
+            s = select(nb_table.c.prefix).where(nb_table.c.uri == namespace)
             res = connection.execute(s)
             rt = [rtTuple[0] for rtTuple in res.fetchall()]
             res.close()
@@ -668,7 +668,7 @@ def namespace(self, prefix):
         try:
             with self.engine.begin() as connection:
                 nb_table = self.tables["namespace_binds"]
-                s = select([nb_table.c.uri]).where(nb_table.c.prefix == prefix_val)
+                s = select(nb_table.c.uri).where(nb_table.c.prefix == prefix_val)
                 res = connection.execute(s)
                 rt = [rtTuple[0] for rtTuple in res.fetchall()]
                 res.close()
@@ -679,7 +679,7 @@ def namespace(self, prefix):
 
     def namespaces(self):
         with self.engine.begin() as connection:
-            res = connection.execute(self.tables["namespace_binds"].select(distinct=True))
+            res = connection.execute(self.tables["namespace_binds"].select().distinct())
             for prefix, uri in res.fetchall():
                 yield prefix, uri
 
@@ -754,7 +754,7 @@ def _remove_context(self, context):
                 for table in [quoted_table, asserted_table,
                               asserted_type_table, literal_table]:
                     clause = self.build_context_clause(context, table)
-                    connection.execute(table.delete(clause))
+                    connection.execute(table.delete().where(clause))
             except Exception:
                 _logger.exception("Context removal failed.")
                 raise

setup.py

@@ -46,7 +46,7 @@
         "alembic>=0.8.8",
         "rdflib>=6,<8",
         "six>=1.10.0",
-        "SQLAlchemy>=1.1.4,<2.0.0",
+        "SQLAlchemy>=2.0.23",
     ],
     entry_points={
         'rdf.plugins.store': [