From b84a18e6bf56a56b3bf7d994f5f42cab0e255765 Mon Sep 17 00:00:00 2001 From: Gord Thompson Date: Wed, 13 Nov 2024 09:24:03 -0700 Subject: [PATCH 1/2] Apply updates for CRDB v24.3 --- .github/workflows/ci.yml | 5 +- README.asyncpg.md | 76 ++++++++++++++++++++++++++++- README.read_committed.md | 37 -------------- dev-requirements.txt | 32 ++++++------ sqlalchemy_cockroachdb/provision.py | 7 +++ test-requirements.txt | 20 ++++---- test/test_suite_sqlalchemy.py | 44 ++++++++--------- 7 files changed, 130 insertions(+), 91 deletions(-) delete mode 100644 README.read_committed.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c1f024..9cc16ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,9 +40,9 @@ jobs: fail-fast: false matrix: crdb-version: [ - "cockroach:latest-v23.2", "cockroach:latest-v24.1", - "cockroach:latest-v24.2" + "cockroach:latest-v24.2", + "cockroach:latest-v24.3" ] db-alias: [ "psycopg2", @@ -84,4 +84,3 @@ jobs: run: pip install --user tox==${TOX_VERSION} - name: Lint run: ${HOME}/.local/bin/tox -e lint - diff --git a/README.asyncpg.md b/README.asyncpg.md index 7c73fa8..812a94f 100644 --- a/README.asyncpg.md +++ b/README.asyncpg.md @@ -9,9 +9,81 @@ There is a customized version of the FastAPI SQL database tutorial for https://github.com/gordthompson/fastapi-tutorial-cockroachdb-async -### Database support for Alembic +### Default transaction isolation level -CockroachDB version 23.1 or later is required to work with Alembic. +Applications using asyncpg that were developed prior to CockroachDB's inclusion of +READ COMMITTED transaction isolation may operate on the assumption that the default +isolation level will be SERIALIZABLE. For example, + +```python +import asyncio + +from sqlalchemy.ext.asyncio import create_async_engine + + +async def async_main(): + engine = create_async_engine( + "cockroachdb+asyncpg://root@localhost:26257/defaultdb", + ) + async with engine.begin() as conn: + result = await conn.exec_driver_sql("select version()") + print(result.scalar().split("(")[0]) # CockroachDB CCL v23.2.4 + + result = await conn.exec_driver_sql("show transaction isolation level") + print(result.scalar()) # serializable + + +asyncio.run(async_main()) +``` + +With current versions of CockroachDB, the default transaction isolation level +**for asyncpg only** is now READ COMMITTED + +```python +import asyncio + +from sqlalchemy.ext.asyncio import create_async_engine + + +async def async_main(): + engine = create_async_engine( + "cockroachdb+asyncpg://root@localhost:26257/defaultdb", + ) + async with engine.begin() as conn: + result = await conn.exec_driver_sql("select version()") + print(result.scalar().split("(")[0]) # CockroachDB CCL v24.3.0 + + result = await conn.exec_driver_sql("show transaction isolation level") + print(result.scalar()) # read committed + + +asyncio.run(async_main()) +``` + +Applications that rely on the original behavior will have to add `isolation_level="SERIALIZABLE"` +to their `create_async_engine()` call + +```python +import asyncio + +from sqlalchemy.ext.asyncio import create_async_engine + + +async def async_main(): + engine = create_async_engine( + "cockroachdb+asyncpg://root@localhost:26257/defaultdb", + isolation_level="SERIALIZABLE", + ) + async with engine.begin() as conn: + result = await conn.exec_driver_sql("select version()") + print(result.scalar().split("(")[0]) # CockroachDB CCL v24.3.0 + + result = await conn.exec_driver_sql("show transaction isolation level") + print(result.scalar()) # serializable + + +asyncio.run(async_main()) +``` ### Testing diff --git a/README.read_committed.md b/README.read_committed.md deleted file mode 100644 index e506271..0000000 --- a/README.read_committed.md +++ /dev/null @@ -1,37 +0,0 @@ -## READ COMMITTED transaction isolation - -CockroachDB v23.2.0 added support for READ COMMITTED transaction isolation as -a "preview feature", meaning that we must opt-in to activate it by sending -the statement - -``` -SET CLUSTER SETTING sql.txn.read_committed_isolation.enabled = true; -``` - -This statement changes a persisted setting in the CockroachDB cluster. It is meant -to be executed one time by a database operator/administrator. - -For testing purposes, this adapter offers a custom `connect=` function that we -can pass to `create_engine()`, which will configure this setting: - -```python -import psycopg2 -from sqlalchemy import create_engine - -def connect_for_read_committed(): - cnx = psycopg2.connect("host=localhost port=26257 user=root dbname=defaultdb") - cnx.autocommit = True - crs = cnx.cursor() - crs.execute("SET CLUSTER SETTING sql.txn.read_committed_isolation.enabled = true;") - cnx.autocommit = False - return cnx - -engine = create_engine( - "cockroachdb+psycopg2://", - creator=connect_for_read_committed, - isolation_level="READ COMMITTED", -) - -with engine.begin() as conn: - conn.exec_driver_sql("UPDATE tbl SET txt = 'SQLAlchemy' WHERE id = 1") -``` diff --git a/dev-requirements.txt b/dev-requirements.txt index 6eb116b..fa68bde 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,12 +1,12 @@ backports-tarfile==1.2.0 # via jaraco-context -certifi==2024.8.30 +certifi==2024.12.14 # via requests cffi==1.17.1 # via cryptography -charset-normalizer==3.4.0 +charset-normalizer==3.4.1 # via requests -cryptography==43.0.1 +cryptography==44.0.0 # via secretstorage distlib==0.3.9 # via virtualenv @@ -32,7 +32,7 @@ jeepney==0.8.0 # via # keyring # secretstorage -keyring==25.4.1 +keyring==25.6.0 # via twine markdown-it-py==3.0.0 # via rich @@ -42,11 +42,13 @@ more-itertools==10.5.0 # via # jaraco-classes # jaraco-functools -nh3==0.2.18 +nh3==0.2.20 # via readme-renderer -packaging==24.1 - # via tox -pkginfo==1.10.0 +packaging==24.2 + # via + # tox + # twine +pkginfo==1.12.0 # via twine platformdirs==4.3.6 # via virtualenv @@ -56,7 +58,7 @@ py==1.11.0 # via tox pycparser==2.22 # via cffi -pygments==2.18.0 +pygments==2.19.1 # via # readme-renderer # rich @@ -70,25 +72,25 @@ requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.9.2 +rich==13.9.4 # via twine secretstorage==3.3.3 # via keyring -six==1.16.0 +six==1.17.0 # via tox toml==0.10.2 # via tox tox==3.23.1 # via -r dev-requirements.in -twine==5.1.1 +twine==6.0.1 # via -r dev-requirements.in typing-extensions==4.12.2 # via rich -urllib3==2.2.3 +urllib3==2.3.0 # via # requests # twine -virtualenv==20.26.6 +virtualenv==20.28.1 # via tox -zipp==3.20.2 +zipp==3.21.0 # via importlib-metadata diff --git a/sqlalchemy_cockroachdb/provision.py b/sqlalchemy_cockroachdb/provision.py index 357fb50..db7e616 100644 --- a/sqlalchemy_cockroachdb/provision.py +++ b/sqlalchemy_cockroachdb/provision.py @@ -1,6 +1,13 @@ from sqlalchemy.testing.provision import temp_table_keyword_args +from sqlalchemy.testing.provision import update_db_opts @temp_table_keyword_args.for_db("cockroachdb") def _cockroachdb_temp_table_keyword_args(cfg, eng): return {"prefixes": ["TEMPORARY"]} + + +@update_db_opts.for_db("cockroachdb") +def _update_db_opts(db_url, db_opts, options): + """Set database options (db_opts) for a test database that we created.""" + db_opts["isolation_level"] = "SERIALIZABLE" diff --git a/test-requirements.txt b/test-requirements.txt index 4b38e41..b9d35d2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,10 +1,10 @@ -alembic==1.13.3 +alembic==1.14.0 # via -r test-requirements.in -async-timeout==4.0.3 +async-timeout==5.0.1 # via asyncpg -asyncpg==0.29.0 +asyncpg==0.30.0 # via -r test-requirements.in -attrs==24.2.0 +attrs==24.3.0 # via pytest futures==3.0.5 # via -r test-requirements.in @@ -12,31 +12,31 @@ greenlet==3.1.1 # via sqlalchemy iniconfig==2.0.0 # via pytest -mako==1.3.5 +mako==1.3.8 # via alembic -markupsafe==3.0.1 +markupsafe==3.0.2 # via mako mock==5.1.0 # via -r test-requirements.in more-itertools==10.5.0 # via -r test-requirements.in -packaging==24.1 +packaging==24.2 # via pytest pluggy==1.5.0 # via pytest psycopg==3.2.3 # via -r test-requirements.in -psycopg2==2.9.9 +psycopg2==2.9.10 # via -r test-requirements.in py==1.11.0 # via pytest pytest==7.1.3 # via -r test-requirements.in -sqlalchemy==2.0.36 +sqlalchemy==2.0.37 # via # -r test-requirements.in # alembic -tomli==2.0.2 +tomli==2.2.1 # via pytest typing-extensions==4.12.2 # via diff --git a/test/test_suite_sqlalchemy.py b/test/test_suite_sqlalchemy.py index 722359d..7c932e3 100644 --- a/test/test_suite_sqlalchemy.py +++ b/test/test_suite_sqlalchemy.py @@ -9,6 +9,7 @@ ) from sqlalchemy.testing.suite import HasIndexTest as _HasIndexTest from sqlalchemy.testing.suite import HasTableTest as _HasTableTest +from sqlalchemy.testing.suite import IntegerTest as _IntegerTest from sqlalchemy.testing.suite import InsertBehaviorTest as _InsertBehaviorTest from sqlalchemy.testing.suite import IsolationLevelTest as _IsolationLevelTest from sqlalchemy.testing.suite import ( @@ -417,24 +418,19 @@ def test_no_results_for_non_returning_insert(self): pass -class IsolationLevelTest(_IsolationLevelTest): - def test_all_levels(self): - if not config.db.dialect._is_v232plus: - # TODO: enable when READ COMMITTED no longer a preview feature, since - # SET CLUSTER SETTING cannot be used inside a multi-statement transaction - super().test_all_levels() +class IntegerTest(_IntegerTest): + @_IntegerTest._huge_ints() + def test_huge_int(self, integer_round_trip, intvalue): + if config.db.dialect.driver != "asyncpg": + super().test_huge_int(integer_round_trip, intvalue) + +class IsolationLevelTest(_IsolationLevelTest): @skip("cockroachdb") def test_dialect_user_setting_is_restored(self): # IndexError: list index out of range pass - def test_non_default_isolation_level(self): - if not config.db.dialect._is_v232plus: - # TODO: enable when READ COMMITTED no longer a preview feature, since - # SET CLUSTER SETTING cannot be used inside a multi-statement transaction - super().test_non_default_isolation_level() - class LongNameBlowoutTest(_LongNameBlowoutTest): @testing.combinations( @@ -450,6 +446,18 @@ def test_long_convention_name(self, type_, metadata, connection): super().test_long_convention_name(type_, metadata, connection, None) +class NumericTest(_NumericTest): + def test_numeric_as_float(self, do_numeric_test): + # psycopg.errors.InvalidParameterValue: unsupported binary operator: + + if config.db.dialect.driver != "psycopg": + super().test_numeric_as_float(do_numeric_test) + + def test_numeric_null_as_float(self, do_numeric_test): + # psycopg.errors.InvalidParameterValue: unsupported binary operator: + + if config.db.dialect.driver != "psycopg": + super().test_numeric_null_as_float(do_numeric_test) + + class QuotedNameArgumentTest(_QuotedNameArgumentTest): def quote_fixtures(fn): return testing.combinations( @@ -464,18 +472,6 @@ def test_get_indexes(self, name): super().test_get_indexes(name, None) -class NumericTest(_NumericTest): - def test_numeric_as_float(self, do_numeric_test): - # psycopg.errors.InvalidParameterValue: unsupported binary operator: + - if config.db.dialect.driver != "psycopg": - super().test_numeric_as_float(do_numeric_test) - - def test_numeric_null_as_float(self, do_numeric_test): - # psycopg.errors.InvalidParameterValue: unsupported binary operator: + - if config.db.dialect.driver != "psycopg": - super().test_numeric_null_as_float(do_numeric_test) - - class TrueDivTest(_TrueDivTest): @skip("cockroachdb") def test_floordiv_integer(self): From 6a9db5877ba7bdd604e94fc9d4fe2ae404600dfc Mon Sep 17 00:00:00 2001 From: Gord Thompson Date: Thu, 6 Feb 2025 14:10:50 -0700 Subject: [PATCH 2/2] Apply updates for SQLA v2.0.38 --- README.asyncpg.md | 76 ----------------------------- dev-requirements.txt | 17 ++++--- sqlalchemy_cockroachdb/provision.py | 7 --- test-requirements.txt | 12 ++--- 4 files changed, 15 insertions(+), 97 deletions(-) diff --git a/README.asyncpg.md b/README.asyncpg.md index 812a94f..039e14e 100644 --- a/README.asyncpg.md +++ b/README.asyncpg.md @@ -9,82 +9,6 @@ There is a customized version of the FastAPI SQL database tutorial for https://github.com/gordthompson/fastapi-tutorial-cockroachdb-async -### Default transaction isolation level - -Applications using asyncpg that were developed prior to CockroachDB's inclusion of -READ COMMITTED transaction isolation may operate on the assumption that the default -isolation level will be SERIALIZABLE. For example, - -```python -import asyncio - -from sqlalchemy.ext.asyncio import create_async_engine - - -async def async_main(): - engine = create_async_engine( - "cockroachdb+asyncpg://root@localhost:26257/defaultdb", - ) - async with engine.begin() as conn: - result = await conn.exec_driver_sql("select version()") - print(result.scalar().split("(")[0]) # CockroachDB CCL v23.2.4 - - result = await conn.exec_driver_sql("show transaction isolation level") - print(result.scalar()) # serializable - - -asyncio.run(async_main()) -``` - -With current versions of CockroachDB, the default transaction isolation level -**for asyncpg only** is now READ COMMITTED - -```python -import asyncio - -from sqlalchemy.ext.asyncio import create_async_engine - - -async def async_main(): - engine = create_async_engine( - "cockroachdb+asyncpg://root@localhost:26257/defaultdb", - ) - async with engine.begin() as conn: - result = await conn.exec_driver_sql("select version()") - print(result.scalar().split("(")[0]) # CockroachDB CCL v24.3.0 - - result = await conn.exec_driver_sql("show transaction isolation level") - print(result.scalar()) # read committed - - -asyncio.run(async_main()) -``` - -Applications that rely on the original behavior will have to add `isolation_level="SERIALIZABLE"` -to their `create_async_engine()` call - -```python -import asyncio - -from sqlalchemy.ext.asyncio import create_async_engine - - -async def async_main(): - engine = create_async_engine( - "cockroachdb+asyncpg://root@localhost:26257/defaultdb", - isolation_level="SERIALIZABLE", - ) - async with engine.begin() as conn: - result = await conn.exec_driver_sql("select version()") - print(result.scalar().split("(")[0]) # CockroachDB CCL v24.3.0 - - result = await conn.exec_driver_sql("show transaction isolation level") - print(result.scalar()) # serializable - - -asyncio.run(async_main()) -``` - ### Testing Assuming that you have an entry in test.cfg that looks something like diff --git a/dev-requirements.txt b/dev-requirements.txt index fa68bde..83ddf9b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ backports-tarfile==1.2.0 # via jaraco-context -certifi==2024.12.14 +certifi==2025.1.31 # via requests cffi==1.17.1 # via cryptography @@ -12,13 +12,15 @@ distlib==0.3.9 # via virtualenv docutils==0.21.2 # via readme-renderer -filelock==3.16.1 +filelock==3.17.0 # via # tox # virtualenv +id==1.5.0 + # via twine idna==3.10 # via requests -importlib-metadata==8.5.0 +importlib-metadata==8.6.1 # via # keyring # twine @@ -38,7 +40,7 @@ markdown-it-py==3.0.0 # via rich mdurl==0.1.2 # via markdown-it-py -more-itertools==10.5.0 +more-itertools==10.6.0 # via # jaraco-classes # jaraco-functools @@ -48,8 +50,6 @@ packaging==24.2 # via # tox # twine -pkginfo==1.12.0 - # via twine platformdirs==4.3.6 # via virtualenv pluggy==1.5.0 @@ -66,6 +66,7 @@ readme-renderer==44.0 # via twine requests==2.32.3 # via + # id # requests-toolbelt # twine requests-toolbelt==1.0.0 @@ -82,7 +83,7 @@ toml==0.10.2 # via tox tox==3.23.1 # via -r dev-requirements.in -twine==6.0.1 +twine==6.1.0 # via -r dev-requirements.in typing-extensions==4.12.2 # via rich @@ -90,7 +91,7 @@ urllib3==2.3.0 # via # requests # twine -virtualenv==20.28.1 +virtualenv==20.29.1 # via tox zipp==3.21.0 # via importlib-metadata diff --git a/sqlalchemy_cockroachdb/provision.py b/sqlalchemy_cockroachdb/provision.py index db7e616..357fb50 100644 --- a/sqlalchemy_cockroachdb/provision.py +++ b/sqlalchemy_cockroachdb/provision.py @@ -1,13 +1,6 @@ from sqlalchemy.testing.provision import temp_table_keyword_args -from sqlalchemy.testing.provision import update_db_opts @temp_table_keyword_args.for_db("cockroachdb") def _cockroachdb_temp_table_keyword_args(cfg, eng): return {"prefixes": ["TEMPORARY"]} - - -@update_db_opts.for_db("cockroachdb") -def _update_db_opts(db_url, db_opts, options): - """Set database options (db_opts) for a test database that we created.""" - db_opts["isolation_level"] = "SERIALIZABLE" diff --git a/test-requirements.txt b/test-requirements.txt index b9d35d2..5647a79 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,10 +1,10 @@ -alembic==1.14.0 +alembic==1.14.1 # via -r test-requirements.in async-timeout==5.0.1 # via asyncpg asyncpg==0.30.0 # via -r test-requirements.in -attrs==24.3.0 +attrs==25.1.0 # via pytest futures==3.0.5 # via -r test-requirements.in @@ -12,19 +12,19 @@ greenlet==3.1.1 # via sqlalchemy iniconfig==2.0.0 # via pytest -mako==1.3.8 +mako==1.3.9 # via alembic markupsafe==3.0.2 # via mako mock==5.1.0 # via -r test-requirements.in -more-itertools==10.5.0 +more-itertools==10.6.0 # via -r test-requirements.in packaging==24.2 # via pytest pluggy==1.5.0 # via pytest -psycopg==3.2.3 +psycopg==3.2.4 # via -r test-requirements.in psycopg2==2.9.10 # via -r test-requirements.in @@ -32,7 +32,7 @@ py==1.11.0 # via pytest pytest==7.1.3 # via -r test-requirements.in -sqlalchemy==2.0.37 +sqlalchemy==2.0.38 # via # -r test-requirements.in # alembic