fixes revision upgrade logic

Author: David Erb

src/dls_normsql/aiomysql.py

@@ -77,11 +77,6 @@ def __init__(self, specification, database_definition_object):
 
         self.__tables = {}
 
-        # Deriving class has not established its latest revision?
-        if not hasattr(self, "LATEST_REVISION") or self.LATEST_REVISION is None:
-            # Presume it is 1.
-            self.LATEST_REVISION = 1
-
         self.__backup_restore_lock = asyncio.Lock()
 
         # Last undo position.
@@ -147,12 +142,13 @@ async def connect(self, should_drop_database=False):
             if should_create_schemas:
                 await self.create_schemas()
                 await self.insert(
-                    Tablenames.REVISION, [{"number": self.LATEST_REVISION}]
+                    Tablenames.REVISION,
+                    [{"number": self.__database_definition_object.LATEST_REVISION}],
                 )
 
             # Emit the name of the database file for positive confirmation on console.
             logger.info(
-                f"{callsign(self)} database name is {self.__database_name} code revision {self.LATEST_REVISION}"
+                f"{callsign(self)} database name is {self.__database_name} database definition revision {self.__database_definition_object.LATEST_REVISION}"
             )
 
     # ----------------------------------------------------------------------------------------
@@ -170,36 +166,38 @@ async def apply_revisions(self):
                     why="get database revision",
                 )
                 if len(records) == 0:
-                    old_revision = 0
+                    database_revision = 0
                 else:
-                    old_revision = records[0]["number"]
+                    database_revision = records[0]["number"]
             except Exception as exception:
                 logger.warning(
                     f"could not get revision, presuming legacy database with no table: {exception}"
                 )
-                old_revision = 0
+                database_revision = 0
 
-            if old_revision < self.LATEST_REVISION:
+            if database_revision < self.__database_definition_object.LATEST_REVISION:
                 # Backup before applying revisions.
                 logger.debug(
-                    f"[BKREVL] backing up before updating from revision {old_revision} to revision {self.LATEST_REVISION}"
+                    f"[BKREVL] backing up before updating from database revision {database_revision}"
+                    f" to definition revision {self.__database_definition_object.LATEST_REVISION}"
                 )
 
                 await self.backup()
 
-                for revision in range(old_revision, self.LATEST_REVISION):
-                    logger.debug(f"updating to revision {revision+1}")
+                for revision in range(
+                    database_revision, self.__database_definition_object.LATEST_REVISION
+                ):
                     await self.apply_revision(revision + 1)
                 await self.update(
                     Tablenames.REVISION,
-                    {"number": self.LATEST_REVISION},
+                    {"number": self.__database_definition_object.LATEST_REVISION},
                     "1 = 1",
                     why="update database revision",
                 )
             else:
                 logger.debug(
-                    f"[BKREVL] no need to update persistent revision {old_revision}"
-                    f" which matches code revision {self.LATEST_REVISION}"
+                    f"[BKREVL] no need to update database revision {database_revision}"
+                    f" which matches definition revision {self.__database_definition_object.LATEST_REVISION}"
                 )
 
     # ----------------------------------------------------------------------------------------
@@ -212,6 +210,9 @@ async def apply_revision(self, revision):
             await self.create_table(Tablenames.REVISION)
             await self.insert(Tablenames.REVISION, [{"revision": revision}])
 
+        # Let the database definition object do its thing.
+        await self.__database_definition_object.apply_revision(self, revision)
+
     # ----------------------------------------------------------------------------------------
     async def disconnect(self):
 
@@ -551,39 +552,7 @@ async def backup(self):
         """
 
         async with self.__backup_restore_lock:
-            # Prune all the restores which were orphaned.
-            directory = self.__backup_directory
-            if directory is None:
-                raise RuntimeError("no backup directory supplied in confirmation")
-
-            basename, suffix = os.path.splitext(os.path.basename(self.__database_name))
-
-            filenames = glob.glob(f"{directory}/{basename}.*{suffix}")
-
-            filenames.sort(reverse=True)
-
-            for restore in range(self.__last_restore):
-                logger.debug(
-                    f"[BACKPRU] removing {restore}-th restore {filenames[restore]}"
-                )
-                os.remove(filenames[restore])
-
-            self.__last_restore = 0
-
-            timestamp = isodatetime_filename()
-            to_filename = f"{directory}/{basename}.{timestamp}{suffix}"
-
-            await self.disconnect()
-            try:
-                await self.__create_directory(to_filename)
-                shutil.copy2(self.__database_name, to_filename)
-                logger.debug(f"backed up to {to_filename}")
-            except Exception:
-                raise RuntimeError(
-                    f"copy {self.__database_name} to {to_filename} failed"
-                )
-            finally:
-                await self.connect()
+            pass
 
     # ----------------------------------------------------------------------------------------
     async def restore(self, nth):
@@ -592,37 +561,7 @@ async def restore(self, nth):
         """
 
         async with self.__backup_restore_lock:
-            directory = self.__backup_directory
-            if directory is None:
-                raise RuntimeError("no backup directory supplied in confirmation")
-
-            basename, suffix = os.path.splitext(os.path.basename(self.__database_name))
-
-            filenames = glob.glob(f"{directory}/{basename}.*{suffix}")
-
-            filenames.sort(reverse=True)
-
-            if nth >= len(filenames):
-                raise RuntimeError(
-                    f"restoration index {nth} is more than available {len(filenames)}"
-                )
-
-            from_filename = filenames[nth]
-
-            await self.disconnect()
-            try:
-                shutil.copy2(from_filename, self.__database_name)
-                logger.debug(
-                    f"restored nth {nth} out of {len(filenames)} from {from_filename}"
-                )
-            except Exception:
-                raise RuntimeError(
-                    f"copy {from_filename} to {self.__database_name} failed"
-                )
-            finally:
-                await self.connect()
-
-            self.__last_restore = nth
+            pass
 
 
 # ----------------------------------------------------------------------------------------

src/dls_normsql/aiosqlite.py

@@ -65,11 +65,6 @@ def __init__(self, specification, database_definition_object):
 
         self.__tables = {}
 
-        # Deriving class has not established its latest revision?
-        if not hasattr(self, "LATEST_REVISION") or self.LATEST_REVISION is None:
-            # Presume it is 1.
-            self.LATEST_REVISION = 1
-
         self.__backup_restore_lock = asyncio.Lock()
 
         # Last undo position.
@@ -126,14 +121,15 @@ async def connect(self, should_drop_database=False):
             if should_create_schemas:
                 await self.create_schemas()
                 await self.insert(
-                    Tablenames.REVISION, [{"number": self.LATEST_REVISION}]
+                    Tablenames.REVISION,
+                    [{"number": self.__database_definition_object.LATEST_REVISION}],
                 )
                 # TODO: Set permission on sqlite file from configuration.
                 os.chmod(self.__filename, 0o666)
 
             # Emit the name of the database file for positive confirmation on console.
             logger.info(
-                f"{callsign(self)} database file is {self.__filename} code revision {self.LATEST_REVISION}"
+                f"{callsign(self)} database file is {self.__filename} database definition revision {self.__database_definition_object.LATEST_REVISION}"
             )
 
     # ----------------------------------------------------------------------------------------
@@ -151,36 +147,38 @@ async def apply_revisions(self):
                     why="get database revision",
                 )
                 if len(records) == 0:
-                    old_revision = 0
+                    database_revision = 0
                 else:
-                    old_revision = records[0]["number"]
+                    database_revision = records[0]["number"]
             except Exception as exception:
                 logger.warning(
                     f"could not get revision, presuming legacy database with no table: {exception}"
                 )
-                old_revision = 0
+                database_revision = 0
 
-            if old_revision < self.LATEST_REVISION:
+            if database_revision < self.__database_definition_object.LATEST_REVISION:
                 # Backup before applying revisions.
                 logger.debug(
-                    f"[BKREVL] backing up before updating from revision {old_revision} to revision {self.LATEST_REVISION}"
+                    f"[BKREVL] backing up before updating from database revision {database_revision}"
+                    f" to definition revision {self.__database_definition_object.LATEST_REVISION}"
                 )
 
                 await self.backup()
 
-                for revision in range(old_revision, self.LATEST_REVISION):
-                    logger.debug(f"updating to revision {revision+1}")
+                for revision in range(
+                    database_revision, self.__database_definition_object.LATEST_REVISION
+                ):
                     await self.apply_revision(revision + 1)
                 await self.update(
                     Tablenames.REVISION,
-                    {"number": self.LATEST_REVISION},
+                    {"number": self.__database_definition_object.LATEST_REVISION},
                     "1 = 1",
                     why="update database revision",
                 )
             else:
                 logger.debug(
-                    f"[BKREVL] no need to update persistent revision {old_revision}"
-                    f" which matches code revision {self.LATEST_REVISION}"
+                    f"[BKREVL] no need to update database revision {database_revision}"
+                    f" which matches definition revision {self.__database_definition_object.LATEST_REVISION}"
                 )
 
     # ----------------------------------------------------------------------------------------
@@ -194,7 +192,7 @@ async def apply_revision(self, revision):
             await self.insert(Tablenames.REVISION, [{"revision": revision}])
 
         # Let the database definition object do its thing.
-        self.__database_definition_object.apply_revisions(self)
+        await self.__database_definition_object.apply_revision(self, revision)
 
     # ----------------------------------------------------------------------------------------
     async def disconnect(self):

tests/my_database_definition.py

@@ -25,6 +25,13 @@ async def apply_revision(self, database, revision):
 
         logger.debug(f"applying revision {revision}")
 
+        if revision == 4:
+            await database.execute("CREATE TABLE `my_table2` (`number` INTEGER)")
+
+            await database.execute(
+                f"ALTER TABLE my_table2 ADD COLUMN string TEXT",
+            )
+
     # ----------------------------------------------------------------------------------------
     async def add_table_definitions(self, database):
         """

tests/test_revision.py

@@ -0,0 +1,102 @@
+import logging
+
+import pytest
+from dls_utilpack.envvar import Envvar
+
+from dls_normsql.constants import ClassTypes, RevisionFieldnames, Tablenames
+from dls_normsql.databases import Databases
+from tests.base_tester import BaseTester
+from tests.my_database_definition import MyDatabaseDefinition
+
+logger = logging.getLogger(__name__)
+
+
+# ----------------------------------------------------------------------------------------
+class TestRevisionAiosqlite:
+    def test(self, logging_setup, output_directory):
+        """
+        Tests the sqlite implementation of Database.
+        """
+
+        # Database specification.
+        database_specification = {
+            "type": ClassTypes.AIOSQLITE,
+            "filename": f"{output_directory}/database.sqlite",
+        }
+
+        # Test direct SQL access to the database.
+        RevisionTester().main(
+            database_specification,
+            output_directory,
+        )
+
+
+# ----------------------------------------------------------------------------------------
+class TestRevisionAiomysql:
+    def test(self, logging_setup, output_directory):
+        """
+        Tests the mysql implementation of Database.
+        """
+
+        host = Envvar("MYSQL_HOST", default="127.0.0.1")
+        assert host.is_set
+        port = Envvar("MYSQL_PORT", default=3306)
+        assert port.is_set
+
+        # Database specification.
+        database_specification = {
+            "type": ClassTypes.AIOMYSQL,
+            "type_specific_tbd": {
+                "database_name": "dls_normsql_pytest",
+                "host": "$MYSQL_HOST",
+                "port": "$MYSQL_PORT",
+                "username": "root",
+                "password": "root",
+            },
+        }
+
+        # Test direct SQL access to the database.
+        RevisionTester().main(
+            database_specification,
+            output_directory,
+        )
+
+
+# ----------------------------------------------------------------------------------------
+class RevisionTester(BaseTester):
+    """
+    Test direct SQL access to the database.
+    """
+
+    async def _main_coroutine(self, database_specification, output_directory):
+        """ """
+
+        database_definition_object = MyDatabaseDefinition()
+        databases = Databases()
+        database1 = databases.build_object(
+            database_specification, database_definition_object
+        )
+
+        try:
+            # Connect to database.
+            await database1.connect(should_drop_database=True)
+
+            with pytest.raises(Exception):
+                # We should not have the mytable2.
+                records = await database1.query("SELECT * FROM my_table2")
+
+            # Back level the database's revision.
+            await database1.update(
+                Tablenames.REVISION, {RevisionFieldnames.NUMBER: "3"}, "1=1"
+            )
+
+            # Apply the revisions up to the definition revision which is 4.
+            await database1.apply_revisions()
+
+            # Now we should have the mytable2.
+            records = await database1.query("SELECT * FROM my_table2")
+            assert len(records) == 0
+
+        finally:
+            # Disonnect from the databases... necessary to allow asyncio loop to exit.
+            await database1.disconnect()