CHANGES

- Massive DB change, all major tables have foreign keys now. Prune / cleanup scripts reduced.

Author: karel26

core/data/impl/instanceimpl.py

@@ -59,8 +59,7 @@ def __post_init__(self):
 #            dirty |= True
         if dirty:
             autoexec.net = net
-
-        server_name = None
+        self._server_name = None
         settings_path = os.path.join(self.home, 'Config', 'serverSettings.lua')
         if os.path.exists(settings_path):
             settings = SettingsDict(self, settings_path, root='cfg')
@@ -69,21 +68,28 @@ def __post_init__(self):
                 settings['port'] = dcs_port
             else:
                 self.locals['dcs_port'] = settings.get('port', 10308)
-            server_name = settings.get('name', 'DCS Server') if settings else None
-            if server_name == 'n/a':
-                server_name = None
-        self.update_instance(server_name)
+            self._server_name = settings.get('name', 'DCS Server') if settings else None
+            if self._server_name == 'n/a':
+                self._server_name = None
+        self.update_instance()
+
+    @property
+    def server_name(self) -> str:
+        if self.server:
+            return self.server.name
+        else:
+            return self._server_name
 
-    def update_instance(self, server_name: str | None = None):
+    def update_instance(self):
         try:
             with self.pool.connection() as conn:
                 with conn.transaction():
                     conn.execute("""
-                        INSERT INTO instances (node, instance, port, server_name)
-                        VALUES (%s, %s, %s, %s) 
+                        INSERT INTO instances (node, instance, port)
+                        VALUES (%s, %s, %s) 
                         ON CONFLICT (node, instance) DO UPDATE 
-                        SET port=excluded.port, server_name=excluded.server_name 
-                    """, (self.node.name, self.name, self.locals.get('bot_port', 6666), server_name))
+                        SET port=excluded.port 
+                    """, (self.node.name, self.name, self.locals.get('bot_port', 6666)))
         except psycopg.errors.UniqueViolation:
             self.log.error(f"bot_port {self.locals.get('bot_port', 6666)} is already in use on node {self.node.name}!")
             raise

core/data/impl/nodeimpl.py

@@ -82,8 +82,8 @@
     "userstats",
     "missionstats",
     "monitoring",
-    "creditsystem",
     "gamemaster",
+    "creditsystem",
     "cloud"
 ]
 
@@ -1066,7 +1066,8 @@ async def check_nodes():
                             return True
                         # The master is not alive, take over
                         elif not master or not await is_node_alive(master, config.get('heartbeat', 30)):
-                            self.log.warning("The master node is not responding, taking over ...")
+                            if master is not None:
+                                self.log.warning(f"The master node {master} is not responding, taking over ...")
                             await take_over()
                             return True
                         # Master is alive, but we are the preferred one
@@ -1211,17 +1212,12 @@ async def rename_file(self, old_name: str, new_name: str, *, force: bool | None
 
     @override
     async def rename_server(self, server: Server, new_name: str):
-        from services.bot import BotService
         from services.servicebus import ServiceBus
 
         if not self.master:
             self.log.error(
                 f"Rename request received for server {server.name} that should have gone to the master node!")
             return
-        # do not rename initially created servers (they should not be there anyway)
-        if server.name != 'n/a':
-            # we are doing the plugin changes, as we are the master
-            await ServiceRegistry.get(BotService).rename_server(server, new_name)
         # update the ServiceBus
         ServiceRegistry.get(ServiceBus).rename_server(server, new_name)
         # change the proxy name for remote servers (ServerImpl will rename local ones)
@@ -1566,8 +1562,12 @@ def change_instance_in_path(data):
             # rename the directory
             os.rename(instance.home, new_home)
             # rename the instance
+            old_name = instance.name
             instance.name = new_name
             instance.locals['home'] = new_home
+            self.instances[new_name] = self.instances.pop(old_name)
+
+            # write the new nodes.yaml
             with open(config_file, mode='w', encoding='utf-8') as outfile:
                 yaml.dump(config, outfile)
 
@@ -1590,7 +1590,8 @@ def change_instance_in_path(data):
                     self.log.exception(f"Failed to start service {list(ServiceRegistry.services().keys())[i]}")
         finally:
             # re-init the attached server instance
-            await instance.server.reload()
+            if instance.server:
+                await instance.server.reload()
 
     @override
     async def find_all_instances(self) -> list[tuple[str, str]]:

core/data/impl/serverimpl.py

@@ -448,22 +448,8 @@ async def update_database(old_name: str, new_name: str):
             async with self.apool.connection() as conn:
                 async with conn.transaction():
                     # we need to remove any older server that might have had the same name
-                    await conn.execute('DELETE FROM servers WHERE server_name = %s', (new_name, ))
-                    await conn.execute("""
-                        UPDATE servers SET server_name = %s WHERE server_name IS NOT DISTINCT FROM %s
-                    """, (new_name, old_name))
-                    await conn.execute('DELETE FROM instances WHERE server_name = %s', (new_name, ))
-                    await conn.execute("""
-                        UPDATE instances 
-                        SET server_name = %s 
-                        WHERE instance = %s AND server_name IS NOT DISTINCT FROM %s
-                    """, (new_name, self.instance.name, old_name))
-                    await conn.execute('DELETE FROM message_persistence WHERE server_name = %s', (new_name, ))
-                    await conn.execute("""
-                        UPDATE message_persistence 
-                        SET server_name = %s 
-                        WHERE server_name IS NOT DISTINCT FROM %s
-                    """, (new_name, old_name))
+                    await conn.execute("UPDATE servers SET server_name = %s WHERE server_name = %s",
+                                       (new_name, old_name))
 
         async def update_cluster(new_name: str):
             # only the master can take care of a cluster-wide rename

core/data/instance.py

@@ -86,3 +86,7 @@ def server(self, server: Server | None):
 
     def set_server(self, server: Server | None):
         self._server = server
+
+    @property
+    def server_name(self) -> str:
+        raise NotImplementedError()

core/plugin.py

@@ -332,8 +332,7 @@ async def before_dcs_update(self) -> None:
     async def after_dcs_update(self) -> None:
         pass
 
-    async def prune(self, conn: psycopg.AsyncConnection, *, days: int = -1, ucids: list[str] = None,
-                    server: str | None = None) -> None:
+    async def prune(self, conn: psycopg.AsyncConnection, days: int) -> None:
         pass
 
     async def _init_db(self) -> bool:
@@ -507,12 +506,8 @@ def get_config(self, server: Server | None = None, *, plugin_name: str | None =
             self._config[server.node.name][server.instance.name] = utils.deep_merge(default, specific)
         return self._config[server.node.name][server.instance.name]
 
-    async def rename(self, conn: psycopg.AsyncConnection, old_name: str, new_name: str) -> None:
-        # this function has to be implemented in your own plugins, if a server rename takes place
-        pass
-
     async def update_ucid(self, conn: psycopg.AsyncConnection, old_ucid: str, new_ucid: str) -> None:
-        # this function has to be implemented in your own plugins, if the ucid of a user changed (steam <=> standalone)
+        # this function has to be implemented in your own plugin if the ucid of a user changed (steam <=> standalone)
         pass
 
     async def on_ready(self) -> None:

plugins/README.md

@@ -128,17 +128,12 @@ class Sample(Plugin[SampleEventListener]):
         # do something after a DCS upgrade took place and before the servers are started
         ...
 
-    async def prune(self, conn: psycopg.AsyncConnection, *, days: int = -1, ucids: list[str] = None,
-                    server: Optional[str] = None) -> None:
-        # cleanup (the database) with data older than days and/or for specific users (ucids) 
-        ...
-
-    async def rename(self, conn: psycopg.AsyncConnection, old_name: str, new_name: str) -> None:
-        # this function has to be implemented in your own plugins if a server rename takes place
+    async def prune(self, conn: psycopg.AsyncConnection, days: int) -> None:
+        # Rare: cleanup (the database) with data older than days (if you have tables with timestamps) 
         ...
 
     async def update_ucid(self, conn: psycopg.AsyncConnection, old_ucid: str, new_ucid: str) -> None:
-        # this function has to be implemented in your own plugins if the ucid of a user changed (steam <=> standalone)
+        # Rare: If you need to do something on an UCID change
         ...
 ```
 > [!NOTE]
@@ -512,10 +507,27 @@ optional "db" directory below your plugin directory.
 tables.sql:
 ```sql
 CREATE TABLE IF NOT EXISTS bans (
-    ucid TEXT PRIMARY KEY, banned_by TEXT NOT NULL, reason TEXT, banned_at TIMESTAMP NOT NULL DEFAULT NOW()
+    ucid TEXT PRIMARY KEY, 
+    banned_by TEXT NOT NULL, 
+    reason TEXT, 
+    banned_at TIMESTAMP NOT NULL DEFAULT NOW()
 );
 ```
 
+> [!NOTE]
+> Whenever using server names (column name server_name) or UCIDs (column name player_ucid or ucid), use foreign keys
+> to the respective master tables.
+> 
+> These are:
+> - servers (server_name) <-> yourtable (server_name)
+> - players (ucid) <-> yourtable (player_ucid)
+> - mission (id) <-> yourtable (mission_id)
+> - squadron (id) <-> yourtable (squadron_id)
+> - campaign (id) <-> yourtable (campaign_id)
+> 
+> For server name and UCID I recommend using ON CASCADE UPDATE and ON CASCADE DELETE, 
+> where you can use ON CASCADE DELETE for all others.
+
 To interact with the database, it is recommended to use the asynchronous database pool offered by each common 
 framework class:
 ```python
@@ -533,6 +545,9 @@ class MyPlugin(Plugin):
                 """, (player.ucid, self.plugin_name, reason))
 ```
 
+> [!NOTE]
+> There is also a synchronous pool, if needed: self.pool
+
 ## Third-party Python libraries
 If your solution needs additional third-party libraries, you can define them in a file named `requirements.local` at 
 the root level of your DCSServerBot installation. 
@@ -584,7 +599,7 @@ class Sample(Plugin[SampleEventListener]):
         if new_version == '1.1':
             # change the config.yaml file to represent the changes introduced in version 1.1
             ...
-            # don't forget to re-read the plugin configuration if you have changed any of it during migration.
+            # remember to re-read the plugin configuration if you have changed any of it during migration.
             self.read_locals()
 ```
 This function handles the tasks necessary for a migration to version `new_version`. 

plugins/admin/commands.py

@@ -714,26 +714,23 @@ async def _prune(self, interaction: discord.Interaction,
                         else:
                             await interaction.followup.send("{} is not a valid UCID!".format(user))
                             return
-                        for plugin in self.bot.cogs.values():  # type: Plugin
-                            await plugin.prune(conn, ucids=[ucid])
-                            await cursor.execute('DELETE FROM players WHERE ucid = %s', (ucid, ))
-                            await cursor.execute('DELETE FROM players_hist WHERE ucid = %s', (ucid, ))
+                        await cursor.execute('DELETE FROM players WHERE ucid = %s', (ucid, ))
                         if isinstance(user, discord.Member):
                             await interaction.followup.send(_("Data of user {} deleted.").format(user.display_name))
                         else:
                             await interaction.followup.send(_("Data of UCID {} deleted.").format(ucid))
                         return
                     elif _server:
-                        for plugin in self.bot.cogs.values():  # type: Plugin
-                            await plugin.prune(conn, server=_server)
-                            await cursor.execute('DELETE FROM servers WHERE server_name = %s', (_server, ))
-                            await cursor.execute('DELETE FROM instances WHERE server_name = %s', (_server, ))
-                            await cursor.execute('DELETE FROM message_persistence WHERE server_name = %s', (_server, ))
+                        await cursor.execute('DELETE FROM servers WHERE server_name = %s', (_server, ))
+                        await cursor.execute('VACCUM FULL statistics')
+                        await cursor.execute('VACUUM FULL missionstats')
                         await interaction.followup.send(_("Data of server {} deleted.").format(_server))
                         return
                     elif view.what in ['users', 'non-members']:
-                        sql = (f"SELECT ucid FROM players "
-                               f"WHERE last_seen < (DATE((now() AT TIME ZONE 'utc')) - interval '{view.age} days')")
+                        sql = f"""
+                            SELECT ucid FROM players 
+                            WHERE last_seen < (DATE((now() AT TIME ZONE 'utc')) - interval '{view.age} days')
+                        """
                         if view.what == 'non-members':
                             sql += ' AND discord_id = -1'
                         await cursor.execute(sql)
@@ -745,20 +742,20 @@ async def _prune(self, interaction: discord.Interaction,
                                 interaction, _("This will delete {} players incl. their stats from the database.\n"
                                                "Are you sure?").format(len(ucids)), ephemeral=ephemeral):
                             return
-                        for plugin in self.bot.cogs.values():  # type: Plugin
-                            await plugin.prune(conn, ucids=ucids)
                         for ucid in ucids:
                             await cursor.execute('DELETE FROM players WHERE ucid = %s', (ucid, ))
-                            await cursor.execute('DELETE FROM players_hist WHERE ucid = %s', (ucid,))
                         await interaction.followup.send(f"{len(ucids)} players pruned.", ephemeral=ephemeral)
                     elif view.what == 'data':
                         days = int(view.age)
                         if not await utils.yn_question(
                                 interaction, _("This will delete all data older than {} days from the database.\n"
                                                "Are you sure?").format(days), ephemeral=ephemeral):
                             return
+                        # some plugins need to prune their data based on the provided days
                         for plugin in self.bot.cogs.values():  # type: Plugin
-                            await plugin.prune(conn, days=days)
+                            await plugin.prune(conn, days)
+                        await cursor.execute('VACCUM FULL statistics')
+                        await cursor.execute('VACUUM FULL missionstats')
                         await interaction.followup.send(_("All data older than {} days pruned.").format(days),
                                                         ephemeral=ephemeral)
         await self.bot.audit(f'pruned the database', user=interaction.user)

plugins/admin/db/tables.sql

@@ -1,3 +1,12 @@
-CREATE TABLE nodestats (id SERIAL PRIMARY KEY, node TEXT NOT NULL, pool_available INTEGER NOT NULL, requests_queued INTEGER NOT NULL, requests_wait_ms INTEGER NOT NULL, dcs_queue INTEGER NOT NULL, asyncio_queue INTEGER NOT NULL, time TIMESTAMP NOT NULL DEFAULT (NOW() AT TIME ZONE 'utc'));
+CREATE TABLE nodestats (
+    id SERIAL PRIMARY KEY,
+    node TEXT NOT NULL,
+    pool_available INTEGER NOT NULL,
+    requests_queued INTEGER NOT NULL,
+    requests_wait_ms INTEGER NOT NULL,
+    dcs_queue INTEGER NOT NULL,
+    asyncio_queue INTEGER NOT NULL,
+    time TIMESTAMP NOT NULL DEFAULT (NOW() AT TIME ZONE 'utc')
+);
 CREATE INDEX IF NOT EXISTS idx_nodestats_node ON nodestats(node);
 CREATE INDEX IF NOT EXISTS idx_nodestats_time ON nodestats(time);

plugins/battleground/commands.py

@@ -1,19 +1,15 @@
 import discord
-import psycopg
-from discord import app_commands
-from discord.app_commands import Group
 
 from core import Plugin, utils, Channel, Coalition, Server, get_translation
+from discord import app_commands
+from discord.app_commands import Group
 from services.bot import DCSServerBot
 
 _ = get_translation(__name__.split('.')[1])
 
 
 class Battleground(Plugin):
 
-    async def rename(self, conn: psycopg.AsyncConnection, old_name: str, new_name: str) -> None:
-        await conn.execute("UPDATE bg_geometry SET server = %s WHERE server= %s", (new_name, old_name))
-
     battleground = Group(name="battleground", description=_("DCSBattleground commands"))
 
     @battleground.command(description=_('Push MGRS coordinates with screenshots to DCS Battleground'))

plugins/battleground/db/tables.sql

@@ -1,3 +1,18 @@
-CREATE TABLE IF NOT EXISTS bg_geometry(id INTEGER PRIMARY KEY, type TEXT NOT NULL, name TEXT, posmgrs TEXT, screenshot TEXT[] DEFAULT '{}'::TEXT[], side TEXT NOT NULL, server TEXT NOT NULL, "position" NUMERIC[] DEFAULT '{}'::NUMERIC[], points NUMERIC[] DEFAULT '{}'::NUMERIC[], center NUMERIC[] DEFAULT '{}'::NUMERIC[], radius NUMERIC DEFAULT 0, discordname TEXT NOT NULL, avatar TEXT NOT NULL);
+CREATE TABLE IF NOT EXISTS bg_geometry(
+    id INTEGER PRIMARY KEY,
+    type TEXT NOT NULL,
+    name TEXT,
+    posmgrs TEXT,
+    screenshot TEXT[] DEFAULT '{}'::TEXT[],
+    side TEXT NOT NULL,
+    server TEXT NOT NULL,
+    "position" NUMERIC[] DEFAULT '{}'::NUMERIC[],
+    points NUMERIC[] DEFAULT '{}'::NUMERIC[],
+    center NUMERIC[] DEFAULT '{}'::NUMERIC[],
+    radius NUMERIC DEFAULT 0,
+    discordname TEXT NOT NULL,
+    avatar TEXT NOT NULL,
+    FOREIGN KEY (server) REFERENCES servers (server_name) ON UPDATE CASCADE ON DELETE CASCADE
+);
 CREATE INDEX IF NOT EXISTS "bg_geometry$server_side" ON bg_geometry USING btree (server ASC, side ASC);
 CREATE SEQUENCE IF NOT EXISTS bg_geometry_id_seq INCREMENT 1 START 20000 MINVALUE 1 OWNED BY bg_geometry."id";