CHANGES:

- usage added to mv_statistics
- several performance improvements on statistics queries (e.g. REST API)

BUGFIX:
- kills_ships missing from mv_statistics

Author: karel26

plugins/creditsystem/listener.py

@@ -114,8 +114,10 @@ async def get_flighttime(self, ucid: str, campaign_id: int) -> int:
         async with self.apool.connection() as conn:
             cursor = await conn.execute("""
                 SELECT COALESCE(ROUND(SUM(EXTRACT(EPOCH FROM (s.hop_off - s.hop_on)))), 0) AS playtime 
-                FROM statistics s, missions m, campaigns c, campaigns_servers cs 
-                WHERE s.player_ucid = %s AND c.id = %s AND s.mission_id = m.id AND cs.campaign_id = c.id 
+                FROM statistics s JOIN missions m ON m.id = s.mission_id 
+                JOIN campaigns c ON c.id = %s
+                JOIN campaigns_servers cs ON cs.campaign_id = c.id
+                WHERE s.player_ucid = %s 
                 AND m.server_name = cs.server_name 
                 AND tsrange(s.hop_on, s.hop_off) && tsrange(c.start, c.stop)
             """, (ucid, campaign_id))

plugins/missionstats/commands.py

@@ -23,10 +23,10 @@ async def player_modules_autocomplete(interaction: discord.Interaction, current:
     async def get_modules(ucid: str) -> list[str]:
         async with interaction.client.apool.connection() as conn:
             return [row[0] async for row in await conn.execute("""
-                SELECT DISTINCT slot, COUNT(*) FROM statistics 
+                SELECT DISTINCT slot, usage FROM mv_statistics 
                 WHERE player_ucid =  %s 
                 AND slot NOT IN ('', '?', '''forward_observer', 'instructor', 'observer', 'artillery_commander') 
-                GROUP BY 1 ORDER BY 2 DESC
+                ORDER BY 2 DESC
             """, (ucid, ))]
 
     try:

plugins/punishment/listener.py

@@ -60,11 +60,8 @@ async def onMissionLoadEnd(self, server: Server, _: dict) -> None:
 
     async def _get_flight_hours(self, player: Player) -> int:
         async with self.apool.connection() as conn:
-            cursor = await conn.execute("""
-                SELECT COALESCE(ROUND(SUM(EXTRACT(EPOCH FROM (COALESCE(hop_off, now() AT TIME ZONE 'utc') - hop_on)))) / 3600, 0) 
-                       AS playtime 
-                FROM statistics WHERE player_ucid = %s
-            """, (player.ucid, ))
+            cursor = await conn.execute("SELECT SUM(playtime) FROM mv_statistics WHERE player_ucid = %s",
+                                        (player.ucid, ))
             return (await cursor.fetchone())[0] if cursor.rowcount > 0 else 0
 
     async def _get_punishment_points(self, player: Player) -> int:

plugins/restapi/commands.py

@@ -1,6 +1,5 @@
 import asyncio
 import os
-
 import psycopg
 import random
 import re
@@ -283,7 +282,7 @@ async def serverstats(self, server_name: str = Query(default=None)):
                     where = ""
                 await cursor.execute(f"""
                     SELECT SUM("totalPlayers") AS "totalPlayers", SUM("totalPlaytime") AS "totalPlaytime",
-                           SUM("avgPlaytime") AS "avgPlaytime", SUM("activePlayers") AS "activePlayers",
+                           SUM("avgPlaytime") AS "avgPlaytime",
                            SUM("totalSorties") AS "totalSorties", SUM("totalKills") AS "totalKills",
                            SUM("totalDeaths") AS "totalDeaths", SUM("totalPvPKills") AS "totalPvPKills",
                            SUM("totalPvPDeaths") AS "totalPvPDeaths" 
@@ -292,6 +291,15 @@ async def serverstats(self, server_name: str = Query(default=None)):
                 """, {"server_name": server_name})
                 serverstats = await cursor.fetchone()
 
+                if server_name:
+                    server = self.bot.servers.get(server_name)
+                    serverstats['activePlayers'] = len(server.get_active_players())
+                else:
+                    active = 0
+                    for server in self.bot.servers.values():
+                        active += len(server.get_active_players())
+                    serverstats['activePlayers'] = active
+
                 if server_name:
                     join = f"JOIN missions m ON s.mission_id = m.id AND m.server_name = %(server_name)s"
                 else:
@@ -439,9 +447,9 @@ async def leaderboard(self, what: str, order: Literal['asc', 'desc'] = 'desc', q
             raise HTTPException(status_code=400, detail="Invalid ordering column supplied")
 
         if server_name:
-            join = "JOIN missions m ON s.mission_id = m.id AND m.server_name = %(server_name)s"
+            where = "WHERE s.server_name = %(server_name)s"
         else:
-            join = ""
+            where = ""
 
         async with self.apool.connection() as conn:
             async with conn.cursor(row_factory=dict_row) as cursor:
@@ -458,14 +466,14 @@ async def leaderboard(self, what: str, order: Literal['asc', 'desc'] = 'desc', q
                         CASE WHEN SUM(s.deaths_pvp) = 0 
                              THEN SUM(s.pvp) ELSE ROUND(SUM(s.pvp::DECIMAL) / SUM(s.deaths_pvp::DECIMAL), 2) 
                         END AS "kdr_pvp",
-                        ROUND(SUM(EXTRACT(EPOCH FROM(COALESCE(s.hop_off, NOW() AT TIME ZONE 'UTC') - s.hop_on))))::BIGINT AS playtime,
+                        SUM(playtime)::BIGINT AS playtime,
                         MAX(COALESCE(c.points, 0)) AS "credits",
-                        COUNT(*) OVER() as total_count
-                        FROM statistics s 
+                        COUNT(s.usage) as total_count
+                        FROM mv_statistics s 
                         JOIN players p ON s.player_ucid = p.ucid 
-                        {join}
                         LEFT OUTER JOIN credits c ON c.player_ucid = s.player_ucid
                         LEFT OUTER JOIN campaigns ca ON ca.id = c.campaign_id AND NOW() AT TIME ZONE 'utc' BETWEEN ca.start AND COALESCE(ca.stop, NOW() AT TIME ZONE 'utc')
+                        {where}
                         GROUP BY 1, 2 
                         ORDER BY {order_column} {order} 
                         LIMIT %(limit)s
@@ -507,18 +515,17 @@ async def trueskill(self, limit: int = Query(default=10), offset: int = Query(de
                         server_name: str = Query(default=None)):
         self.log.debug(f'Calling /trueskill with limit={limit}, server_name={server_name}')
         if server_name:
-            join = "JOIN missions m ON s.mission_id = m.id AND m.server_name = %(server_name)s"
+            where = "WHERE s.server_name = %(server_name)s"
         else:
-            join = ""
+            where = ""
         async with self.apool.connection() as conn:
             async with conn.cursor(row_factory=dict_row) as cursor:
                 await cursor.execute(f"""
                     SELECT p.name AS "nick", DATE_TRUNC('second', p.last_seen) AS "date",
                     SUM(pvp) AS "kills_pvp", SUM(deaths_pvp) AS "deaths_pvp", t.skill_mu AS "TrueSkill" 
-                    FROM statistics s JOIN players p ON s.player_ucid = p.ucid
+                    FROM mv_statistics s JOIN players p ON s.player_ucid = p.ucid
                     JOIN trueskill t ON t.player_ucid = p.ucid
-                    {join}
-                    WHERE s.hop_on > (now() AT TIME ZONE 'utc') - interval '1 month' 
+                    {where}
                     GROUP BY 1, 2, 5 ORDER BY 5 DESC 
                     LIMIT {limit} OFFSET {offset}
                 """, {"server_name": server_name})
@@ -633,11 +640,11 @@ async def stats(self, nick: str = Form(...), date: str | None = Form(None),
 
         ucid = await self.get_ucid(nick, date)
         if server_name:
-            join = "JOIN missions m ON s.mission_id = m.id AND m.server_name = %(server_name)s"
+            where = "AND s.server_name = %(server_name)s"
         else:
-            join = ""
+            where = ""
         query = f"""
-            SELECT COALESCE(ROUND(SUM(EXTRACT(EPOCH FROM(COALESCE(s.hop_off, NOW() AT TIME ZONE 'UTC') - s.hop_on)))), 0)::BIGINT AS playtime, 
+            SELECT COALESCE(SUM(playtime), 0)::BIGINT AS playtime, 
                    COALESCE(SUM(s.kills), 0) as "kills", 
                    COALESCE(SUM(s.deaths_planes + s.deaths_helicopters + s.deaths_ships + s.deaths_sams + s.deaths_ground), 0) AS "deaths", 
                    COALESCE(SUM(s.pvp), 0) AS "kills_pvp", 
@@ -657,9 +664,9 @@ async def stats(self, nick: str = Form(...), date: str | None = Form(None),
                    COALESCE(SUM(s.ejections), 0) AS "ejections",
                    COALESCE(SUM(s.crashes), 0) AS "crashes", 
                    COALESCE(SUM(s.teamkills), 0) AS "teamkills"
-            FROM statistics s
-            {join}
+            FROM mv_statistics s
             WHERE s.player_ucid = %(ucid)s
+            {where}
         """
         if last_session:
             if server_name:
@@ -685,19 +692,19 @@ async def stats(self, nick: str = Form(...), date: str | None = Form(None),
 
                 await cursor.execute("""
                                      SELECT slot AS "module", SUM(kills) AS "kills"
-                                     FROM statistics
+                                     FROM mv_statistics
                                      WHERE player_ucid = %s
                                      GROUP BY 1
                                      HAVING SUM(kills) > 1
                                      ORDER BY 2 DESC
                                      """, (ucid,))
                 data['killsByModule'] = await cursor.fetchall()
                 await cursor.execute("""
-                                     SELECT slot                                           AS "module",
+                                     SELECT slot AS "module",
                                             CASE
                                                 WHEN SUM(deaths) = 0 THEN SUM(kills)
                                                 ELSE ROUND(SUM(kills) / SUM(deaths::DECIMAL), 2) END AS "kdr"
-                                     FROM statistics
+                                     FROM mv_statistics
                                      WHERE player_ucid = %s
                                      GROUP BY 1
                                      HAVING SUM(kills) > 1
@@ -713,18 +720,18 @@ async def modulestats(self, nick: str = Form(...), date: str | None = Form(None)
 
         ucid = await self.get_ucid(nick, date)
         if server_name:
-            join = "JOIN missions m ON s.mission_id = m.id AND m.server_name = %(server_name)s"
+            where = "AND s.server_name = %(server_name)s"
         else:
-            join = ""
+            where = ""
         query = f"""
             SELECT s.slot AS "module", 
                    SUM(s.kills) AS "kills",
                    SUM(s.deaths_planes + s.deaths_helicopters + s.deaths_ships + s.deaths_sams + s.deaths_ground) AS "deaths",
                    CASE WHEN SUM(s.deaths_planes + s.deaths_helicopters + s.deaths_ships + s.deaths_sams + s.deaths_ground) = 0 
                         THEN SUM(s.kills) ELSE SUM(s.kills)::DECIMAL / SUM((s.deaths_planes + s.deaths_helicopters + s.deaths_ships + s.deaths_sams + s.deaths_ground)::DECIMAL) END AS "kdr" 
-            FROM statistics s
-            {join}
+            FROM mv_statistics s
             WHERE player_ucid = %(ucid)s 
+            {where}
             GROUP BY 1 HAVING SUM(kills) > 1 
             ORDER BY 2 DESC
         """

plugins/restapi/db/tables.sql

@@ -1,15 +1,13 @@
 CREATE MATERIALIZED VIEW mv_serverstats AS
-SELECT m.server_name, COUNT(DISTINCT p.ucid) AS "totalPlayers",
-       ROUND(SUM(EXTRACT(EPOCH FROM(COALESCE(s.hop_off, NOW() AT TIME ZONE 'UTC') - s.hop_on))) / 3600)::INTEGER AS "totalPlaytime",
-       ROUND(AVG(EXTRACT(EPOCH FROM(COALESCE(s.hop_off, NOW() AT TIME ZONE 'UTC') - s.hop_on))))::INTEGER AS "avgPlaytime",
-       SUM(CASE WHEN s.hop_off IS NULL THEN 1 ELSE 0 END) AS "activePlayers",
-       COUNT(*) AS "totalSorties",
+SELECT s.server_name, COUNT(DISTINCT p.ucid) AS "totalPlayers",
+       (SUM(s.playtime) / 3600)::INTEGER AS "totalPlaytime",
+       (SUM(s.playtime) / SUM(s.usage))::INTEGER AS "avgPlaytime",
+       SUM(s.usage) AS "totalSorties",
        SUM(s.kills) AS "totalKills",
        SUM(s.deaths) AS "totalDeaths",
        SUM(s.pvp) AS "totalPvPKills",
        SUM(s.deaths_pvp) AS "totalPvPDeaths",
        NOW() AT TIME ZONE 'UTC' as "timestamp"
 FROM players p
-JOIN statistics s ON p.ucid = s.player_ucid
-JOIN missions m on s.mission_id = m.id
+JOIN mv_statistics s ON p.ucid = s.player_ucid
 GROUP BY 1;

plugins/restapi/db/update_v1.2.sql

@@ -0,0 +1,14 @@
+DROP MATERIALIZED VIEW IF EXISTS mv_serverstats;
+CREATE MATERIALIZED VIEW mv_serverstats AS
+SELECT s.server_name, COUNT(DISTINCT p.ucid) AS "totalPlayers",
+       (SUM(s.playtime) / 3600)::INTEGER AS "totalPlaytime",
+       (SUM(s.playtime) / SUM(s.usage))::INTEGER AS "avgPlaytime",
+       SUM(s.usage) AS "totalSorties",
+       SUM(s.kills) AS "totalKills",
+       SUM(s.deaths) AS "totalDeaths",
+       SUM(s.pvp) AS "totalPvPKills",
+       SUM(s.deaths_pvp) AS "totalPvPDeaths",
+       NOW() AT TIME ZONE 'UTC' as "timestamp"
+FROM players p
+JOIN mv_statistics s ON p.ucid = s.player_ucid
+GROUP BY 1;

plugins/restapi/version.py

@@ -1 +1 @@
-__version__ = "1.2"
+__version__ = "1.3"

plugins/userstats/db/tables.sql

@@ -6,8 +6,8 @@ CREATE TABLE IF NOT EXISTS squadron_members (squadron_id INTEGER NOT NULL, playe
 CREATE UNIQUE INDEX IF NOT EXISTS idx_squadron_members ON squadron_members (player_ucid);
 CREATE MATERIALIZED VIEW IF NOT EXISTS mv_statistics AS
     SELECT s.player_ucid, m.server_name, s.slot, s.tail_no, s.side,
-       SUM(s.kills) AS kills, SUM(s.pvp) AS pvp, SUM(s.deaths) AS deaths, SUM(s.ejections) AS ejections, SUM(s.crashes) AS crashes, SUM(s.teamkills) AS teamkills, SUM(s.takeoffs) AS takeoffs, SUM(s.landings) AS landings,
-	   SUM(s.kills_planes) AS kills_planes, SUM(s.kills_helicopters) AS kills_helicopters, SUM(s.kills_sams) AS kills_sams, SUM(s.kills_ground) AS kills_ground,
+       COUNT(s.slot) AS usage, SUM(s.kills) AS kills, SUM(s.pvp) AS pvp, SUM(s.deaths) AS deaths, SUM(s.ejections) AS ejections, SUM(s.crashes) AS crashes, SUM(s.teamkills) AS teamkills, SUM(s.takeoffs) AS takeoffs, SUM(s.landings) AS landings,
+	   SUM(s.kills_planes) AS kills_planes, SUM(s.kills_helicopters) AS kills_helicopters, SUM(s.kills_ships) AS kills_ships, SUM(s.kills_sams) AS kills_sams, SUM(s.kills_ground) AS kills_ground,
 	   SUM(s.deaths_pvp) AS deaths_pvp, SUM(s.deaths_planes) AS deaths_planes, SUM(s.deaths_helicopters) AS deaths_helicopters, SUM(s.deaths_ships) AS deaths_ships, SUM(s.deaths_sams) AS deaths_sams, SUM(s.deaths_ground) AS deaths_ground,
 	   ROUND(SUM(EXTRACT(EPOCH FROM(COALESCE(s.hop_off, NOW() AT TIME ZONE 'UTC') - s.hop_on)))) AS playtime
     FROM statistics s JOIN missions m ON s.mission_id = m.id

plugins/userstats/db/update_v3.10.sql

@@ -0,0 +1,11 @@
+DROP MATERIALIZED VIEW IF EXISTS mv_statistics;
+CREATE MATERIALIZED VIEW mv_statistics AS
+    SELECT s.player_ucid, m.server_name, s.slot, s.tail_no, s.side,
+       COUNT(s.slot) AS usage, SUM(s.kills) AS kills, SUM(s.pvp) AS pvp, SUM(s.deaths) AS deaths, SUM(s.ejections) AS ejections, SUM(s.crashes) AS crashes, SUM(s.teamkills) AS teamkills, SUM(s.takeoffs) AS takeoffs, SUM(s.landings) AS landings,
+	   SUM(s.kills_planes) AS kills_planes, SUM(s.kills_helicopters) AS kills_helicopters, SUM(s.kills_ships) AS kills_ships, SUM(s.kills_sams) AS kills_sams, SUM(s.kills_ground) AS kills_ground,
+	   SUM(s.deaths_pvp) AS deaths_pvp, SUM(s.deaths_planes) AS deaths_planes, SUM(s.deaths_helicopters) AS deaths_helicopters, SUM(s.deaths_ships) AS deaths_ships, SUM(s.deaths_sams) AS deaths_sams, SUM(s.deaths_ground) AS deaths_ground,
+	   ROUND(SUM(EXTRACT(EPOCH FROM(COALESCE(s.hop_off, NOW() AT TIME ZONE 'UTC') - s.hop_on)))) AS playtime
+    FROM statistics s JOIN missions m ON s.mission_id = m.id
+    GROUP BY 1, 2, 3, 4, 5;
+CREATE INDEX IF NOT EXISTS idx_mv_statistics_ucid ON mv_statistics (player_ucid);
+CREATE INDEX IF NOT EXISTS idx_mv_statistics_tail_no ON mv_statistics (tail_no);

plugins/userstats/version.py

@@ -1 +1 @@
-__version__ = "3.10"
+__version__ = "3.11"