Merge pull request #2574 from IntersectMBO/chore/optimize-connection-pool-and-drep-list-loading

chore: optimize the db connection and drep list sql

Author: MSzalowski

govtool/backend/app/Main.hs

@@ -26,7 +26,7 @@ import           Data.Monoid                            (mempty)
 import           Data.OpenApi                           (OpenApi, Server (Server), _openApiServers,
                                                          _serverDescription, _serverUrl, _serverVariables,
                                                          servers)
-import           Data.Pool                              (createPool)
+import           Data.Pool                              (createPool, Pool)
 import           Data.Proxy
 import           Data.String                            (fromString)
 import           Data.String.Conversions                (cs)
@@ -36,7 +36,7 @@ import qualified Data.Text.IO                           as Text
 import qualified Data.Text.Lazy                         as LazyText
 import qualified Data.Text.Lazy.Encoding                as LazyText
 
-import           Database.PostgreSQL.Simple             (close, connectPostgreSQL)
+import           Database.PostgreSQL.Simple             (close, connectPostgreSQL, Connection)
 
 import           Network.Wai
 import           Network.Wai.Handler.Warp
@@ -65,6 +65,15 @@ import           VVA.Types                              (AppEnv (..),
                                                          AppError (CriticalError, InternalError, NotFoundError, ValidationError),
                                                          CacheEnv (..))
 
+-- Function to create a connection pool with optimized settings
+createOptimizedConnectionPool :: BS.ByteString -> IO (Pool Connection)
+createOptimizedConnectionPool connectionString = createPool
+  (connectPostgreSQL connectionString) -- Connection creation function
+  close                                -- Connection destruction function
+  1                                    -- Number of stripes (sub-pools)
+  60                                   -- Idle timeout (seconds)
+  50                                   -- Maximum number of connections per stripe
+
 proxyAPI :: Proxy (VVAApi :<|> SwaggerAPI)
 proxyAPI = Proxy
 
@@ -125,7 +134,10 @@ startApp vvaConfig sentryService = do
       , dRepListCache
       , networkMetricsCache
       }
-  connectionPool <- createPool (connectPostgreSQL (encodeUtf8 (dbSyncConnectionString $ getter vvaConfig))) close 1 10 60
+
+  let connectionString = encodeUtf8 (dbSyncConnectionString $ getter vvaConfig)
+  connectionPool <- createOptimizedConnectionPool connectionString
+
   let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool }
   server' <- mkVVAServer appEnv
   runSettings settings server'

govtool/backend/sql/list-dreps.sql

@@ -1,15 +1,15 @@
 WITH DRepDistr AS (
   SELECT
-    *,
+    drep_distr.*,
     ROW_NUMBER() OVER (PARTITION BY drep_hash.id ORDER BY drep_distr.epoch_no DESC) AS rn
   FROM
     drep_distr
     JOIN drep_hash ON drep_hash.id = drep_distr.hash_id
 ),
 DRepActivity AS (
   SELECT
-    drep_activity AS drep_activity,
-    epoch_no AS epoch_no
+    drep_activity,
+    epoch_no
   FROM
     epoch_param
   WHERE
@@ -26,7 +26,7 @@ SELECT
   encode(va.data_hash, 'hex'),
   dr_deposit.deposit,
   DRepDistr.amount,
-  (DRepActivity.epoch_no - Max(coalesce(block.epoch_no, block_first_register.epoch_no))) <= DRepActivity.drep_activity AS active,
+  (DRepActivity.epoch_no - COALESCE(block.epoch_no, block_first_register.epoch_no)) <= DRepActivity.drep_activity AS active,
   encode(dr_voting_anchor.tx_hash, 'hex') AS tx_hash,
   newestRegister.time AS last_register_time,
   COALESCE(latestDeposit.deposit, 0),
@@ -50,17 +50,17 @@ FROM
     FROM
       drep_registration dr
     WHERE
-      dr.deposit IS NOT NULL) AS dr_deposit ON dr_deposit.drep_hash_id = dh.id
-  AND dr_deposit.rn = 1
-    JOIN (
+      dr.deposit IS NOT NULL
+  ) AS dr_deposit ON dr_deposit.drep_hash_id = dh.id AND dr_deposit.rn = 1
+  JOIN (
     SELECT
       dr.id,
       dr.drep_hash_id,
       dr.deposit,
       ROW_NUMBER() OVER (PARTITION BY dr.drep_hash_id ORDER BY dr.tx_id DESC) AS rn
     FROM
-      drep_registration dr) AS latestDeposit ON latestDeposit.drep_hash_id = dh.id
-  AND latestDeposit.rn = 1
+      drep_registration dr
+  ) AS latestDeposit ON latestDeposit.drep_hash_id = dh.id AND latestDeposit.rn = 1
   LEFT JOIN (
     SELECT
       dr.id,
@@ -70,9 +70,11 @@ FROM
       tx.hash AS tx_hash
     FROM
       drep_registration dr
-      JOIN tx ON tx.id = dr.tx_id) AS dr_voting_anchor ON dr_voting_anchor.drep_hash_id = dh.id
-    AND dr_voting_anchor.rn = 1
-    LEFT JOIN (
+      JOIN tx ON tx.id = dr.tx_id
+    WHERE
+      dr.deposit IS NOT NULL AND dr.deposit >= 0
+  ) AS dr_voting_anchor ON dr_voting_anchor.drep_hash_id = dh.id AND dr_voting_anchor.rn = 1
+  LEFT JOIN (
     SELECT
       dr.id,
       dr.drep_hash_id,
@@ -82,36 +84,42 @@ FROM
     FROM
       drep_registration dr
       JOIN tx ON tx.id = dr.tx_id
-      WHERE dr.deposit is not null
-      AND dr.deposit >= 0) AS dr_non_deregister_voting_anchor ON dr_non_deregister_voting_anchor.drep_hash_id = dh.id
-    AND dr_non_deregister_voting_anchor.rn = 1
+  ) AS dr_non_deregister_voting_anchor ON dr_non_deregister_voting_anchor.drep_hash_id = dh.id AND dr_non_deregister_voting_anchor.rn = 1
   LEFT JOIN (
     SELECT
       dr.id,
       dr.drep_hash_id,
       dr.voting_anchor_id,
       ROW_NUMBER() OVER (PARTITION BY dr.drep_hash_id ORDER BY dr.tx_id DESC) AS rn
     FROM
-      drep_registration dr) AS second_to_newest_drep_registration ON second_to_newest_drep_registration.drep_hash_id = dh.id
-    AND second_to_newest_drep_registration.rn = 2
-  LEFT JOIN DRepDistr ON DRepDistr.hash_id = dh.id
-    AND DRepDistr.rn = 1
+      drep_registration dr
+  ) AS second_to_newest_drep_registration ON second_to_newest_drep_registration.drep_hash_id = dh.id AND second_to_newest_drep_registration.rn = 2
+  LEFT JOIN DRepDistr ON DRepDistr.hash_id = dh.id AND DRepDistr.rn = 1
   LEFT JOIN voting_anchor va ON va.id = dr_voting_anchor.voting_anchor_id
   LEFT JOIN voting_anchor non_deregister_voting_anchor ON non_deregister_voting_anchor.id = dr_non_deregister_voting_anchor.voting_anchor_id
   LEFT JOIN (
-    SELECT fetch_error as message, voting_anchor_id
-    FROM off_chain_vote_fetch_error
-    WHERE fetch_time = (
-      SELECT max(fetch_time)
-      FROM off_chain_vote_fetch_error)
-    GROUP BY fetch_error, voting_anchor_id
+    SELECT
+      fetch_error as message,
+      voting_anchor_id
+    FROM
+      off_chain_vote_fetch_error
+    WHERE
+      fetch_time = (
+        SELECT
+          max(fetch_time)
+        FROM
+          off_chain_vote_fetch_error
+      )
+    GROUP BY
+      fetch_error,
+      voting_anchor_id
   ) AS fetch_error ON fetch_error.voting_anchor_id = va.id
   LEFT JOIN off_chain_vote_data ON off_chain_vote_data.voting_anchor_id = va.id
-  LEFT JOIN off_chain_vote_drep_data on off_chain_vote_drep_data.off_chain_vote_data_id = off_chain_vote_data.id 
+  LEFT JOIN off_chain_vote_drep_data ON off_chain_vote_drep_data.off_chain_vote_data_id = off_chain_vote_data.id 
   CROSS JOIN DRepActivity
-  LEFT JOIN voting_procedure AS voting_procedure ON voting_procedure.drep_voter = dh.id
-  LEFT JOIN tx AS tx ON tx.id = voting_procedure.tx_id
-  LEFT JOIN block AS block ON block.id = tx.block_id
+  LEFT JOIN voting_procedure ON voting_procedure.drep_voter = dh.id
+  LEFT JOIN tx ON tx.id = voting_procedure.tx_id
+  LEFT JOIN block ON block.id = tx.block_id
   LEFT JOIN (
     SELECT
       block.time,
@@ -122,16 +130,16 @@ FROM
       JOIN tx ON tx.id = dr.tx_id
       JOIN block ON block.id = tx.block_id
     WHERE
-      NOT (dr.deposit < 0)) AS newestRegister ON newestRegister.drep_hash_id = dh.id
-  AND newestRegister.rn = 1
+      NOT (dr.deposit < 0)
+  ) AS newestRegister ON newestRegister.drep_hash_id = dh.id AND newestRegister.rn = 1
   LEFT JOIN (
     SELECT
       dr.tx_id,
       dr.drep_hash_id,
       ROW_NUMBER() OVER (PARTITION BY dr.drep_hash_id ORDER BY dr.tx_id ASC) AS rn
     FROM
-      drep_registration dr) AS dr_first_register ON dr_first_register.drep_hash_id = dh.id
-    AND dr_first_register.rn = 1
+      drep_registration dr
+  ) AS dr_first_register ON dr_first_register.drep_hash_id = dh.id AND dr_first_register.rn = 1
   LEFT JOIN tx AS tx_first_register ON tx_first_register.id = dr_first_register.tx_id
   LEFT JOIN block AS block_first_register ON block_first_register.id = tx_first_register.block_id
 WHERE
@@ -142,6 +150,8 @@ WHERE
     off_chain_vote_drep_data.given_name ILIKE ?
   )
 GROUP BY
+  block_first_register.epoch_no,
+  block.epoch_no,
   dh.raw,
   second_to_newest_drep_registration.voting_anchor_id,
   dh.view,
@@ -163,4 +173,4 @@ GROUP BY
   off_chain_vote_drep_data.motivations,
   off_chain_vote_drep_data.qualifications,
   off_chain_vote_drep_data.image_url,
-  off_chain_vote_drep_data.image_hash
+  off_chain_vote_drep_data.image_hash;