Remove sequelize pagination in full db tasks cause uh... it has a lot of problems that make no sense to me.

Author: alexankitty

lib/dboptimize.js

@@ -34,22 +34,38 @@ export async function optimizeDatabaseKws() {
   let dbLength = await File.count();
   let optimizeTasks = [];
   let resolvedTasks = [];
+  let promiseIndex = 0;
+  let currentIndex = 0;
+  let result = await File.findAll({
+    order: ["id", "filename"],
+    attributes: [
+      "id",
+      "filename",
+      "filenamekws",
+      "category",
+      "categorykws",
+      "subcategories",
+      "subcategorieskws",
+      "region",
+      "regionkws",
+      "type",
+      "nongame"
+    ],
+  });
   for (let i = 0; i < dbLength; ) {
+    let loopIndexStart = i;
     singleLineStatus(
       `Optimizing Keywords: ${i} / ${dbLength} ${((i / dbLength) * 100).toFixed(
         2
       )}% (${proctime.elapsed()})`
     );
-    let result = await File.findAndCountAll({
-      limit: BATCH_SIZE,
-      offset: i,
-      order: ["id", "filename"],
-    });
-    for (let x = 0; x < result.rows.length; x++) {
-      debugPrint(`Submitting job for: ${result.rows[x]["filename"]}`);
+
+    for (let x = i; x < currentIndex + BATCH_SIZE; x++) {
+      if(x >= dbLength) break; //Abort abandon ship, otherwise we sink
+      debugPrint(`Submitting job for: ${result[x].filename}`);
       let data = [];
       for (let column in keywords) {
-        data[column] = result.rows[x][column];
+        data[column] = result[x][column];
       }
       optimizeTasks.push(
         piscina
@@ -66,23 +82,28 @@ export async function optimizeDatabaseKws() {
       );
       i++;
     }
+    currentIndex = i;
     let settledTasks = await Promise.all(optimizeTasks);
     resolvedTasks.push(...settledTasks);
     debugPrint(`Resolving ${resolvedTasks.length} optimization tasks.`);
     for (let y = 0; y < resolvedTasks.length; y++) {
       let changed = false;
       for (let column in keywords) {
-        if (result.rows[y][column + "kws"] == resolvedTasks[y][column + "kws"])
+        if (
+          result[promiseIndex][column + "kws"] ==
+          resolvedTasks[y][column + "kws"]
+        )
           continue;
-        result.rows[y][column + "kws"] = resolvedTasks[y][column + "kws"];
+        result[promiseIndex][column + "kws"] = resolvedTasks[y][column + "kws"];
         changed = true;
       }
       if (changed) {
-        result.rows[y].save();
+        result[promiseIndex].save();
         changes++;
       }
+      promiseIndex++;
     }
-    await bulkIndexFiles(result.rows);
+    await bulkIndexFiles(result.slice(loopIndexStart, currentIndex));
     optimizeTasks = [];
     resolvedTasks = [];
   }

lib/metadatasearch.js

@@ -115,82 +115,70 @@ export default class MetadataSearch {
   }
 
   async matchAllMetadata(fuzzy = false) {
-    let games = await File.findAndCountAll({
+    let games = await File.findAll({
       where: {
         nongame: false,
-        //detailsId: null,
+        detailsId: null,
       },
-      limit: 1000,
+      attributes: ["id", "filename"],
       order: ["id", "filename"],
+      include: { model: Metadata, as: "details" },
     });
-    let count = games.count;
-    let pages = Math.ceil(games.count / 1000);
+    let count = games.length;
     let timer = new Timer();
     let found = 0;
     console.log(`Matching ${count} games to metadata.`);
-    for (let x = 0; x < pages; x++) {
-      games = await File.findAndCountAll({
-        where: {
-          nongame: false,
-          //detailsId: null,
-        },
-        limit: 1000,
-        offset: x * 1000 - found,
-        order: ["id", "filename"],
-        include: { model: Metadata, as: "details" },
-      });
-      for (let y = 0; y < games.rows.length; y++) {
-        singleLineStatus(
-          `Matching metadata: ${x * 1000 + y} / ${count} ${(
-            ((x * 1000 + y) / count) *
-            100
-          ).toFixed(2)}% (${timer.elapsed()}) Total Matches: ${found}`
+    for (let x = 0; x < count; x++) {
+      singleLineStatus(
+        `Matching metadata: ${x} / ${count} ${(
+          ((x) / count) *
+          100
+        ).toFixed(2)}% (${timer.elapsed()}) Total Matches: ${found}`
+      );
+      let game = games[x];
+      let metadata = await Metadata.searchByText(
+        "title",
+        this.normalizeName(game.filename),
+        game.category
+      );
+      if (metadata?.length == 0) {
+        // repeat the search under one of the alternate titles
+        metadata = await Metadata.searchByText(
+          "alternatetitles",
+          this.normalizeName(game.filename),
+          game.category
         );
-        let game = games.rows[y];
-        let metadata = await Metadata.searchByText(
+      }
+      if (metadata?.length >= 1) {
+        let md = await Metadata.findByPk(metadata[0].id);
+        await game.setDetails(md);
+        await md.addFile(game);
+        await game.save();
+        await md.save();
+        found++;
+      } else if (fuzzy) {
+        //this is much slower and should only be used if the faster full text search can't find it.
+        let metadata = await Metadata.fuzzySearchByText(
           "title",
           this.normalizeName(game.filename),
+          0.8,
           game.category
         );
-        if (!metadata.length) {
-          // repeat the search under one of the alternate titles
-          metadata = await Metadata.searchByText(
+        if (!metadata?.length == 0) {
+          metadata = await Metadata.fuzzySearchByText(
             "alternatetitles",
             this.normalizeName(game.filename),
+            0.8,
             game.category
           );
         }
-        if (metadata.length) {
+        if (metadata?.length >= 1) {
           let md = await Metadata.findByPk(metadata[0].id);
           await game.setDetails(md);
           await md.addFile(game);
           await game.save();
           await md.save();
           found++;
-        } else if (fuzzy) {
-          //this is much slower and should only be used if the faster full text search can't find it.
-          let metadata = await Metadata.fuzzySearchByText(
-            "title",
-            this.normalizeName(game.filename),
-            0.8,
-            game.category
-          );
-          if (!metadata.length) {
-            metadata = await Metadata.fuzzySearchByText(
-              "alternatetitles",
-              this.normalizeName(game.filename),
-              0.8,
-              game.category
-            );
-          }
-          if (metadata.length) {
-            let md = await Metadata.findByPk(metadata[0].id);
-            await game.setDetails(md);
-            await md.addFile(game);
-            await game.save();
-            await md.save();
-            found++;
-          }
         }
       }
     }

lib/models/metadata.js

@@ -105,14 +105,14 @@ export default function (sequelize) {
     if (platform && platform != "Others") {
       platformClause = `AND '${platform}' = ANY(platforms)`;
     }
+    let fieldName = field + 'Vector'
     const query = `
             SELECT id FROM "Metadata"
-            WHERE $1 @@ plainto_tsquery('english', $2) ${platformClause}
+            WHERE "${fieldName}" @@ plainto_tsquery('english', $1) ${platformClause}
             ORDER BY length(title) ${limitClause}
         `;
     return await sequelize.query(query, {
-      model: Metadata,
-      bind: [(field + 'Vector'), searchQuery],
+      bind: [searchQuery],
       type: sequelize.QueryTypes.SELECT,
     });
   };

server.js

@@ -40,6 +40,7 @@ let indexPage = "pages/index";
 let flags = new Flag();
 let consoleIcons = new ConsoleIcons(emulatorsData);
 
+
 // Initialize databases
 await initDB();
 await initElasticsearch();
@@ -579,5 +580,4 @@ if (
 
 cron.schedule("0 30 2 * * *", getFilesJob);
 
-await metadataSearch.syncAllMetadata()
-await metadataSearch.matchAllMetadata()
+await optimizeDatabaseKws()