reorganize project, fix a couple of miscellaneous issues.

Author: alexankitty

lib/dircrawl.js lib/crawler/dircrawl.jslib/dircrawl.js renamed to lib/crawler/dircrawl.js

@@ -1,13 +1,13 @@
-import { getTableRows, parseOutFile } from "./fileworker.js";
+import { getTableRows, parseOutFile } from "./workers/fileworker.js";
 import { Piscina, FixedQueue } from "piscina";
 import { resolve } from "path";
-import debugPrint from "./debugprint.js";
-import { File } from "./models/index.js";
-import { bulkIndexFiles } from "./services/elasticsearch.js";
-import { Timer } from "./time.js";
+import debugPrint from "../utility/printutils.js";
+import { File } from "../database/models/index.js";
+import { bulkIndexFiles } from "../services/elasticsearch.js";
+import { Timer } from "../utility/time.js";
 
 let piscina = new Piscina({
-  filename: resolve("./lib", "fileworker.js"),
+  filename: resolve("./lib/crawler/workers/", "fileworker.js"),
   taskQueue: new FixedQueue(),
 });
 

lib/filehandler.js lib/crawler/filehandler.jslib/filehandler.js renamed to lib/crawler/filehandler.js

@@ -16,13 +16,13 @@ import {
   limit,
   offset,
 } from "@phalcode/ts-igdb-client";
-import { File, Metadata } from "./database.js";
-import TaskQueue from "./taskqueue.js";
-import { singleLineStatus } from "./debugprint.js";
-import { Timer } from "./time.js";
+import { File, Metadata } from "../database/database.js";
+import TaskQueue from "../utility/taskqueue.js";
+import { singleLineStatus } from "../utility/printutils.js";
+import { Timer } from "../utility/time.js";
 import { readFileSync } from "fs";
 
-export default class MetadataSearch {
+export default class MetadataManager {
   constructor() {
     this.twitchSecrets = {
       client_id: process.env.TWITCH_CLIENT_ID,

lib/metadatasearch.js lib/crawler/metadatamanager.jslib/metadatasearch.js renamed to lib/crawler/metadatamanager.js

@@ -181,7 +181,7 @@ function getNonGameTerms() {
   const __filename = fileURLToPath(import.meta.url);
   const __dirname = dirname(__filename);
 
-  const nonGameTermsPath = resolve(__dirname, 'nonGameTerms.json');
+  const nonGameTermsPath = resolve(__dirname, '../../json/terms/nonGameTerms.json');
   nonGameTermsCache = JSON.parse(readFileSync(nonGameTermsPath, 'utf8'));
 
   return nonGameTermsCache;

lib/fileworker.js lib/crawler/workers/fileworker.jslib/fileworker.js renamed to lib/crawler/workers/fileworker.js

@@ -123,8 +123,11 @@ export async function initDB() {
 
     // Only force sync if explicitly requested
     if (process.env.FORCE_FILE_REBUILD === "1") {
+      let queryCount = (await QueryCount.findOne())?.count || 0;
       await sequelize.sync({ force: true });
       await enableTrigram();
+      //restore query count after force sync
+      await QueryCount.create({ count: queryCount });
       console.log("DB forcefully synchronized.");
     }
   } catch (error) {

lib/database.js lib/database/database.jslib/database.js renamed to lib/database/database.js

@@ -1,21 +1,21 @@
-import debugPrint from "./debugprint.js";
-import { bulkIndexFiles } from "./services/elasticsearch.js";
+import debugPrint from "../utility/printutils.js";
+import { bulkIndexFiles } from "../services/elasticsearch.js";
 import { File } from "./models/index.js";
 import { readFileSync } from "fs";
 import { fileURLToPath } from "url";
 import { dirname, resolve } from "path";
 import { Piscina, FixedQueue } from "piscina";
-import { Timer } from "./time.js";
+import { Timer } from "../utility/time.js";
 
 let piscina = new Piscina({
-  filename: resolve("./lib", "dbkwworker.js"),
+  filename: resolve("./lib/database/workers", "dbkwworker.js"),
   taskQueue: new FixedQueue(),
 });
 
-const BATCH_SIZE = 1000;
+const BATCH_SIZE = 100;
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
-const relatedKwRoot = "../lib/json/relatedkeywords/";
+const relatedKwRoot = "../json/relatedkeywords/";
 const catKwPath = resolve(__dirname, relatedKwRoot + "categories.json");
 const nameKwpath = resolve(__dirname, relatedKwRoot + "names.json");
 const regionKwpath = resolve(__dirname, relatedKwRoot + "regions.json");
@@ -57,7 +57,7 @@ export async function optimizeDatabaseKws() {
     singleLineStatus(
       `Optimizing Keywords: ${i} / ${dbLength} ${((i / dbLength) * 100).toFixed(
         2
-      )}% (${proctime.elapsed()})`
+      )}% (${proctime.elapsed()}) Optimized Rows: ${changes}`
     );
 
     for (let x = i; x < currentIndex + BATCH_SIZE; x++) {
@@ -98,7 +98,7 @@ export async function optimizeDatabaseKws() {
         changed = true;
       }
       if (changed) {
-        result[promiseIndex].save();
+        await result[promiseIndex].save();
         changes++;
       }
       promiseIndex++;