From 96449a48174a6d91a901c77ba547fdbfa5c163d2 Mon Sep 17 00:00:00 2001
From: Alexandra <alexankitty@gmail.com>
Date: Sat, 31 May 2025 13:53:13 -0600
Subject: [PATCH] reorganize project, fix a couple of miscellaneous issues.

---
 lib/{ => crawler}/dircrawl.js                 | 12 ++--
 lib/{ => crawler}/filehandler.js              |  0
 .../metadatamanager.js}                       | 10 +--
 lib/{ => crawler/workers}/fileworker.js       |  2 +-
 lib/{ => database}/database.js                |  3 +
 lib/{ => database}/dboptimize.js              | 16 ++---
 lib/{ => database}/models/file.js             |  0
 lib/{ => database}/models/index.js            |  0
 lib/{ => database}/models/metadata.js         |  0
 lib/{ => database}/models/queryCount.js       |  0
 lib/{ => database/workers}/dbkwworker.js      |  2 +-
 lib/{ => emulator}/emulatorConfig.js          |  2 +-
 lib/{ => images}/consoleicons.js              |  0
 lib/{ => images}/flag.js                      |  2 +-
 lib/{ => json/dynamic_content}/emulators.json |  0
 lib/{ => json/terms}/categories.json          |  0
 lib/{ => json/terms}/nonGameTerms.json        |  0
 lib/{ => search}/search.js                    |  6 +-
 lib/services/elasticsearch.js                 |  6 +-
 lib/{ => utility}/asciiart.js                 |  0
 lib/{debugprint.js => utility/printutils.js}  |  0
 lib/{ => utility}/taskqueue.js                |  0
 lib/{ => utility}/time.js                     |  0
 server.js                                     | 66 +++++++++----------
 24 files changed, 62 insertions(+), 65 deletions(-)
 rename lib/{ => crawler}/dircrawl.js (94%)
 rename lib/{ => crawler}/filehandler.js (100%)
 rename lib/{metadatasearch.js => crawler/metadatamanager.js} (97%)
 rename lib/{ => crawler/workers}/fileworker.js (98%)
 rename lib/{ => database}/database.js (96%)
 rename lib/{ => database}/dboptimize.js (89%)
 rename lib/{ => database}/models/file.js (100%)
 rename lib/{ => database}/models/index.js (100%)
 rename lib/{ => database}/models/metadata.js (100%)
 rename lib/{ => database}/models/queryCount.js (100%)
 rename lib/{ => database/workers}/dbkwworker.js (98%)
 rename lib/{ => emulator}/emulatorConfig.js (99%)
 rename lib/{ => images}/consoleicons.js (100%)
 rename lib/{ => images}/flag.js (92%)
 rename lib/{ => json/dynamic_content}/emulators.json (100%)
 rename lib/{ => json/terms}/categories.json (100%)
 rename lib/{ => json/terms}/nonGameTerms.json (100%)
 rename lib/{ => search}/search.js (83%)
 rename lib/{ => utility}/asciiart.js (100%)
 rename lib/{debugprint.js => utility/printutils.js} (100%)
 rename lib/{ => utility}/taskqueue.js (100%)
 rename lib/{ => utility}/time.js (100%)

diff --git a/lib/dircrawl.js b/lib/crawler/dircrawl.js
similarity index 94%
rename from lib/dircrawl.js
rename to lib/crawler/dircrawl.js
index 7381b7a..9b977f4 100644
--- a/lib/dircrawl.js
+++ b/lib/crawler/dircrawl.js
@@ -1,13 +1,13 @@
-import { getTableRows, parseOutFile } from "./fileworker.js";
+import { getTableRows, parseOutFile } from "./workers/fileworker.js";
 import { Piscina, FixedQueue } from "piscina";
 import { resolve } from "path";
-import debugPrint from "./debugprint.js";
-import { File } from "./models/index.js";
-import { bulkIndexFiles } from "./services/elasticsearch.js";
-import { Timer } from "./time.js";
+import debugPrint from "../utility/printutils.js";
+import { File } from "../database/models/index.js";
+import { bulkIndexFiles } from "../services/elasticsearch.js";
+import { Timer } from "../utility/time.js";
 
 let piscina = new Piscina({
-  filename: resolve("./lib", "fileworker.js"),
+  filename: resolve("./lib/crawler/workers/", "fileworker.js"),
   taskQueue: new FixedQueue(),
 });
 
diff --git a/lib/filehandler.js b/lib/crawler/filehandler.js
similarity index 100%
rename from lib/filehandler.js
rename to lib/crawler/filehandler.js
diff --git a/lib/metadatasearch.js b/lib/crawler/metadatamanager.js
similarity index 97%
rename from lib/metadatasearch.js
rename to lib/crawler/metadatamanager.js
index a6d8bd7..8b326ef 100644
--- a/lib/metadatasearch.js
+++ b/lib/crawler/metadatamanager.js
@@ -16,13 +16,13 @@ import {
   limit,
   offset,
 } from "@phalcode/ts-igdb-client";
-import { File, Metadata } from "./database.js";
-import TaskQueue from "./taskqueue.js";
-import { singleLineStatus } from "./debugprint.js";
-import { Timer } from "./time.js";
+import { File, Metadata } from "../database/database.js";
+import TaskQueue from "../utility/taskqueue.js";
+import { singleLineStatus } from "../utility/printutils.js";
+import { Timer } from "../utility/time.js";
 import { readFileSync } from "fs";
 
-export default class MetadataSearch {
+export default class MetadataManager {
   constructor() {
     this.twitchSecrets = {
       client_id: process.env.TWITCH_CLIENT_ID,
diff --git a/lib/fileworker.js b/lib/crawler/workers/fileworker.js
similarity index 98%
rename from lib/fileworker.js
rename to lib/crawler/workers/fileworker.js
index ebfceb1..1843523 100644
--- a/lib/fileworker.js
+++ b/lib/crawler/workers/fileworker.js
@@ -181,7 +181,7 @@ function getNonGameTerms() {
   const __filename = fileURLToPath(import.meta.url);
   const __dirname = dirname(__filename);
 
-  const nonGameTermsPath = resolve(__dirname, 'nonGameTerms.json');
+  const nonGameTermsPath = resolve(__dirname, '../../json/terms/nonGameTerms.json');
   nonGameTermsCache = JSON.parse(readFileSync(nonGameTermsPath, 'utf8'));
 
   return nonGameTermsCache;
diff --git a/lib/database.js b/lib/database/database.js
similarity index 96%
rename from lib/database.js
rename to lib/database/database.js
index 395ac39..ac12825 100644
--- a/lib/database.js
+++ b/lib/database/database.js
@@ -123,8 +123,11 @@ export async function initDB() {
 
     // Only force sync if explicitly requested
     if (process.env.FORCE_FILE_REBUILD === "1") {
+      let queryCount = (await QueryCount.findOne())?.count || 0;
       await sequelize.sync({ force: true });
       await enableTrigram();
+      //restore query count after force sync
+      await QueryCount.create({ count: queryCount });
       console.log("DB forcefully synchronized.");
     }
   } catch (error) {
diff --git a/lib/dboptimize.js b/lib/database/dboptimize.js
similarity index 89%
rename from lib/dboptimize.js
rename to lib/database/dboptimize.js
index 95a0b22..e685abc 100644
--- a/lib/dboptimize.js
+++ b/lib/database/dboptimize.js
@@ -1,21 +1,21 @@
-import debugPrint from "./debugprint.js";
-import { bulkIndexFiles } from "./services/elasticsearch.js";
+import debugPrint from "../utility/printutils.js";
+import { bulkIndexFiles } from "../services/elasticsearch.js";
 import { File } from "./models/index.js";
 import { readFileSync } from "fs";
 import { fileURLToPath } from "url";
 import { dirname, resolve } from "path";
 import { Piscina, FixedQueue } from "piscina";
-import { Timer } from "./time.js";
+import { Timer } from "../utility/time.js";
 
 let piscina = new Piscina({
-  filename: resolve("./lib", "dbkwworker.js"),
+  filename: resolve("./lib/database/workers", "dbkwworker.js"),
   taskQueue: new FixedQueue(),
 });
 
-const BATCH_SIZE = 1000;
+const BATCH_SIZE = 100;
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
-const relatedKwRoot = "../lib/json/relatedkeywords/";
+const relatedKwRoot = "../json/relatedkeywords/";
 const catKwPath = resolve(__dirname, relatedKwRoot + "categories.json");
 const nameKwpath = resolve(__dirname, relatedKwRoot + "names.json");
 const regionKwpath = resolve(__dirname, relatedKwRoot + "regions.json");
@@ -57,7 +57,7 @@ export async function optimizeDatabaseKws() {
     singleLineStatus(
       `Optimizing Keywords: ${i} / ${dbLength} ${((i / dbLength) * 100).toFixed(
         2
-      )}% (${proctime.elapsed()})`
+      )}% (${proctime.elapsed()}) Optimized Rows: ${changes}`
     );
 
     for (let x = i; x < currentIndex + BATCH_SIZE; x++) {
@@ -98,7 +98,7 @@ export async function optimizeDatabaseKws() {
         changed = true;
       }
       if (changed) {
-        result[promiseIndex].save();
+        await result[promiseIndex].save();
         changes++;
       }
       promiseIndex++;
diff --git a/lib/models/file.js b/lib/database/models/file.js
similarity index 100%
rename from lib/models/file.js
rename to lib/database/models/file.js
diff --git a/lib/models/index.js b/lib/database/models/index.js
similarity index 100%
rename from lib/models/index.js
rename to lib/database/models/index.js
diff --git a/lib/models/metadata.js b/lib/database/models/metadata.js
similarity index 100%
rename from lib/models/metadata.js
rename to lib/database/models/metadata.js
diff --git a/lib/models/queryCount.js b/lib/database/models/queryCount.js
similarity index 100%
rename from lib/models/queryCount.js
rename to lib/database/models/queryCount.js
diff --git a/lib/dbkwworker.js b/lib/database/workers/dbkwworker.js
similarity index 98%
rename from lib/dbkwworker.js
rename to lib/database/workers/dbkwworker.js
index b1cb5dc..58c4e71 100644
--- a/lib/dbkwworker.js
+++ b/lib/database/workers/dbkwworker.js
@@ -1,5 +1,5 @@
 import { ToWords } from "to-words";
-import { getSample } from "./services/elasticsearch.js";
+import { getSample } from "../../services/elasticsearch.js";
 
 const toWords = new ToWords({
   localeCode: "en-US",
diff --git a/lib/emulatorConfig.js b/lib/emulator/emulatorConfig.js
similarity index 99%
rename from lib/emulatorConfig.js
rename to lib/emulator/emulatorConfig.js
index cb3f9f3..fcf39b3 100644
--- a/lib/emulatorConfig.js
+++ b/lib/emulator/emulatorConfig.js
@@ -1,4 +1,4 @@
-import debugPrint from './debugprint.js'
+import debugPrint from '../utility/printutils.js'
 
 // See https://emulatorjs.org/docs/systems for available cores
 const systemConfigs = {
diff --git a/lib/consoleicons.js b/lib/images/consoleicons.js
similarity index 100%
rename from lib/consoleicons.js
rename to lib/images/consoleicons.js
diff --git a/lib/flag.js b/lib/images/flag.js
similarity index 92%
rename from lib/flag.js
rename to lib/images/flag.js
index 9ff6b55..1f11cad 100644
--- a/lib/flag.js
+++ b/lib/images/flag.js
@@ -3,7 +3,7 @@ import { fileURLToPath } from "url";
 import fs from "fs";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const flagsDir = path.join(__dirname, "../views/public/images/flags");
+const flagsDir = path.join(__dirname, "../../views/public/images/flags");
 
 
 export default class Flags {
diff --git a/lib/emulators.json b/lib/json/dynamic_content/emulators.json
similarity index 100%
rename from lib/emulators.json
rename to lib/json/dynamic_content/emulators.json
diff --git a/lib/categories.json b/lib/json/terms/categories.json
similarity index 100%
rename from lib/categories.json
rename to lib/json/terms/categories.json
diff --git a/lib/nonGameTerms.json b/lib/json/terms/nonGameTerms.json
similarity index 100%
rename from lib/nonGameTerms.json
rename to lib/json/terms/nonGameTerms.json
diff --git a/lib/search.js b/lib/search/search.js
similarity index 83%
rename from lib/search.js
rename to lib/search/search.js
index e1dae80..ecae49f 100644
--- a/lib/search.js
+++ b/lib/search/search.js
@@ -1,9 +1,9 @@
-import debugPrint from "./debugprint.js";
+import debugPrint from "../utility/printutils.js";
 import {
   search as elasticSearch,
   getSuggestions as elasticSuggestions,
-} from "./services/elasticsearch.js";
-import { File, Metadata } from "./models/index.js";
+} from "../services/elasticsearch.js";
+import { File, Metadata } from "../database/models/index.js";
 
 export default class Searcher {
   constructor(fields) {
diff --git a/lib/services/elasticsearch.js b/lib/services/elasticsearch.js
index aaa456a..bff5062 100644
--- a/lib/services/elasticsearch.js
+++ b/lib/services/elasticsearch.js
@@ -1,7 +1,7 @@
 import { Client } from "@elastic/elasticsearch";
-import debugPrint from "../debugprint.js";
-import { File, Metadata } from "../models/index.js";
-import { Timer } from "../time.js";
+import debugPrint from "../utility/printutils.js";
+import { File, Metadata } from "../database/models/index.js";
+import { Timer } from "../utility/time.js";
 
 const client = new Client({
   node: process.env.ELASTICSEARCH_URL || "http://localhost:9200",
diff --git a/lib/asciiart.js b/lib/utility/asciiart.js
similarity index 100%
rename from lib/asciiart.js
rename to lib/utility/asciiart.js
diff --git a/lib/debugprint.js b/lib/utility/printutils.js
similarity index 100%
rename from lib/debugprint.js
rename to lib/utility/printutils.js
diff --git a/lib/taskqueue.js b/lib/utility/taskqueue.js
similarity index 100%
rename from lib/taskqueue.js
rename to lib/utility/taskqueue.js
diff --git a/lib/time.js b/lib/utility/time.js
similarity index 100%
rename from lib/time.js
rename to lib/utility/time.js
diff --git a/server.js b/server.js
index 0f89045..2c1ba93 100644
--- a/server.js
+++ b/server.js
@@ -1,38 +1,38 @@
-import getAllFiles from "./lib/dircrawl.js";
-import { optimizeDatabaseKws } from "./lib/dboptimize.js";
-import FileHandler from "./lib/filehandler.js";
-import Searcher from "./lib/search.js";
+import getAllFiles from "./lib/crawler/dircrawl.js";
+import { optimizeDatabaseKws } from "./lib/database/dboptimize.js";
+import FileHandler from "./lib/crawler/filehandler.js";
+import Searcher from "./lib/search/search.js";
 import cron from "node-cron";
 import "dotenv/config";
 import express from "express";
 import http from "http";
 import sanitize from "sanitize";
-import debugPrint from "./lib/debugprint.js";
+import debugPrint from "./lib/utility/printutils.js";
 import compression from "compression";
 import cookieParser from "cookie-parser";
-import { generateAsciiArt } from "./lib/asciiart.js";
+import { generateAsciiArt } from "./lib/utility/asciiart.js";
 import {
   getEmulatorConfig,
   isEmulatorCompatible,
   isNonGameContent,
-} from "./lib/emulatorConfig.js";
+} from "./lib/emulator/emulatorConfig.js";
 import fetch from "node-fetch";
-import { initDB, File, QueryCount, Metadata } from "./lib/database.js";
+import { initDB, File, QueryCount, Metadata } from "./lib/database/database.js";
 import { initElasticsearch } from "./lib/services/elasticsearch.js";
 import i18n, { locales } from "./config/i18n.js";
 import { v4 as uuidv4 } from "uuid";
-import MetadataSearch from "./lib/metadatasearch.js";
-import Flag from "./lib/flag.js";
-import ConsoleIcons from "./lib/consoleicons.js";
+import Flag from "./lib/images/flag.js";
+import ConsoleIcons from "./lib/images/consoleicons.js";
+import MetadataManager from "./lib/crawler/metadatamanager.js";
 
-let categoryListPath = "./lib/categories.json";
-let nonGameTermsPath = "./lib/nonGameTerms.json";
-let emulatorsPath = "./lib/emulators.json";
-let localeNamePath = "./lib/json/maps/name_localization.json"
+let categoryListPath = "./lib/json/terms/categories.json";
+let nonGameTermsPath = "./lib/json/terms/nonGameTerms.json";
+let emulatorsPath = "./lib/json/dynamic_content/emulators.json";
+let localeNamePath = "./lib/json/maps/name_localization.json";
 let categoryList = await FileHandler.parseJsonFile(categoryListPath);
 let nonGameTerms = await FileHandler.parseJsonFile(nonGameTermsPath);
 let emulatorsData = await FileHandler.parseJsonFile(emulatorsPath);
-let localeNames = await FileHandler.parseJsonFile(localeNamePath)
+let localeNames = await FileHandler.parseJsonFile(localeNamePath);
 let crawlTime = 0;
 let queryCount = 0;
 let fileCount = 0;
@@ -40,7 +40,6 @@ let indexPage = "pages/index";
 let flags = new Flag();
 let consoleIcons = new ConsoleIcons(emulatorsData);
 
-
 // Initialize databases
 await initDB();
 await initElasticsearch();
@@ -73,11 +72,11 @@ for (let field in searchFields) {
 }
 
 let search = new Searcher(searchFields);
-let metadataSearch = new MetadataSearch();
+let metadataManager = new MetadataManager();
 
 async function getFilesJob() {
   console.log("Updating the file list.");
-  let oldFileCount = fileCount || 0
+  let oldFileCount = fileCount || 0;
   fileCount = await getAllFiles(categoryList);
   if (!fileCount) {
     console.log("File update failed");
@@ -85,16 +84,16 @@ async function getFilesJob() {
   }
   crawlTime = Date.now();
   console.log(`Finished updating file list. ${fileCount} found.`);
-  if(await Metadata.count() < await metadataSearch.getIGDBGamesCount()){
-    await metadataSearch.syncAllMetadata();
+  if ((await Metadata.count()) < (await metadataManager.getIGDBGamesCount())) {
+    await metadataManager.syncAllMetadata();
   }
-  if(fileCount > oldFileCount){
-    await metadataSearch.matchAllMetadata()
+  if (fileCount > oldFileCount) {
+    await metadataManager.matchAllMetadata();
   }
   await optimizeDatabaseKws();
   //this is less important and needs to run last.
-  if(fileCount > oldFileCount){
-    metadataSearch.matchAllMetadata(true)
+  if (fileCount > oldFileCount) {
+    metadataManager.matchAllMetadata(true);
   }
 }
 
@@ -220,7 +219,7 @@ app.get("/search", async function (req, res) {
     delete settings.combineWith;
   }
   let loadOldResults =
-    req.query.old === "true" || !await Metadata.count() ? true : false;
+    req.query.old === "true" || !(await Metadata.count()) ? true : false;
   settings.pageSize = loadOldResults ? 100 : 10;
   settings.page = pageNum - 1;
   settings.sort = req.query.o || "";
@@ -243,7 +242,7 @@ app.get("/search", async function (req, res) {
     settings: settings,
     flags: flags,
     consoleIcons: consoleIcons,
-    localeNames: localeNames
+    localeNames: localeNames,
   };
   let page = loadOldResults ? "resultsold" : "results";
   options = buildOptions(page, options);
@@ -278,7 +277,7 @@ app.get("/lucky", async function (req, res) {
 app.get("/settings", async function (req, res) {
   let options = { defaultSettings: defaultSettings };
   let page = "settings";
-  options.oldSettingsAvailable = await Metadata.count() ? true : false;
+  options.oldSettingsAvailable = (await Metadata.count()) ? true : false;
   options = buildOptions(page, options);
   res.render(indexPage, options);
 });
@@ -333,10 +332,6 @@ app.get("/play/:id", async function (req, res) {
 app.get("/info/:id", async function (req, res) {
   //set header to allow video embed
   res.setHeader("Cross-Origin-Embedder-Policy", "unsafe-non");
-  if (!metadataSearch.authorized) {
-    res.redirect("/");
-    return;
-  }
   let romId = parseInt(req.params.id);
   let romFile = await search.findIndex(romId);
   if (!romFile) {
@@ -345,14 +340,14 @@ app.get("/info/:id", async function (req, res) {
   }
   let options = {
     file: {
-      ...romFile.dataValues
+      ...romFile.dataValues,
     },
     metadata: {
-      ...romFile?.details?.dataValues
+      ...romFile?.details?.dataValues,
     },
     flags: flags,
     consoleIcons: consoleIcons,
-    localeNames: localeNames
+    localeNames: localeNames,
   };
   let page = "info";
   options = buildOptions(page, options);
@@ -579,4 +574,3 @@ if (
 }
 
 cron.schedule("0 30 2 * * *", getFilesJob);
-