reorganize project, fix a couple of miscellaneous issues.

lib/crawler/dircrawl.js

@@ -1,13 +1,13 @@
-import { getTableRows, parseOutFile } from "./fileworker.js";
+import { getTableRows, parseOutFile } from "./workers/fileworker.js";
 import { Piscina, FixedQueue } from "piscina";
 import { resolve } from "path";
-import debugPrint from "./debugprint.js";
-import { File } from "./models/index.js";
-import { bulkIndexFiles } from "./services/elasticsearch.js";
-import { Timer } from "./time.js";
+import debugPrint from "../utility/printutils.js";
+import { File } from "../database/models/index.js";
+import { bulkIndexFiles } from "../services/elasticsearch.js";
+import { Timer } from "../utility/time.js";
 
 let piscina = new Piscina({
-  filename: resolve("./lib", "fileworker.js"),
+  filename: resolve("./lib/crawler/workers/", "fileworker.js"),
   taskQueue: new FixedQueue(),
 });
 

lib/crawler/metadatamanager.js

@@ -16,13 +16,13 @@ import {
   limit,
   offset,
 } from "@phalcode/ts-igdb-client";
-import { File, Metadata } from "./database.js";
-import TaskQueue from "./taskqueue.js";
-import { singleLineStatus } from "./debugprint.js";
-import { Timer } from "./time.js";
+import { File, Metadata } from "../database/database.js";
+import TaskQueue from "../utility/taskqueue.js";
+import { singleLineStatus } from "../utility/printutils.js";
+import { Timer } from "../utility/time.js";
 import { readFileSync } from "fs";
 
-export default class MetadataSearch {
+export default class MetadataManager {
   constructor() {
     this.twitchSecrets = {
       client_id: process.env.TWITCH_CLIENT_ID,

lib/crawler/workers/fileworker.js

@@ -181,7 +181,7 @@ function getNonGameTerms() {
   const __filename = fileURLToPath(import.meta.url);
   const __dirname = dirname(__filename);
 
-  const nonGameTermsPath = resolve(__dirname, 'nonGameTerms.json');
+  const nonGameTermsPath = resolve(__dirname, '../../json/terms/nonGameTerms.json');
   nonGameTermsCache = JSON.parse(readFileSync(nonGameTermsPath, 'utf8'));
 
   return nonGameTermsCache;

lib/database/database.js

@@ -123,8 +123,11 @@ export async function initDB() {
 
     // Only force sync if explicitly requested
     if (process.env.FORCE_FILE_REBUILD === "1") {
+      let queryCount = (await QueryCount.findOne())?.count || 0;
       await sequelize.sync({ force: true });
       await enableTrigram();
+      //restore query count after force sync
+      await QueryCount.create({ count: queryCount });
       console.log("DB forcefully synchronized.");
     }
   } catch (error) {

lib/database/dboptimize.js

@@ -1,21 +1,21 @@
-import debugPrint from "./debugprint.js";
-import { bulkIndexFiles } from "./services/elasticsearch.js";
+import debugPrint from "../utility/printutils.js";
+import { bulkIndexFiles } from "../services/elasticsearch.js";
 import { File } from "./models/index.js";
 import { readFileSync } from "fs";
 import { fileURLToPath } from "url";
 import { dirname, resolve } from "path";
 import { Piscina, FixedQueue } from "piscina";
-import { Timer } from "./time.js";
+import { Timer } from "../utility/time.js";
 
 let piscina = new Piscina({
-  filename: resolve("./lib", "dbkwworker.js"),
+  filename: resolve("./lib/database/workers", "dbkwworker.js"),
   taskQueue: new FixedQueue(),
 });
 
-const BATCH_SIZE = 1000;
+const BATCH_SIZE = 100;
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
-const relatedKwRoot = "../lib/json/relatedkeywords/";
+const relatedKwRoot = "../json/relatedkeywords/";
 const catKwPath = resolve(__dirname, relatedKwRoot + "categories.json");
 const nameKwpath = resolve(__dirname, relatedKwRoot + "names.json");
 const regionKwpath = resolve(__dirname, relatedKwRoot + "regions.json");
@@ -57,7 +57,7 @@ export async function optimizeDatabaseKws() {
     singleLineStatus(
       `Optimizing Keywords: ${i} / ${dbLength} ${((i / dbLength) * 100).toFixed(
         2
-      )}% (${proctime.elapsed()})`
+      )}% (${proctime.elapsed()}) Optimized Rows: ${changes}`
     );
 
     for (let x = i; x < currentIndex + BATCH_SIZE; x++) {
@@ -98,7 +98,7 @@ export async function optimizeDatabaseKws() {
         changed = true;
       }
       if (changed) {
-        result[promiseIndex].save();
+        await result[promiseIndex].save();
         changes++;
       }
       promiseIndex++;

lib/database/workers/dbkwworker.js

@@ -1,5 +1,5 @@
 import { ToWords } from "to-words";
-import { getSample } from "./services/elasticsearch.js";
+import { getSample } from "../../services/elasticsearch.js";
 
 const toWords = new ToWords({
   localeCode: "en-US",

lib/emulator/emulatorConfig.js

@@ -1,4 +1,4 @@
-import debugPrint from './debugprint.js'
+import debugPrint from '../utility/printutils.js'
 
 // See https://emulatorjs.org/docs/systems for available cores
 const systemConfigs = {

lib/images/flag.js

@@ -3,7 +3,7 @@ import { fileURLToPath } from "url";
 import fs from "fs";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const flagsDir = path.join(__dirname, "../views/public/images/flags");
+const flagsDir = path.join(__dirname, "../../views/public/images/flags");
 
 
 export default class Flags {

lib/search/search.js

@@ -1,9 +1,9 @@
-import debugPrint from "./debugprint.js";
+import debugPrint from "../utility/printutils.js";
 import {
   search as elasticSearch,
   getSuggestions as elasticSuggestions,
-} from "./services/elasticsearch.js";
-import { File, Metadata } from "./models/index.js";
+} from "../services/elasticsearch.js";
+import { File, Metadata } from "../database/models/index.js";
 
 export default class Searcher {
   constructor(fields) {

lib/services/elasticsearch.js

@@ -1,7 +1,7 @@
 import { Client } from "@elastic/elasticsearch";
-import debugPrint from "../debugprint.js";
-import { File, Metadata } from "../models/index.js";
-import { Timer } from "../time.js";
+import debugPrint from "../utility/printutils.js";
+import { File, Metadata } from "../database/models/index.js";
+import { Timer } from "../utility/time.js";
 
 const client = new Client({
   node: process.env.ELASTICSEARCH_URL || "http://localhost:9200",

server.js

@@ -1,38 +1,38 @@
-import getAllFiles from "./lib/dircrawl.js";
-import { optimizeDatabaseKws } from "./lib/dboptimize.js";
-import FileHandler from "./lib/filehandler.js";
-import Searcher from "./lib/search.js";
+import getAllFiles from "./lib/crawler/dircrawl.js";
+import { optimizeDatabaseKws } from "./lib/database/dboptimize.js";
+import FileHandler from "./lib/crawler/filehandler.js";
+import Searcher from "./lib/search/search.js";
 import cron from "node-cron";
 import "dotenv/config";
 import express from "express";
 import http from "http";
 import sanitize from "sanitize";
-import debugPrint from "./lib/debugprint.js";
+import debugPrint from "./lib/utility/printutils.js";
 import compression from "compression";
 import cookieParser from "cookie-parser";
-import { generateAsciiArt } from "./lib/asciiart.js";
+import { generateAsciiArt } from "./lib/utility/asciiart.js";
 import {
   getEmulatorConfig,
   isEmulatorCompatible,
   isNonGameContent,
-} from "./lib/emulatorConfig.js";
+} from "./lib/emulator/emulatorConfig.js";
 import fetch from "node-fetch";
-import { initDB, File, QueryCount, Metadata } from "./lib/database.js";
+import { initDB, File, QueryCount, Metadata } from "./lib/database/database.js";
 import { initElasticsearch } from "./lib/services/elasticsearch.js";
 import i18n, { locales } from "./config/i18n.js";
 import { v4 as uuidv4 } from "uuid";
-import MetadataSearch from "./lib/metadatasearch.js";
-import Flag from "./lib/flag.js";
-import ConsoleIcons from "./lib/consoleicons.js";
+import Flag from "./lib/images/flag.js";
+import ConsoleIcons from "./lib/images/consoleicons.js";
+import MetadataManager from "./lib/crawler/metadatamanager.js";
 
-let categoryListPath = "./lib/categories.json";
-let nonGameTermsPath = "./lib/nonGameTerms.json";
-let emulatorsPath = "./lib/emulators.json";
-let localeNamePath = "./lib/json/maps/name_localization.json"
+let categoryListPath = "./lib/json/terms/categories.json";
+let nonGameTermsPath = "./lib/json/terms/nonGameTerms.json";
+let emulatorsPath = "./lib/json/dynamic_content/emulators.json";
+let localeNamePath = "./lib/json/maps/name_localization.json";
 let categoryList = await FileHandler.parseJsonFile(categoryListPath);
 let nonGameTerms = await FileHandler.parseJsonFile(nonGameTermsPath);
 let emulatorsData = await FileHandler.parseJsonFile(emulatorsPath);
-let localeNames = await FileHandler.parseJsonFile(localeNamePath)
+let localeNames = await FileHandler.parseJsonFile(localeNamePath);
 let crawlTime = 0;
 let queryCount = 0;
 let fileCount = 0;
@@ -40,7 +40,6 @@ let indexPage = "pages/index";
 let flags = new Flag();
 let consoleIcons = new ConsoleIcons(emulatorsData);
 
-
 // Initialize databases
 await initDB();
 await initElasticsearch();
@@ -73,11 +72,11 @@ for (let field in searchFields) {
 }
 
 let search = new Searcher(searchFields);
-let metadataSearch = new MetadataSearch();
+let metadataManager = new MetadataManager();
 
 async function getFilesJob() {
   console.log("Updating the file list.");
-  let oldFileCount = fileCount || 0
+  let oldFileCount = fileCount || 0;
   fileCount = await getAllFiles(categoryList);
   if (!fileCount) {
     console.log("File update failed");
@@ -85,16 +84,16 @@ async function getFilesJob() {
   }
   crawlTime = Date.now();
   console.log(`Finished updating file list. ${fileCount} found.`);
-  if(await Metadata.count() < await metadataSearch.getIGDBGamesCount()){
-    await metadataSearch.syncAllMetadata();
+  if ((await Metadata.count()) < (await metadataManager.getIGDBGamesCount())) {
+    await metadataManager.syncAllMetadata();
   }
-  if(fileCount > oldFileCount){
-    await metadataSearch.matchAllMetadata()
+  if (fileCount > oldFileCount) {
+    await metadataManager.matchAllMetadata();
   }
   await optimizeDatabaseKws();
   //this is less important and needs to run last.
-  if(fileCount > oldFileCount){
-    metadataSearch.matchAllMetadata(true)
+  if (fileCount > oldFileCount) {
+    metadataManager.matchAllMetadata(true);
   }
 }
 
@@ -220,7 +219,7 @@ app.get("/search", async function (req, res) {
     delete settings.combineWith;
   }
   let loadOldResults =
-    req.query.old === "true" || !await Metadata.count() ? true : false;
+    req.query.old === "true" || !(await Metadata.count()) ? true : false;
   settings.pageSize = loadOldResults ? 100 : 10;
   settings.page = pageNum - 1;
   settings.sort = req.query.o || "";
@@ -243,7 +242,7 @@ app.get("/search", async function (req, res) {
     settings: settings,
     flags: flags,
     consoleIcons: consoleIcons,
-    localeNames: localeNames
+    localeNames: localeNames,
   };
   let page = loadOldResults ? "resultsold" : "results";
   options = buildOptions(page, options);
@@ -278,7 +277,7 @@ app.get("/lucky", async function (req, res) {
 app.get("/settings", async function (req, res) {
   let options = { defaultSettings: defaultSettings };
   let page = "settings";
-  options.oldSettingsAvailable = await Metadata.count() ? true : false;
+  options.oldSettingsAvailable = (await Metadata.count()) ? true : false;
   options = buildOptions(page, options);
   res.render(indexPage, options);
 });
@@ -333,10 +332,6 @@ app.get("/play/:id", async function (req, res) {
 app.get("/info/:id", async function (req, res) {
   //set header to allow video embed
   res.setHeader("Cross-Origin-Embedder-Policy", "unsafe-non");
-  if (!metadataSearch.authorized) {
-    res.redirect("/");
-    return;
-  }
   let romId = parseInt(req.params.id);
   let romFile = await search.findIndex(romId);
   if (!romFile) {
@@ -345,14 +340,14 @@ app.get("/info/:id", async function (req, res) {
   }
   let options = {
     file: {
-      ...romFile.dataValues
+      ...romFile.dataValues,
     },
     metadata: {
-      ...romFile?.details?.dataValues
+      ...romFile?.details?.dataValues,
     },
     flags: flags,
     consoleIcons: consoleIcons,
-    localeNames: localeNames
+    localeNames: localeNames,
   };
   let page = "info";
   options = buildOptions(page, options);
@@ -579,4 +574,3 @@ if (
 }
 
 cron.schedule("0 30 2 * * *", getFilesJob);
-