reorganize project, fix a couple of miscellaneous issues.

lib/crawler/dircrawl.js

@@ -1,13 +1,13 @@
-import { getTableRows, parseOutFile } from "./fileworker.js";
+import { getTableRows, parseOutFile } from "./workers/fileworker.js";
 import { Piscina, FixedQueue } from "piscina";
 import { resolve } from "path";
-import debugPrint from "./debugprint.js";
-import { File } from "./models/index.js";
-import { bulkIndexFiles } from "./services/elasticsearch.js";
-import { Timer } from "./time.js";
+import debugPrint from "../utility/printutils.js";
+import { File } from "../database/models/index.js";
+import { bulkIndexFiles } from "../services/elasticsearch.js";
+import { Timer } from "../utility/time.js";
 
 let piscina = new Piscina({
-  filename: resolve("./lib", "fileworker.js"),
+  filename: resolve("./lib/crawler/workers/", "fileworker.js"),
   taskQueue: new FixedQueue(),
 });
 

lib/crawler/metadatamanager.js

@@ -16,13 +16,13 @@ import {
   limit,
   offset,
 } from "@phalcode/ts-igdb-client";
-import { File, Metadata } from "./database.js";
-import TaskQueue from "./taskqueue.js";
-import { singleLineStatus } from "./debugprint.js";
-import { Timer } from "./time.js";
+import { File, Metadata } from "../database/database.js";
+import TaskQueue from "../utility/taskqueue.js";
+import { singleLineStatus } from "../utility/printutils.js";
+import { Timer } from "../utility/time.js";
 import { readFileSync } from "fs";
 
-export default class MetadataSearch {
+export default class MetadataManager {
   constructor() {
     this.twitchSecrets = {
       client_id: process.env.TWITCH_CLIENT_ID,

lib/crawler/workers/fileworker.js

@@ -181,7 +181,7 @@ function getNonGameTerms() {
   const __filename = fileURLToPath(import.meta.url);
   const __dirname = dirname(__filename);
 
-  const nonGameTermsPath = resolve(__dirname, 'nonGameTerms.json');
+  const nonGameTermsPath = resolve(__dirname, '../../json/terms/nonGameTerms.json');
   nonGameTermsCache = JSON.parse(readFileSync(nonGameTermsPath, 'utf8'));
 
   return nonGameTermsCache;

lib/database/database.js

@@ -123,8 +123,11 @@ export async function initDB() {
 
     // Only force sync if explicitly requested
     if (process.env.FORCE_FILE_REBUILD === "1") {
+      let queryCount = (await QueryCount.findOne())?.count || 0;
       await sequelize.sync({ force: true });
       await enableTrigram();
+      //restore query count after force sync
+      await QueryCount.create({ count: queryCount });
       console.log("DB forcefully synchronized.");
     }
   } catch (error) {

lib/database/dboptimize.js

@@ -1,21 +1,21 @@
-import debugPrint from "./debugprint.js";
-import { bulkIndexFiles } from "./services/elasticsearch.js";
+import debugPrint from "../utility/printutils.js";
+import { bulkIndexFiles } from "../services/elasticsearch.js";
 import { File } from "./models/index.js";
 import { readFileSync } from "fs";
 import { fileURLToPath } from "url";
 import { dirname, resolve } from "path";
 import { Piscina, FixedQueue } from "piscina";
-import { Timer } from "./time.js";
+import { Timer } from "../utility/time.js";
 
 let piscina = new Piscina({
-  filename: resolve("./lib", "dbkwworker.js"),
+  filename: resolve("./lib/database/workers", "dbkwworker.js"),
   taskQueue: new FixedQueue(),
 });
 
-const BATCH_SIZE = 1000;
+const BATCH_SIZE = 100;
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
-const relatedKwRoot = "../lib/json/relatedkeywords/";
+const relatedKwRoot = "../json/relatedkeywords/";
 const catKwPath = resolve(__dirname, relatedKwRoot + "categories.json");
 const nameKwpath = resolve(__dirname, relatedKwRoot + "names.json");
 const regionKwpath = resolve(__dirname, relatedKwRoot + "regions.json");
@@ -57,7 +57,7 @@ export async function optimizeDatabaseKws() {
     singleLineStatus(
       `Optimizing Keywords: ${i} / ${dbLength} ${((i / dbLength) * 100).toFixed(
         2
-      )}% (${proctime.elapsed()})`
+      )}% (${proctime.elapsed()}) Optimized Rows: ${changes}`
     );
 
     for (let x = i; x < currentIndex + BATCH_SIZE; x++) {
@@ -98,7 +98,7 @@ export async function optimizeDatabaseKws() {
         changed = true;
       }
       if (changed) {
-        result[promiseIndex].save();
+        await result[promiseIndex].save();
         changes++;
       }
       promiseIndex++;

lib/database/workers/dbkwworker.js

@@ -1,5 +1,5 @@
 import { ToWords } from "to-words";
-import { getSample } from "./services/elasticsearch.js";
+import { getSample } from "../../services/elasticsearch.js";
 
 const toWords = new ToWords({
   localeCode: "en-US",

lib/emulator/emulatorConfig.js

@@ -1,4 +1,4 @@
-import debugPrint from './debugprint.js'
+import debugPrint from '../utility/printutils.js'
 
 // See https://emulatorjs.org/docs/systems for available cores
 const systemConfigs = {

lib/images/flag.js

@@ -3,7 +3,7 @@ import { fileURLToPath } from "url";
 import fs from "fs";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const flagsDir = path.join(__dirname, "../views/public/images/flags");
+const flagsDir = path.join(__dirname, "../../views/public/images/flags");
 
 
 export default class Flags {

lib/search/search.js

@@ -1,9 +1,9 @@
-import debugPrint from "./debugprint.js";
+import debugPrint from "../utility/printutils.js";
 import {
   search as elasticSearch,
   getSuggestions as elasticSuggestions,
-} from "./services/elasticsearch.js";
-import { File, Metadata } from "./models/index.js";
+} from "../services/elasticsearch.js";
+import { File, Metadata } from "../database/models/index.js";
 
 export default class Searcher {
   constructor(fields) {

lib/services/elasticsearch.js

@@ -1,7 +1,7 @@
 import { Client } from "@elastic/elasticsearch";
-import debugPrint from "../debugprint.js";
-import { File, Metadata } from "../models/index.js";
-import { Timer } from "../time.js";
+import debugPrint from "../utility/printutils.js";
+import { File, Metadata } from "../database/models/index.js";
+import { Timer } from "../utility/time.js";
 
 const client = new Client({
   node: process.env.ELASTICSEARCH_URL || "http://localhost:9200",