further tweaks

config/locales/en.json

@@ -36,7 +36,10 @@
     "filename": "Filename:",
     "release_group": "Release group:",
     "upload_date": "Upload date:",
-    "more_info": "More Info"
+    "more_info": "More Info",
+    "size": "Size:",
+    "old_experience": "Using the old search experience.",
+    "new_experience": "Using the new search experience."
   },
   "about": {
     "title": "About",

lib/dboptimize.js

@@ -35,10 +35,15 @@ export async function optimizeDatabaseKws() {
   let optimizeTasks = [];
   let resolvedTasks = [];
   for (let i = 0; i < dbLength; ) {
-    singleLineStatus(`Optimizing Keywords: ${i} / ${dbLength}`);
+    singleLineStatus(
+      `Optimizing Keywords: ${i} / ${dbLength} ${((i / dbLength) * 100).toFixed(
+        2
+      )} (${proctime.elapsed()})}`
+    );
     let result = await File.findAndCountAll({
       limit: BATCH_SIZE,
       offset: i,
+      order: ["id"],
     });
     for (let x = 0; x < result.rows.length; x++) {
       debugPrint(`Submitting job for: ${result.rows[x]["filename"]}`);

lib/metadatasearch.js

@@ -22,6 +22,8 @@ import { singleLineStatus } from "./debugprint.js";
 import { Timer } from "./time.js";
 import { readFileSync } from "fs";
 import { dirname, resolve } from "path";
+import { Model } from "sequelize";
+import { Console } from "console";
 
 export default class MetadataSearch {
   constructor() {
@@ -55,9 +57,7 @@ export default class MetadataSearch {
     "videos.video_id",
   ];
 
-  getPlatformMapping() {
-    
-  }
+  getPlatformMapping() {}
 
   async setupClient() {
     try {
@@ -116,17 +116,47 @@ export default class MetadataSearch {
     let games = await File.findAndCountAll({
       where: {
         nongame: false,
+        detailsId: null
       },
       limit: 1000,
+      order: ["id"],
     });
-    for (let x in games) {
-      let game = games[x];
-      let metadata = await Metadata.searchByText(this.normalizeName(game.filename), game.category);
-      if (metadata) {
-        await game.setDetails(metadata);
-        await metadata.addFile(game);
+    let count = games.count;
+    let pages = Math.ceil(games.count / 1000);
+    let timer = new Timer();
+    let found = 0;
+    console.log(`Matching ${count} games to metadata.`);
+    for (let x = 0; x < pages; x++) {
+      games = await File.findAndCountAll({
+        where: {
+          nongame: false,
+          detailsId: null
+        },
+        limit: 1000,
+        offset: x * 1000,
+        include: { model: Metadata, as: "details" },
+      });
+      for (let y = 0; y < games.rows.length; y++) {
+        singleLineStatus(
+          `Matching metadata: ${x * 1000 + y} / ${count} ${(
+            ((x * 1000 + y) / count) *
+            100
+          ).toFixed(2)}% (${timer.elapsed()}) Total Matches: ${found}`
+        );
+        let game = games.rows[y];
+        let metadata = await Metadata.searchByText(
+          this.normalizeName(game.filename),
+          game.category
+        );
+        if (metadata.length) {
+          let md = await Metadata.findByPk(metadata[0].id);
+          await game.setDetails(md);
+          await md.addFile(game);
+          found++;
+        }
       }
     }
+    console.log(`Completed matching metadata to files in ${timer.elapsed()}`)
   }
 
   async syncAllMetadata(retrying = false) {
@@ -179,6 +209,8 @@ export default class MetadataSearch {
         }
         retryCount = 0;
       }
+      console.log(`Finished syncing metadata in ${timer.elapsed()}`);
+      this.matchAllMetadata()
     } catch (error) {
       if (error.code === "ERR_BAD_REQUEST" && !retrying) {
         this.setupClient();
@@ -197,9 +229,7 @@ export default class MetadataSearch {
             id: metadata.id,
           },
           {
-            returning: true,
-            updateOnDuplicate: ["id"],
-            include: File,
+            include: File
           }
         );
       }
@@ -213,7 +243,9 @@ export default class MetadataSearch {
         : null;
       md.genre = metadata.genres?.map((genre) => genre.name);
       md.gamemodes = metadata.game_modes?.map((gm) => gm.name);
-      md.platforms = metadata.platforms?.map((platform) => this.platformMap[platform.name] || platform.name);
+      md.platforms = metadata.platforms?.map(
+        (platform) => this.platformMap[platform.name] || platform.name
+      );
       md.screenshots = metadata.screenshots?.map((ss) => ss.image_id);
       md.videos = metadata.videos?.map((v) => v.video_id);
       md.developers = metadata.involved_companies
@@ -240,7 +272,7 @@ export default class MetadataSearch {
         );
       }
       //this needs to remain json as we want the keys to be retained
-      md.alternatetiles = JSON.stringify(alternates);
+      md.alternatetitles = alternates.length ? JSON.stringify(alternates) : null;
       await md.save();
       if (game) {
         await game.setDetails(md);

lib/models/file.js

@@ -51,11 +51,6 @@ export default function (sequelize) {
     group: {
       type: DataTypes.TEXT
     },
-    blockmetadata: {
-      type: DataTypes.BOOLEAN,
-      defaultValue: false,
-      allowNull: false
-    },
     nongame: {
       type: DataTypes.BOOLEAN,
       defaultValue: false,

lib/models/metadata.js

@@ -14,7 +14,7 @@ export default function (sequelize) {
         allowNull: false,
       },
       alternatetitles: {
-        type: DataTypes.STRING(1024),
+        type: DataTypes.STRING(4096),
       },
       description: {
         type: DataTypes.STRING(16384),
@@ -68,11 +68,15 @@ export default function (sequelize) {
 
   Metadata.beforeSave("addVector", async (instance) => {
     const title = instance.title || "";
+    const alternateTitles =
+      JSON.parse(instance.alternatetitles || "[]")
+        .map((title) => title.name)
+        .join(", ") || "";
     const query = `
-                SELECT to_tsvector('english', $1)
+                SELECT to_tsvector('english', $1 || ', ' || $2)
             `;
     const [results] = await sequelize.query(query, {
-      bind: [title],
+      bind: [title, alternateTitles],
       raw: true,
     });
     instance.searchVector = results[0].to_tsvector;
@@ -81,22 +85,18 @@ export default function (sequelize) {
   // Add a class method for full-text search
   Metadata.searchByText = async function (searchQuery, platform, limit = 1) {
     let platformClause = "";
-    let limitClause = `limit ${limit}`;
+    let limitClause = `LIMIT ${limit}`;
     if (platform) {
       platformClause = `AND '${platform}' = ANY(platforms)`;
     }
     const query = `
-            SELECT * FROM "Metadata"
-            WHERE "searchVector" @@ plainto_tsquery('english', :search) :platformClause
-            ORDER BY ts_rank("searchVector", plainto_tsquery('english', :search)) DESC :limit
+            SELECT id FROM "Metadata"
+            WHERE "searchVector" @@ plainto_tsquery('english', $1) ${platformClause}
+            ORDER BY ts_rank("searchVector", plainto_tsquery('english', $1 )) ${limitClause}
         `;
     return await sequelize.query(query, {
       model: Metadata,
-      replacements: {
-        search: searchQuery,
-        platformClause: platformClause,
-        limit: limitClause,
-      },
+      bind: [searchQuery],
       type: sequelize.QueryTypes.SELECT,
     });
   };

lib/services/elasticsearch.js

@@ -168,7 +168,8 @@ export async function search(query, options) {
     // Fetch full records from PostgreSQL for the search results
     const ids = response.hits.hits.map((hit) => hit._id);
     const fullRecords = await File.findAll({
-      where: { id: ids }
+      where: { id: ids },
+      include: {model: Metadata, as: "details"},
     });
 
     // Create a map of full records by id
@@ -192,7 +193,6 @@ export async function search(query, options) {
     const elapsed = timer.elapsedSeconds();
     return {
       items: results,
-      db: fullRecords,
       count: response.hits.total.value || 0,
       elapsed,
     };

server.js

@@ -1,4 +1,5 @@
 import getAllFiles from "./lib/dircrawl.js";
+import { optimizeDatabaseKws } from "./dboptimize.js";
 import FileHandler from "./lib/filehandler.js";
 import Searcher from "./lib/search.js";
 import cron from "node-cron";
@@ -16,11 +17,10 @@ import {
   isNonGameContent,
 } from "./lib/emulatorConfig.js";
 import fetch from "node-fetch";
-import { initDB, File, QueryCount } from "./lib/database.js";
+import { initDB, File, QueryCount, Metadata } from "./lib/database.js";
 import { initElasticsearch } from "./lib/services/elasticsearch.js";
 import i18n, { locales } from "./config/i18n.js";
 import { v4 as uuidv4 } from "uuid";
-import { optimizeDatabaseKws } from "./lib/dboptimize.js";
 import MetadataSearch from "./lib/metadatasearch.js";
 import Flag from "./lib/flag.js";
 import ConsoleIcons from "./lib/consoleicons.js";
@@ -81,6 +81,10 @@ async function getFilesJob() {
   }
   crawlTime = Date.now();
   console.log(`Finished updating file list. ${fileCount} found.`);
+  if(Metadata.count() > metadataSearch.getIGDBGamesCount()){
+    await metadataSearch.syncAllMetadata();
+  }
+  optimizeDatabaseKws();
 }
 
 function buildOptions(page, options) {
@@ -205,16 +209,12 @@ app.get("/search", async function (req, res) {
     delete settings.combineWith;
   }
   let loadOldResults =
-    req.query.old === "true" || !metadataSearch.authorized ? true : false;
+    req.query.old === "true" || !Metadata.count() ? true : false;
   settings.pageSize = loadOldResults ? 100 : 10;
   settings.page = pageNum - 1;
   settings.sort = req.query.o || "";
   let results = await search.findAllMatches(query, settings);
   debugPrint(results);
-  let metas = [];
-  if (!loadOldResults) {
-    metas = await metadataSearch.queueGetGamesMetadata(results.db);
-  }
   if (results.count && pageNum == 1) {
     queryCount += 1;
     await QueryCount.update({ count: queryCount }, { where: { id: 1 } });
@@ -222,7 +222,7 @@ app.get("/search", async function (req, res) {
   }
   let options = {
     query: query,
-    results: metas?.length ? metas : results.items,
+    results: results.items,
     count: results.count,
     elapsed: results.elapsed,
     pageNum: pageNum,
@@ -266,7 +266,7 @@ app.get("/lucky", async function (req, res) {
 app.get("/settings", function (req, res) {
   let options = { defaultSettings: defaultSettings };
   let page = "settings";
-  options.oldSettingsAvailable = metadataSearch.authorized;
+  options.oldSettingsAvailable = Metadata.count() ? true : false;
   options = buildOptions(page, options);
   res.render(indexPage, options);
 });

views/pages/info.ejs

@@ -78,6 +78,9 @@
           <div>
             <p><span class="info"><%= __('search.filename') %></span> <%= file.filename %></p>
           </div>
+          <div>
+            <p><span class="info"><%= __('search.size') %></span> <%= file.size %></p>
+          </div>
           <div>
             <p><span class="info"><%= __('search.upload_date') %></span> <%= file.date %></p>
           </div>

views/partials/result.ejs

@@ -22,7 +22,7 @@
     <p class="description"><%= __('search.no_metadata') %></p>
     <% } %>
     <% if(metadata.title) {%>
-    <p class="file"><%= __('search.filename') %> <%= file.filename %> | <%= __('search.upload_date')%> <%= file.date %></p>
+    <p class="file"><%= __('search.filename') %> <%= file.filename %> | <%= __('search.size')%> <%= file.size %> | <%= __('search.upload_date')%> <%= file.date %></p>
     <% } %>
     <p class="group"><%= __('search.release_group') %> <%= file.group %></p>
     <p class="actions">