it works but it's a mess

lib/database.js

@@ -18,7 +18,7 @@ export const File = defineFile(sequelize);
 export const QueryCount = defineQueryCount(sequelize);
 export const Metadata = defineMetadata(sequelize)
 Metadata.hasMany(File)
-File.belongsTo(Metadata)
+File.belongsTo(Metadata, {as: "details"})
 
 export async function initDB() {
   try {

lib/dboptimize.js

@@ -5,7 +5,7 @@ import { readFileSync } from "fs";
 import { fileURLToPath } from "url";
 import { dirname, resolve } from "path";
 import { Piscina, FixedQueue } from "piscina";
-import { timer } from "./time.js";
+import { Timer } from "./time.js";
 
 let piscina = new Piscina({
   filename: resolve("./lib", "dbkwworker.js"),
@@ -28,7 +28,7 @@ const keywords = {
 };
 
 export async function optimizeDatabaseKws() {
-  let proctime = new timer();
+  let proctime = new Timer();
   let changes = 0;
   console.log("Optimizing DB Keywords...");
   let dbLength = await File.count();

lib/dircrawl.js

@@ -5,7 +5,7 @@ import debugPrint from "./debugprint.js";
 import { File } from './models/index.js';
 import { bulkIndexFiles } from './services/elasticsearch.js';
 import { optimizeDatabaseKws } from "./dboptimize.js";
-import { timer } from "./time.js";
+import { Timer } from "./time.js";
 
 let piscina = new Piscina({
   filename: resolve("./lib", "fileworker.js"),
@@ -15,7 +15,7 @@ let piscina = new Piscina({
 const BATCH_SIZE = 1000; // Process files in batches for better performance
 
 export default async function getAllFiles(catList) {
-  var proctime = new timer()
+  var proctime = new Timer()
   const url = "https://myrient.erista.me/files/";
   let parentRows = await getTableRows({ url: url, base: "" });
   let parents = [];
@@ -161,7 +161,8 @@ async function processBatch(files) {
           type: file.type,
           date: file.date,
           region: file.region,
-          group: file.group
+          group: file.group,
+          nongame: file.nongame
         })),
         {
           returning: true,

lib/fileworker.js

@@ -1,5 +1,8 @@
 import innertext from "innertext";
 import HTMLParse from "node-html-parser";
+import { fileURLToPath } from 'url';
+import { dirname, resolve } from "path";
+import { readFileSync } from "fs";
 
 export async function getTableRows(data) {
   let retryLeft = 5;
@@ -55,7 +58,8 @@ export async function parseOutFile(data) {
     type: findType(fullName, data.catList),
     date: innertext(file.querySelector(".date").innerHTML).trim(),
     region: findRegion(fullName, data.catList),
-    group: findGroup(fullName)
+    group: findGroup(fullName),
+    nongame: checkNonGame(name)
   };
   return processedFile;
 }
@@ -166,6 +170,29 @@ function findGroup(str){
   }
 }
 
+// Cache for nonGameTerms
+let nonGameTermsCache = null;
+
+function getNonGameTerms() {
+  if (nonGameTermsCache) {
+    return nonGameTermsCache;
+  }
+
+  const __filename = fileURLToPath(import.meta.url);
+  const __dirname = dirname(__filename);
+
+  const nonGameTermsPath = resolve(__dirname, 'nonGameTerms.json');
+  nonGameTermsCache = JSON.parse(readFileSync(nonGameTermsPath, 'utf8'));
+
+  return nonGameTermsCache;
+}
+
+function checkNonGame(str){
+  const nonGameTerms = getNonGameTerms();
+  const termPatterns = nonGameTerms.terms.map(term => new RegExp(term, 'i'));
+  return termPatterns.some(pattern => pattern.test(str));
+}
+
 class HTTPResponseError extends Error {
   constructor(response) {
     super(`HTTP Error Response: ${response.status} ${response.statusText}`);

lib/metadatasearch.js

@@ -0,0 +1,223 @@
+import {
+  twitchAccessToken,
+  igdb,
+  request,
+  multi,
+} from "@phalcode/ts-igdb-client";
+import {
+  fields,
+  or,
+  and,
+  where,
+  whereIn,
+  WhereFlags,
+  WhereInFlags,
+  sort,
+  limit,
+  offset,
+} from "@phalcode/ts-igdb-client";
+import { File, Metadata } from "./database.js";
+import { Sequelize } from "sequelize";
+
+export default class MetadataSearch {
+  constructor() {
+    this.twitchSecrets = {
+      client_id: process.env.TWITCH_CLIENT_ID,
+      client_secret: process.env.TWITCH_CLIENT_SECRET,
+    };
+    this.setupClient();
+  }
+  gameFields = [
+    "name",
+    "alternative_names.comment",
+    "alternative_names.name",
+    "cover.image_id",
+    "total_rating",
+    "first_release_date",
+    "summary",
+    "genres.name",
+    "involved_companies.company.name",
+    "involved_companies.developer",
+    "involved_companies.publisher",
+    "involved_companies.supporting",
+    "game_modes.name",
+    "game_localizations.name",
+    "game_localizations.region",
+    "game_localizations.region.name",
+    "platforms.name",
+    "game_type.type",
+  ];
+
+  async setupClient() {
+    try {
+      if (this.twitchSecrets.client_id && this.twitchSecrets.client_secret) {
+        this.accessToken = await twitchAccessToken(this.twitchSecrets);
+        this.client = igdb(this.twitchSecrets.client_id, this.accessToken);
+        if (this.accessToken) {
+          this.authorized = true;
+          return;
+        }
+      }
+      this.authorized = false; //disable
+    } catch (error) {
+      this.authorized = false;
+    }
+  }
+
+  async getMetadata(query, retrying = false) {
+    try {
+      if (!this.authorized) return;
+      const { data } = await this.client
+        .multi(...this.buildGameMultiQuery(query))
+        .execute();
+      return data;
+    } catch (error) {
+      if (error === "ERR_BAD_REQUEST" && !retrying) {
+        this.setupClient();
+        return this.getMetadata(query, true);
+      }
+      console.error("Failed to retrieve metadata:", error);
+    }
+  }
+
+  buildGameMultiQuery(query) {
+    let multiQuery = [];
+    for (let x in query) {
+      multiQuery.push(
+        request("games")
+          .alias(x)
+          .pipe(
+            fields(this.gameFields),
+            and(
+              ...this.buildAndClauses("name", "~", query[x].name),
+              where("game_type.type", "=", "Main Game"),
+              where("platforms.name", "~", query[x].platform)
+            ),
+            limit(1)
+          )
+      );
+    }
+    return multiQuery;
+  }
+
+  buildAndClauses(field, op, string) {
+    let andClauses = [];
+    let name = [...new Set(string.split(" "))].filter((n) => n); //dedupe;
+    for (let x in name) {
+      andClauses.push(where(field, op, name[x], WhereFlags.CONTAINS));
+    }
+    return andClauses;
+  }
+
+  normalizeName(filename) {
+    if (!filename) return;
+    return filename
+      .replace(/\.[A-z]{3,3}|\.|&|-|,|v[0-9]+\.[0-9]+|\[.*?\]|\(.*?\)/g, "")
+      .trim();
+  }
+
+  async getGamesMetadata(games) {
+    try {
+      if (!this.authorized || !games.length) return [];
+      let gameQuery = [];
+      for (let x in games) {
+        if (!(await games[x].getDetails()))
+          if (!games[x].nongame) {
+            if (!games[x].blockmetadata) {
+              gameQuery.push({
+                name: this.normalizeName(games[x].filename),
+                platform: games[x].category,
+                id: x,
+              });
+            }
+          }
+      }
+      if (!gameQuery.length) return [];
+      let gameMetas = await this.getMetadata(gameQuery);
+      if (!gameMetas.length) return [];
+      for (let x in gameMetas) {
+        if (gameMetas[x].result.length) {
+          await this.addMetadataToDb(
+            gameMetas[x].result[0],
+            games[gameQuery[x].id]
+          );
+        }
+      }
+      let details = await Promise.all(games.map((game) => game.getDetails()));
+      return details.map((details) => details?.dataValues);
+    } catch (error) {
+      console.error("Error getting metadata:", error);
+    }
+  }
+
+  async addMetadataToDb(metadata, game) {
+    try {
+      let md = await Metadata.findOne({
+        where: {
+          id: metadata.id,
+        },
+      });
+      if (!md) {
+        md = await Metadata.build(
+          {
+            id: metadata.id,
+            title: metadata.name,
+
+            description: metadata.summary,
+            rating: metadata.total_rating,
+            coverartid: metadata.cover?.image_id,
+            releasedate: metadata.first_release_date
+              ? new Date(metadata.first_release_date * 1000)
+              : null,
+            genre: JSON.stringify(metadata.genres?.map((genre) => genre.name)),
+            gamemodes: JSON.stringify(
+              metadata.game_modes?.map((gm) => gm.name)
+            ),
+            platforms: JSON.stringify(
+              metadata.platforms?.map((platform) => platform.name)
+            ),
+          },
+          {
+            returning: true,
+            updateOnDuplicate: ["id"],
+            include: File,
+          }
+        );
+      }
+      //these don't work right unless I do them after the fact.
+      md.developers = JSON.stringify(
+        metadata.involved_companies
+          ?.filter((ic) => ic.developer)
+          ?.map((ic) => ic.company.name)
+      );
+      md.publishers = JSON.stringify(
+        metadata.involved_companies
+          ?.filter((ic) => ic.publisher)
+          ?.map((ic) => ic.company.name)
+      );
+      let alternates = [];
+      if (metadata.alternative_names) {
+        alternates.push(
+          metadata.alternative_names.map((an) => ({
+            type: an.comment,
+            name: an.name,
+          }))
+        );
+      }
+      if (metadata.game_localizations) {
+        alternates.push(
+          metadata.game_localizations.map((gn) => ({
+            type: gn.region.name,
+            name: gn.name,
+          }))
+        );
+      }
+      md.alternatetiles = JSON.stringify(alternates);
+      await md.save();
+      await game.setDetails(md);
+      await md.addFile(game);
+    } catch (error) {
+      console.error("Error adding metadata:", error);
+    }
+  }
+}

lib/metadataworker.js

@@ -1,113 +0,0 @@
-import { twitchAccessToken, igdb, request } from "@phalcode/ts-igdb-client";
-import {
-  fields,
-  or,
-  and,
-  where,
-  whereIn,
-  WhereFlags,
-  WhereInFlags,
-  sort,
-  limit,
-  offset,
-} from "@phalcode/ts-igdb-client";
-
-const twitchSecrets = {
-  client_id: process.env.TWITCH_CLIENT_ID,
-  client_secret: process.env.TWITCH_CLIENT_SECRET,
-};
-const accessToken = await twitchAccessToken(twitchSecrets);
-
-const client = igdb(twitchSecrets.client_id, accessToken);
-
-const gameFields = [
-  "name",
-  "alternative_names.comment",
-  "alternative_names.name",
-  "cover.image_id",
-  "total_rating",
-  "first_release_date",
-  "summary",
-  "genres.name",
-  "involved_companies.company.name",
-  "involved_companies.developer",
-  "involved_companies.publisher",
-  "involved_companies.supporting",
-  "multiplayer_modes.*",
-  "game_localizations.name",
-  "game_localizations.region",
-  "platforms.name",
-];
-
-export async function getMetadata(query) {
-  const data = await client
-    .request("games")
-    .pipe(
-      fields(gameFields),
-      or(...buildOrAndClauses("name", "~", query)),
-      sort("name", "asc")
-    )
-    .execute();
-  return data;
-}
-
-function buildOrClauses(field, op, queries) {
-  let orClauses = [];
-  for (let x in queries) {
-    orClauses.push(where(field, op, queries[x], WhereFlags.CONTAINS));
-  }
-  return orClauses;
-}
-
-function buildOrAndClauses(field, op, queries) {
-  let orClauses = [];
-
-  for (let x in queries) {
-    let name = [...new Set(queries[x].split(" "))]; //dedupe;
-    let andClauses = [];
-    for (let y in name) {
-      andClauses.push(where(field, op, name[y], WhereFlags.CONTAINS));
-    }
-    orClauses.push(and(...andClauses));
-  }
-  return orClauses;
-}
-
-function normalizeName(filename) {
-  if (!filename) return;
-  return filename
-    .replace(/\.[A-z]{3,3}|\.|&|-|,|v[0-9]+\.[0-9]+|\[.*?\]|\(.*?\)/g, "")
-    .trim();
-}
-
-function getBestMatch(filename, data) {
-  const words = filename.split(" ");
-  let bestIndex = null;
-  let bestMatchCount = 0;
-  let lengthDifference = 0;
-  for (let x in data) {
-    let matchingWords = 0;
-    for (let y in words) {
-      if (data[x].name.toLowerCase().includes(words[y].toLowerCase()))
-        matchingWords++;
-    }
-    let diff = matchingWords - dataWords.length;
-    if (matchingWords > bestMatchCount && diff < lengthDifference) {
-      bestIndex = x;
-      bestMatchCount = matchingWords;
-      lengthDifference = diff;
-      if (lengthDifference < 0) lengthDifference = 0;
-    }
-  }
-  if (bestIndex != null) {
-    return data[bestIndex];
-  }
-  return;
-}
-
-let games = await getMetadata([
-  "The Legend of Zelda A Link to the Past",
-  "Super Mario Sunshine",
-]);
-console.log(JSON.stringify(games.data, null, 2));
-//console.log(await getMetadata(games))

lib/models/file.js

@@ -55,6 +55,11 @@ export default function (sequelize) {
       type: DataTypes.BOOLEAN,
       defaultValue: false,
       allowNull: false
+    },
+    nongame: {
+      type: DataTypes.BOOLEAN,
+      defaultValue: false,
+      allowNull: false
     }
   }, {
     indexes: [

lib/models/metadata.js

@@ -2,10 +2,9 @@ import { DataTypes } from "sequelize"
 
 export default function (sequelize) {
   const Metadata = sequelize.define('Metadata', {
-        id: {
+        id: {//these will match the igdbid to make things a little easier
             type: DataTypes.INTEGER,
             primaryKey: true,
-            autoIncrement: true
         },
         title: {
             type: DataTypes.STRING,
@@ -15,23 +14,17 @@ export default function (sequelize) {
             type: DataTypes.STRING
         },
         description: {
-            type: DataTypes.STRING
+            type: DataTypes.TEXT
         },
         rating: {
             type: DataTypes.STRING
         },
-        coverarturl: {
+        coverartid: {
             type: DataTypes.STRING
         },
         releasedate: {
             type: DataTypes.DATE
         },
-        igdbid: {
-            type: DataTypes.INTEGER
-        },
-        timetobeat: {
-            type: DataTypes.STRING
-        },
         genre: {
             type: DataTypes.STRING
         },
@@ -43,11 +36,14 @@ export default function (sequelize) {
         },
         gamemodes:{
             type: DataTypes.STRING
+        },
+        platforms: {
+            type: DataTypes.STRING
         }
     }, {
     indexes: [
         { fields: ['title'] },
-        { fields: ['description'] }//If this slows down the db may want to not index this.
+        { fields: ['description'] },//If this slows down the db may want to not index this.
         ]
     })
     

lib/nonGameTerms.json

@@ -7,6 +7,7 @@
     "beta",
     "box",
     "boxart",
+    "cbr",
     "cheat",
     "config",
     "cfg",
@@ -32,6 +33,7 @@
     "mod",
     "movie",
     "music",
+    "mp4",
     "ost",
     "overlay",
     "patch",

lib/services/elasticsearch.js

@@ -1,32 +1,13 @@
-import { Client } from '@elastic/elasticsearch';
+import { Client } from "@elastic/elasticsearch";
-import debugPrint from '../debugprint.js';
+import debugPrint from "../debugprint.js";
-import { File } from '../models/index.js';
+import { File } from "../models/index.js";
-import { readFileSync } from 'fs';
+import { Timer } from "../time.js";
-import { fileURLToPath } from 'url';
-import { dirname, resolve } from 'path';
 
 const client = new Client({
-  node: process.env.ELASTICSEARCH_URL || 'http://localhost:9200'
+  node: process.env.ELASTICSEARCH_URL || "http://localhost:9200",
 });
 
-const INDEX_NAME = 'myrient_files';
+const INDEX_NAME = "myrient_files";
-
-// Cache for nonGameTerms
-let nonGameTermsCache = null;
-
-function getNonGameTerms() {
-  if (nonGameTermsCache) {
-    return nonGameTermsCache;
-  }
-
-  const __filename = fileURLToPath(import.meta.url);
-  const __dirname = dirname(__filename);
-
-  const nonGameTermsPath = resolve(__dirname, '../../lib/nonGameTerms.json');
-  nonGameTermsCache = JSON.parse(readFileSync(nonGameTermsPath, 'utf8'));
-
-  return nonGameTermsCache;
-}
 
 export async function initElasticsearch() {
   try {
@@ -40,56 +21,59 @@ export async function initElasticsearch() {
             analysis: {
               analyzer: {
                 filename_analyzer: {
-                  type: 'custom',
+                  type: "custom",
-                  tokenizer: 'standard',
+                  tokenizer: "standard",
-                  filter: ['lowercase', 'word_delimiter_graph']
+                  filter: ["lowercase", "word_delimiter_graph"],
-                }
+                },
-              }
+              },
-            }
+            },
           },
           mappings: {
             properties: {
               filename: {
-                type: 'text',
+                type: "text",
-                analyzer: 'filename_analyzer'
+                analyzer: "filename_analyzer",
               },
               category: {
-                type: 'text',
+                type: "text",
-                analyzer: 'standard',
+                analyzer: "standard",
                 fields: {
                   keyword: {
-                    type: 'keyword'
+                    type: "keyword",
-                  }
+                  },
-                }
+                },
               },
               type: {
-                type: 'text',
+                type: "text",
-                analyzer: 'standard'
+                analyzer: "standard",
               },
               region: {
-                type: 'text',
+                type: "text",
-                analyzer: 'standard'
+                analyzer: "standard",
               },
               filenamekws: {
-                type: 'text',
+                type: "text",
-                analyzer: 'standard'
+                analyzer: "standard",
               },
               categorykws: {
-                type: 'text',
+                type: "text",
-                analyzer: 'standard'
+                analyzer: "standard",
               },
               regionkws: {
-                type: 'text',
+                type: "text",
-                analyzer: 'standard'
+                analyzer: "standard",
-              }
+              },
-            }
+              nongame: {
-          }
+                type: "boolean",
-        }
+              },
+            },
+          },
+        },
       });
-      console.log('Elasticsearch index created');
+      console.log("Elasticsearch index created");
     }
   } catch (error) {
-    console.error('Elasticsearch init error:', error);
+    console.error("Elasticsearch init error:", error);
     process.exit(1);
   }
 }
@@ -99,16 +83,16 @@ export async function indexFile(file) {
     await client.index({
       index: INDEX_NAME,
       id: file.id.toString(),
-      document: file
+      document: file,
     });
     debugPrint(`Indexed file: ${file.filename}`);
   } catch (error) {
-    console.error('Error indexing file:', error);
+    console.error("Error indexing file:", error);
   }
 }
 
 export async function bulkIndexFiles(files) {
-  const operations = files.flatMap(file => [
+  const operations = files.flatMap((file) => [
     { index: { _index: INDEX_NAME, _id: file.id.toString() } },
     {
       filename: file.filename,
@@ -117,19 +101,20 @@ export async function bulkIndexFiles(files) {
       region: file.region,
       filenamekws: file.filenamekws,
       categorykws: file.categorykws,
-      regionkws: file.regionkws
+      regionkws: file.regionkws,
-    }
+      nongame: file.nongame
+    },
   ]);
 
   try {
     const { errors, items } = await client.bulk({
       refresh: true,
-      operations
+      operations,
     });
 
     if (errors) {
-      console.error('Bulk indexing had errors');
+      console.error("Bulk indexing had errors");
-      items.forEach(item => {
+      items.forEach((item) => {
         if (item.index.error) {
           console.error(item.index.error);
         }
@@ -138,46 +123,52 @@ export async function bulkIndexFiles(files) {
 
     debugPrint(`Bulk indexed ${files.length} files`);
   } catch (error) {
-    console.error('Bulk indexing error:', error);
+    console.error("Bulk indexing error:", error);
   }
 }
 
 export async function search(query, options) {
   //add kws for selected fields
-  let builtFields = []
+  let builtFields = [];
-  for(let field in options.fields){
+  for (let field in options.fields) {
-    builtFields.push(options.fields[field])
+    builtFields.push(options.fields[field]);
-    builtFields.push(options.fields[field] + 'kws')
+    builtFields.push(options.fields[field] + "kws");
   }
   const searchQuery = {
     index: INDEX_NAME,
     body: {
-      size: 1500,
+      size: options.pageSize,
+      from: options.pageSize * options.page,
       query: {
         bool: {
           must: buildMustClauses(query, options, builtFields),
-          should: buildShouldClauses(query, options, builtFields)
+          should: buildShouldClauses(query, options, builtFields),
-        }
+        },
       },
       highlight: {
         fields: {
           filename: {},
           category: {},
           type: {},
-          region: {}
+          region: {},
-        }
+        },
-      }
+      },
-    }
+    },
   };
+  if (options.hideNonGame) {
+    searchQuery.body.query.bool["filter"] = {
+      term: { nongame: false },
+    };
+  }
 
   try {
-    const startTime = process.hrtime();
+    let timer = new Timer();
     const response = await client.search(searchQuery);
 
     // Fetch full records from PostgreSQL for the search results
-    const ids = response.hits.hits.map(hit => hit._id);
+    const ids = response.hits.hits.map((hit) => hit._id);
     const fullRecords = await File.findAll({
-      where: { id: ids }
+      where: { id: ids },
     });
 
     // Create a map of full records by id
@@ -187,49 +178,44 @@ export async function search(query, options) {
     }, {});
 
     // Build results with full PostgreSQL records
-    let results = response.hits.hits.map(hit => ({
+    let results = response.hits.hits.map((hit) => ({
-      ...recordMap[hit._id].dataValues,
+      ...recordMap[hit._id]?.dataValues,
       score: hit._score,
-      highlights: hit.highlight
+      highlights: hit.highlight,
     }));
 
-    // Apply non-game content filtering in JavaScript if the option is enabled
+    //Filter out anything that couldn't be found in postgres
-    if (options.hideNonGame) {
+    results = results.filter(result => result.filename)
-      const nonGameTerms = getNonGameTerms();
-      const termPatterns = nonGameTerms.terms.map(term => new RegExp(term, 'i'));
 
-      // Filter results in JavaScript (much faster than complex Elasticsearch queries)
+    const elapsed = timer.elapsedSeconds();
-      results = results.filter(item => {
-        // Check if filename contains any of the non-game terms
-        return !termPatterns.some(pattern => pattern.test(item.filename));
-      });
-    }
-
-    const elapsed = parseHrtimeToSeconds(process.hrtime(startTime));
     return {
       items: results,
-      elapsed
+      db: fullRecords,
+      count: response.hits.total.value || 0,
+      elapsed,
     };
   } catch (error) {
-    console.error('Search error:', error);
+    console.error("Search error:", error);
-    return { items: [], elapsed: 0 };
+    return { items: [], elapsed: 0, count: 0 };
   }
 }
 
 function buildMustClauses(query, options, builtFields) {
   const clauses = [];
 
-  if (options.combineWith === 'AND') {
+  if (options.combineWith === "AND") {
-    query.split(' ').forEach(term => {
+    query.split(" ").forEach((term) => {
       clauses.push({
         multi_match: {
           query: term,
-          fields: builtFields.map(field =>
+          fields: builtFields.map((field) =>
-            field === 'filename' || 'filenamekws' ? `${field}^2` : field
+            field === "filename" || field === "filenamekws"
+              ? `${field}^2`
+              : field
           ),
           fuzziness: options.fuzzy || 0,
-          type: 'best_fields'
+          type: "best_fields",
-        }
+        },
       });
     });
   }
@@ -240,26 +226,22 @@ function buildMustClauses(query, options, builtFields) {
 function buildShouldClauses(query, options, builtFields) {
   const clauses = [];
 
-  if (options.combineWith !== 'AND') {
+  if (options.combineWith !== "AND") {
     clauses.push({
       multi_match: {
         query,
-        fields: builtFields.map(field =>
+        fields: builtFields.map((field) =>
-          field === 'filename' || 'filenamekws' ? `${field}^2` : field
+          field === "filename" || field === "filenamekws" ? `${field}^2` : field
         ),
         fuzziness: options.fuzzy || 0,
-        type: 'best_fields'
+        type: "best_fields",
-      }
+      },
     });
   }
 
   return clauses;
 }
 
-function parseHrtimeToSeconds(hrtime) {
-  return (hrtime[0] + (hrtime[1] / 1e9)).toFixed(3);
-}
-
 export async function getSuggestions(query, options) {
   try {
     const response = await client.search({
@@ -268,26 +250,26 @@ export async function getSuggestions(query, options) {
         query: {
           multi_match: {
             query,
-            fields: ['filename^2', 'filenamekws^2', 'category', 'categorykws'],
+            fields: ["filename^2", "filenamekws^2", "category", "categorykws"],
-            fuzziness: 'AUTO',
+            fuzziness: "AUTO",
-            type: 'best_fields'
+            type: "best_fields",
-          }
+          },
         },
-        _source: ['filename', 'category'],
+        _source: ["filename", "category"],
-        size: 10
+        size: 10,
-      }
+      },
     });
 
-    return response.hits.hits.map(hit => ({
+    return response.hits.hits.map((hit) => ({
-      suggestion: hit._source.filename
+      suggestion: hit._source.filename,
     }));
   } catch (error) {
-    console.error('Suggestion error:', error);
+    console.error("Suggestion error:", error);
     return [];
   }
 }
 
-export async function getSample(query, options){
+export async function getSample(query, options) {
   try {
     const response = await client.search({
       index: INDEX_NAME,
@@ -295,18 +277,18 @@ export async function getSample(query, options){
         query: {
           match: {
             filename: query,
-          }
+          },
         },
-        _source: ['filename'],
+        _source: ["filename"],
-        size: 30
+        size: 30,
-      }
+      },
     });
 
-    return response.hits.hits.map(hit => ({
+    return response.hits.hits.map((hit) => ({
-      sample: hit._source.filename
+      sample: hit._source.filename,
     }));
   } catch (error) {
-    console.error('Sample error:', error);
+    console.error("Sample error:", error);
     return [];
   }
-}
+}

lib/time.js

@@ -1,4 +1,4 @@
-export class timer {
+export class Timer {
   constructor() {
     this.startTime = process.hrtime();
   }
@@ -13,4 +13,7 @@ export class timer {
     let s = Math.floor(elapsed % 60);
     return `${h ? h + "h" : ""}${m ? m + "m" : ""}${s + "s"}`;
   }
-}
+  elapsedSeconds(){
+    return this.parseHrtimetoSeconds(process.hrtime(this.startTime));
+  }
+}

package-lock.json

@@ -24,7 +24,7 @@
         "pg-hstore": "^2.3.4",
         "piscina": "^4.7.0",
         "sanitize": "^2.1.2",
-        "sequelize": "^6.37.1",
+        "sequelize": "^6.37.7",
         "sequelize-cli": "^6.6.2",
         "to-words": "^4.5.1",
         "uuid": "^11.1.0"
@@ -3175,9 +3175,9 @@
       "license": "MIT"
     },
     "node_modules/sequelize": {
-      "version": "6.37.5",
+      "version": "6.37.7",
-      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.37.5.tgz",
+      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.37.7.tgz",
-      "integrity": "sha512-10WA4poUb3XWnUROThqL2Apq9C2NhyV1xHPMZuybNMCucDsbbFuKg51jhmyvvAUyUqCiimwTZamc3AHhMoBr2Q==",
+      "integrity": "sha512-mCnh83zuz7kQxxJirtFD7q6Huy6liPanI67BSlbzSYgVNl5eXVdE2CN1FuAeZwG1SNpGsNRCV+bJAVVnykZAFA==",
       "funding": [
         {
           "type": "opencollective",

package.json

@@ -19,7 +19,7 @@
     "pg-hstore": "^2.3.4",
     "piscina": "^4.7.0",
     "sanitize": "^2.1.2",
-    "sequelize": "^6.37.1",
+    "sequelize": "^6.37.7",
     "sequelize-cli": "^6.6.2",
     "to-words": "^4.5.1",
     "uuid": "^11.1.0"

server.js

@@ -21,6 +21,7 @@ import { initElasticsearch } from "./lib/services/elasticsearch.js";
 import i18n, { locales } from "./config/i18n.js";
 import { v4 as uuidv4 } from "uuid";
 import { optimizeDatabaseKws } from "./lib/dboptimize.js";
+import MetadataSearch from "./lib/metadatasearch.js";
 
 let categoryListPath = "./lib/categories.json";
 let nonGameTermsPath = "./lib/nonGameTerms.json";
@@ -51,6 +52,7 @@ let defaultSettings = {
   fuzzy: 0,
   prefix: true,
   hideNonGame: true,
+  useOldResults: false,
 };
 
 //programmatically set the default boosts while reducing overhead when adding another search field
@@ -64,6 +66,7 @@ for (let field in searchFields) {
 }
 
 let search = new Searcher(searchFields);
+let metadataSearch = new MetadataSearch();
 
 async function getFilesJob() {
   console.log("Updating the file list.");
@@ -106,7 +109,7 @@ app.use((req, res, next) => {
 });
 
 //static files
-app.use('/public', express.static('views/public'))
+app.use("/public", express.static("views/public"));
 
 //middleware
 app.use(sanitize.middleware);
@@ -193,22 +196,35 @@ app.get("/search", async function (req, res) {
   if (settings.combineWith != "AND") {
     delete settings.combineWith;
   }
+  settings.pageSize = settings.useOldResults ? 100 : 10;
+  settings.page = pageNum - 1;
   let results = await search.findAllMatches(query, settings);
   debugPrint(results);
-  if (results.items.length && pageNum == 1) {
+  let metas = await metadataSearch.getGamesMetadata(results.db);
+  if (results.count && pageNum == 1) {
     queryCount += 1;
     await QueryCount.update({ count: queryCount }, { where: { id: 1 } });
     updateDefaults();
   }
+  let resultOutput = [];
+  for (let x in results.items) {
+    resultOutput.push({
+      file: results.items[x],
+      metadata: metas[x] || [],
+    });
+  }
   let options = {
     query: query,
-    results: results,
+    results: resultOutput,
+    count: results.count,
+    elapsed: results.elapsed,
     pageNum: pageNum,
+    pageCount: Math.ceil(results.count / settings.pageSize),
     indexing: search.indexing,
     urlPrefix: urlPrefix,
     settings: settings,
   };
-  let page = "resultsnew";
+  let page = settings.useOldResults ? "resultsold" : "results";
   options = buildOptions(page, options);
   res.render(indexPage, options);
 });
@@ -511,4 +527,4 @@ if (
   await getFilesJob();
 }
 
-cron.schedule("0 30 2 * * *", getFilesJob);
+cron.schedule("0 30 2 * * *", getFilesJob);

views/pages/results.ejs

@@ -1,12 +1,8 @@
 <%
-    let pageCount = Math.ceil(results.items.length / 100)
     pageCount = pageCount ? pageCount : 1 //always ensure 1 page
     if(pageNum > pageCount){
         pageNum = 1
     }
-    let entryStart = Math.floor((pageNum - 1) * 100)
-    let entryEnd = entryStart + 100
-    entryEnd = entryEnd > results.items.length ? results.items.length : entryEnd
 %>
 <script src='https://code.jquery.com/jquery-3.7.1.js' crossorigin="anonymous"></script>
 <script src='https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js' crossorigin="anonymous"></script>
@@ -31,7 +27,7 @@
             <ul class="SuggestionList col-sm-12" id="suggestionList" style="width: 50%;left: 195px;"></ul>
         </div>
         <p class="m-2">
-            <%= __('search.found_plural', { count: results.items.length }) %> <%= __('search.in_seconds', { seconds: results.elapsed }) %>.
+            <%= __('search.found_plural', { count: count }) %> <%= __('search.in_seconds', { seconds: elapsed }) %>.
             <%= indexing ? __('search.indexing') : "" %>
             <% if (settings.hideNonGame) { %>
             <span class="badge badge-info" data-toggle="tooltip" data-placement="top" title="<%= __('settings.extras.hide_non_game.tooltip') %>">
@@ -43,63 +39,11 @@
     </form>
 
     <div class="col-sm-12 w-100 mt-3">
-        <p><%= __('search.displaying_results', { start: entryStart, end: entryEnd }) %></p>
+        <div>
-        <table class="table text-white table-bordered" id="results">
+            <% for (let x = 0; x < results.length; x++) { %>
-            <thead>
+            <%- include("../partials/result", {result: results[x]}) %>
-                <tr>
-                    <th class="dt-orderable"><span><%= __('results.table.name') %></span><span class="dt-column-order"></span></th>
-                    <th><%= __('results.table.group') %></th>
-                    <th><%= __('results.table.category') %></th>
-                    <th><%= __('results.table.region') %></th>
-                    <th><%= __('results.table.type') %></th>
-                    <th><%= __('results.table.size') %></th>
-                    <th><%= __('results.table.date') %></th>
-                    <th><%= __('results.table.score') %></th>
-                    <% if (process.env.EMULATOR_ENABLED === 'true') { %>
-                        <th><%= __('results.table.play') %></th>
-                    <% } %>
-                </tr>
-            </thead>
-            <% for (let x = entryStart; x < entryEnd; x++) { %>
-            <tr>
-                <td>
-                    <a href="<%= results.items[x].path %>">
-                        <%= results.items[x].filename %>
-                    </a>
-                </td>
-                <td>
-                    <%= results.items[x].group %>
-                </td>
-                <td>
-                    <%= results.items[x].category %>
-                </td>
-                <td>
-                    <%= results.items[x].region %>
-                </td>
-                <td>
-                    <%= results.items[x].type %>
-                </td>
-                <td>
-                    <%= results.items[x].size %>
-                </td>
-                <td>
-                    <%= results.items[x].date %>
-                </td>
-                <td>
-                    <%= results.items[x].score.toFixed(2) %>
-                </td>
-                <% if (process.env.EMULATOR_ENABLED === 'true') { %>
-                <td>
-                    <% if (isEmulatorCompatible(results.items[x].category)) { %>
-                        <a href="/play/<%= results.items[x].id %>" class="btn btn-sm btn-secondary"><%= __('emulator.play') %></a>
-                    <% } else { %>
-                        <button class="btn btn-sm btn-secondary" disabled><%= __('emulator.not_available') %></button>
-                    <% } %>
-                </td>
-                <% } %>
-            </tr>
             <% } %>
-        </table>
+        </div>
         <%
         if(pageCount > 1) {
         %>
@@ -150,33 +94,4 @@
         </div>
         <% } %>
     </div>
-</div>
+</div>
-<script defer>
-    /*resultTable = new DataTable('#results', {
-        "order": [[7, 'desc']],
-        "columns": [
-            { "data": "name" },      // Name
-            { "data": "group" },     // Group
-            { "data": "category" },  // Category
-            { "data": "region" },    // Region
-            { "data": "type" },      // Type
-            { "data": "size" },      // Size
-            { "data": "date" },      // Date
-            { "data": "score" },     // Search Score
-            <% if (process.env.EMULATOR_ENABLED === 'true') { %>
-                { "data": "play", "orderable": false }  // Play button column
-            <% } %>
-        ],
-        "lengthMenu": [100, { label: 'All', value: -1 }, 50, 25, 10],
-        "paging": false,
-        "filter": false,
-        "layout": {
-            "bottomStart": ''
-        }
-    });*/
-
-    // Initialize tooltips
-    $(function () {
-        $('[data-toggle="tooltip"]').tooltip()
-    });
-</script>

views/pages/resultsold.ejs

@@ -44,11 +44,62 @@
 
     <div class="col-sm-12 w-100 mt-3">
         <p><%= __('search.displaying_results', { start: entryStart, end: entryEnd }) %></p>
-        <div>
+        <table class="table text-white table-bordered" id="results">
+            <thead>
+                <tr>
+                    <th class="dt-orderable"><span><%= __('results.table.name') %></span><span class="dt-column-order"></span></th>
+                    <th><%= __('results.table.group') %></th>
+                    <th><%= __('results.table.category') %></th>
+                    <th><%= __('results.table.region') %></th>
+                    <th><%= __('results.table.type') %></th>
+                    <th><%= __('results.table.size') %></th>
+                    <th><%= __('results.table.date') %></th>
+                    <th><%= __('results.table.score') %></th>
+                    <% if (process.env.EMULATOR_ENABLED === 'true') { %>
+                        <th><%= __('results.table.play') %></th>
+                    <% } %>
+                </tr>
+            </thead>
             <% for (let x = entryStart; x < entryEnd; x++) { %>
-            <%- include("../partials/result", {result: results.items[x]}) %>
+            <tr>
+                <td>
+                    <a href="<%= results.items[x].path %>">
+                        <%= results.items[x].filename %>
+                    </a>
+                </td>
+                <td>
+                    <%= results.items[x].group %>
+                </td>
+                <td>
+                    <%= results.items[x].category %>
+                </td>
+                <td>
+                    <%= results.items[x].region %>
+                </td>
+                <td>
+                    <%= results.items[x].type %>
+                </td>
+                <td>
+                    <%= results.items[x].size %>
+                </td>
+                <td>
+                    <%= results.items[x].date %>
+                </td>
+                <td>
+                    <%= results.items[x].score.toFixed(2) %>
+                </td>
+                <% if (process.env.EMULATOR_ENABLED === 'true') { %>
+                <td>
+                    <% if (isEmulatorCompatible(results.items[x].category)) { %>
+                        <a href="/play/<%= results.items[x].id %>" class="btn btn-sm btn-secondary"><%= __('emulator.play') %></a>
+                    <% } else { %>
+                        <button class="btn btn-sm btn-secondary" disabled><%= __('emulator.not_available') %></button>
+                    <% } %>
+                </td>
+                <% } %>
+            </tr>
             <% } %>
-        </div>
+        </table>
         <%
         if(pageCount > 1) {
         %>
@@ -99,4 +150,33 @@
         </div>
         <% } %>
     </div>
-</div>
+</div>
+<script defer>
+    /*resultTable = new DataTable('#results', {
+        "order": [[7, 'desc']],
+        "columns": [
+            { "data": "name" },      // Name
+            { "data": "group" },     // Group
+            { "data": "category" },  // Category
+            { "data": "region" },    // Region
+            { "data": "type" },      // Type
+            { "data": "size" },      // Size
+            { "data": "date" },      // Date
+            { "data": "score" },     // Search Score
+            <% if (process.env.EMULATOR_ENABLED === 'true') { %>
+                { "data": "play", "orderable": false }  // Play button column
+            <% } %>
+        ],
+        "lengthMenu": [100, { label: 'All', value: -1 }, 50, 25, 10],
+        "paging": false,
+        "filter": false,
+        "layout": {
+            "bottomStart": ''
+        }
+    });*/
+
+    // Initialize tooltips
+    $(function () {
+        $('[data-toggle="tooltip"]').tooltip()
+    });
+</script>

views/partials/opengraphresults.ejs

@@ -1,9 +1,8 @@
 <%
-let resultStart = Math.floor((pageNum - 1) * 100)
+let length = results.length > 5 ? 5 : results.length
-let length = results.items.length > 5 + resultStart ? 5 + resultStart : results.items.length
 let resultString = ''
-for(let x = resultStart ; x < length; x++){
+for(let x = 0 ; x < length; x++){
-    resultString += `${x + 1}: ${results.items[x].filename}\n\n`
+    resultString += `${x + 1}: ${results[x].filename}\n\n`
 }
 resultString = resultString.trim()
 %>

views/partials/result.ejs

@@ -1,14 +1,19 @@
+<% 
+    const metadata = result.metadata || new Object()
+    const file = result.file || new Object()
+    const coverUrl = metadata.coverartid ? `/proxy-image?url=https://images.igdb.com/igdb/image/upload/t_cover_big/${metadata.coverartid}.webp` : "/public/images/coverart/nocoverart.png"
+%>
 <div class="col-md-auto row align-items-start searchresult">
     <div class="">
-        <img class="coverart" src="<%= result.image || "/public/images/coverart/nocoverart.png"%>">
+        <img class="coverart" src="<%= coverUrl %>">
     </div>
     <div class="col-md">
-        <p class="title"><%= result.title || result.filename %></p>
+        <p class="title"><%= metadata.title || file.filename %></p>
-        <p class="info"<span class="infoitem">Released: <%= result.releaseDate || result.date %></span> <span class="infoitem">Region: <%= result.region %></span></p>
+        <p class="info"<span class="infoitem">Released: <%= metadata.releasedate || file.date %></span> <span class="infoitem">Region: <%= file.region %></span> <span class="infoitem">Platform: <%= file.category %></span></p>
-        <p class="description"><%= result.description || "No description was found." %></p>
+        <p class="description"><%= metadata.description || "No description was found." %></p>
-        <% if(result.title) {%>
+        <% if(metadata.title) {%>
-        <p class="file">Filename: <%= result.filename %></p>
+        <p class="file">Filename: <%= file.filename %></p>
         <% } %>
-        <p class="actions"><button class="btn btn-sm btn-secondary" href="">More Info</button> <button class="btn btn-sm btn-secondary" href="<%= result.path %>">Download</button> <button class="btn btn-sm btn-secondary" href="/play/<%= result.id %>">Play In Browser</a></button>
+        <p class="actions"><button class="btn btn-sm btn-secondary" href="">More Info</button> <button class="btn btn-sm btn-secondary" href="<%= file.path %>">Download</button> <button class="btn btn-sm btn-secondary" href="/play/<%= file.id %>">Play In Browser</a></button>
     </div>
 </div>