From 0457533f76ce67dff6c98be0d040af3d8be58952 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 30 May 2025 04:26:40 -0600 Subject: [PATCH] fix suggestions fix info title page remove carousel for games without applicable metadata --- lib/dboptimize.js | 2 +- lib/dircrawl.js | 4 ++-- lib/metadatasearch.js | 16 +++++++------ lib/services/elasticsearch.js | 43 +++++++++++++++++++++++++++-------- server.js | 12 +++++----- views/pages/info.ejs | 4 +++- 6 files changed, 54 insertions(+), 27 deletions(-) diff --git a/lib/dboptimize.js b/lib/dboptimize.js index a603b8f..0ab9d3c 100644 --- a/lib/dboptimize.js +++ b/lib/dboptimize.js @@ -38,7 +38,7 @@ export async function optimizeDatabaseKws() { singleLineStatus( `Optimizing Keywords: ${i} / ${dbLength} ${((i / dbLength) * 100).toFixed( 2 - )} (${proctime.elapsed()})}` + )}% (${proctime.elapsed()})` ); let result = await File.findAndCountAll({ limit: BATCH_SIZE, diff --git a/lib/dircrawl.js b/lib/dircrawl.js index 4939cc8..7381b7a 100644 --- a/lib/dircrawl.js +++ b/lib/dircrawl.js @@ -95,7 +95,7 @@ export default async function getAllFiles(catList) { fetchTasks = []; dirStatus = `Directories Remaining: ${ dirs.length - }, Files Found: ${fileCount} (${proctime.elapsed()}`; + }, Files Found: ${fileCount} (${proctime.elapsed()})`; } if (dirs.length == 0 && parseTasks.length > 0) { @@ -125,7 +125,7 @@ export default async function getAllFiles(catList) { parseTasks = []; dirStatus = `Directories Remaining: ${ dirs.length - }, Files Found: ${fileCount} (${proctime.elapsed()}`; + }, Files Found: ${fileCount} (${proctime.elapsed()})`; } if (dirStatus) { diff --git a/lib/metadatasearch.js b/lib/metadatasearch.js index 12563e9..399d20a 100644 --- a/lib/metadatasearch.js +++ b/lib/metadatasearch.js @@ -55,8 +55,6 @@ export default class MetadataSearch { "videos.video_id", ]; - getPlatformMapping() {} - async setupClient() { try { if (this.twitchSecrets.client_id && this.twitchSecrets.client_secret) { @@ -66,12 +64,14 @@ export default class MetadataSearch { this.platformMap = JSON.parse(readFileSync(mapFilePath, "utf8")); if (this.accessToken) { this.authorized = true; - this.syncAllMetadata() + this.ready = true; return; } } + this.ready = true; this.authorized = false; //disable } catch (error) { + this.ready = true; this.authorized = false; } } @@ -89,7 +89,11 @@ export default class MetadataSearch { async getIGDBGamesCount(retrying = false) { try { - if (!this.authorized) return 0; + // hack to ensure the client is ready before we do anything + while(!this.ready){ + await this.sleep(500) + } + if (this.authorized === false) return 0; const { data } = await this.client .request("games/count") .pipe( @@ -300,9 +304,7 @@ export default class MetadataSearch { } } //this needs to remain json as we want the keys to be retained - md.alternatetitles = alternates - ? JSON.stringify(alternates) - : null; + md.alternatetitles = alternates ? JSON.stringify(alternates) : null; await md.save(); } catch (error) { console.error("Error adding metadata:", error); diff --git a/lib/services/elasticsearch.js b/lib/services/elasticsearch.js index fe58409..aaa456a 100644 --- a/lib/services/elasticsearch.js +++ b/lib/services/elasticsearch.js @@ -250,21 +250,33 @@ export async function getSuggestions(query, options) { index: INDEX_NAME, body: { query: { - multi_match: { - query, - fields: ["filename^2", "filenamekws^2", "category", "categorykws"], - fuzziness: "AUTO", - type: "best_fields", - }, + bool:{ + must: { + multi_match: { + query, + fields: ["filename^2", "filenamekws^2", "category", "categorykws"], + fuzziness: "AUTO", + type: "best_fields", + }, + }, + filter: { + term:{ + nongame: false + } + } + } }, _source: ["filename", "category"], - size: 10, + size: 30, }, }); - return response.hits.hits.map((hit) => ({ - suggestion: hit._source.filename, - })); + let suggestions = response.hits.hits.map((hit) => + normalizeName(hit._source.filename), + ); + return [...new Set(suggestions)].map(suggestion => ({ + suggestion: suggestion + })) } catch (error) { console.error("Suggestion error:", error); return []; @@ -294,3 +306,14 @@ export async function getSample(query, options) { return []; } } + +function normalizeName(filename) { + if (!filename) return; + return filename + .replace( + /\.[A-z]{3,3}|\.|&|-|\+|,|v[0-9]+\.[0-9]+|\[.*?\]|\(.*?\)|the|usa/gi, + "" + ) + .replace(/\s{2,}/g, " ") + .trim(); + } \ No newline at end of file diff --git a/server.js b/server.js index eddf250..87c5120 100644 --- a/server.js +++ b/server.js @@ -83,10 +83,10 @@ async function getFilesJob() { } crawlTime = Date.now(); console.log(`Finished updating file list. ${fileCount} found.`); - if(Metadata.count() > metadataSearch.getIGDBGamesCount()){ + if(await Metadata.count() < await metadataSearch.getIGDBGamesCount()){ await metadataSearch.syncAllMetadata(); } - optimizeDatabaseKws(); + await optimizeDatabaseKws(); //this is less important and needs to run last. metadataSearch.matchAllMetadata(true) } @@ -213,11 +213,11 @@ app.get("/search", async function (req, res) { delete settings.combineWith; } let loadOldResults = - req.query.old === "true" || !Metadata.count() ? true : false; + req.query.old === "true" || !await Metadata.count() ? true : false; settings.pageSize = loadOldResults ? 100 : 10; settings.page = pageNum - 1; settings.sort = req.query.o || ""; - let results = await search.findAllMatches(query, settings); + let results = await search.findAllMatches(query.trim(), settings); debugPrint(results); if (results.count && pageNum == 1) { queryCount += 1; @@ -268,10 +268,10 @@ app.get("/lucky", async function (req, res) { updateDefaults(); }); -app.get("/settings", function (req, res) { +app.get("/settings", async function (req, res) { let options = { defaultSettings: defaultSettings }; let page = "settings"; - options.oldSettingsAvailable = Metadata.count() ? true : false; + options.oldSettingsAvailable = await Metadata.count() ? true : false; options = buildOptions(page, options); res.render(indexPage, options); }); diff --git a/views/pages/info.ejs b/views/pages/info.ejs index 165f190..20cf37c 100644 --- a/views/pages/info.ejs +++ b/views/pages/info.ejs @@ -24,7 +24,7 @@
-

<%= title %>

+

<%= title || file.filename %>

<%= file.category %> <%- consoleIcons.createConsoleImage(file.category) %>

@@ -107,9 +107,11 @@
+ <% if(images.length || videos.length){ %>
<%- include("../partials/carousel", {images: images, videos: videos})%>
+ <% } %>