From 2973211b7d6a8f15d6f9ab34aa705e4b9e903fc5 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Thu, 29 May 2025 13:07:08 -0600 Subject: [PATCH] further tweaks --- config/locales/en.json | 5 ++- lib/dboptimize.js | 7 +++- lib/metadatasearch.js | 60 +++++++++++++++++++++++++++-------- lib/models/file.js | 5 --- lib/models/metadata.js | 24 +++++++------- lib/services/elasticsearch.js | 4 +-- server.js | 18 +++++------ views/pages/info.ejs | 3 ++ views/partials/result.ejs | 2 +- 9 files changed, 83 insertions(+), 45 deletions(-) diff --git a/config/locales/en.json b/config/locales/en.json index b58161e..e86876f 100644 --- a/config/locales/en.json +++ b/config/locales/en.json @@ -36,7 +36,10 @@ "filename": "Filename:", "release_group": "Release group:", "upload_date": "Upload date:", - "more_info": "More Info" + "more_info": "More Info", + "size": "Size:", + "old_experience": "Using the old search experience.", + "new_experience": "Using the new search experience." }, "about": { "title": "About", diff --git a/lib/dboptimize.js b/lib/dboptimize.js index 4b9f764..269581a 100644 --- a/lib/dboptimize.js +++ b/lib/dboptimize.js @@ -35,10 +35,15 @@ export async function optimizeDatabaseKws() { let optimizeTasks = []; let resolvedTasks = []; for (let i = 0; i < dbLength; ) { - singleLineStatus(`Optimizing Keywords: ${i} / ${dbLength}`); + singleLineStatus( + `Optimizing Keywords: ${i} / ${dbLength} ${((i / dbLength) * 100).toFixed( + 2 + )} (${proctime.elapsed()})}` + ); let result = await File.findAndCountAll({ limit: BATCH_SIZE, offset: i, + order: ["id"], }); for (let x = 0; x < result.rows.length; x++) { debugPrint(`Submitting job for: ${result.rows[x]["filename"]}`); diff --git a/lib/metadatasearch.js b/lib/metadatasearch.js index 7a74095..c67b7e2 100644 --- a/lib/metadatasearch.js +++ b/lib/metadatasearch.js @@ -22,6 +22,8 @@ import { singleLineStatus } from "./debugprint.js"; import { Timer } from "./time.js"; import { readFileSync } from "fs"; import { dirname, resolve } from "path"; +import { Model } from "sequelize"; +import { Console } from "console"; export default class MetadataSearch { constructor() { @@ -55,9 +57,7 @@ export default class MetadataSearch { "videos.video_id", ]; - getPlatformMapping() { - - } + getPlatformMapping() {} async setupClient() { try { @@ -116,17 +116,47 @@ export default class MetadataSearch { let games = await File.findAndCountAll({ where: { nongame: false, + detailsId: null }, limit: 1000, + order: ["id"], }); - for (let x in games) { - let game = games[x]; - let metadata = await Metadata.searchByText(this.normalizeName(game.filename), game.category); - if (metadata) { - await game.setDetails(metadata); - await metadata.addFile(game); + let count = games.count; + let pages = Math.ceil(games.count / 1000); + let timer = new Timer(); + let found = 0; + console.log(`Matching ${count} games to metadata.`); + for (let x = 0; x < pages; x++) { + games = await File.findAndCountAll({ + where: { + nongame: false, + detailsId: null + }, + limit: 1000, + offset: x * 1000, + include: { model: Metadata, as: "details" }, + }); + for (let y = 0; y < games.rows.length; y++) { + singleLineStatus( + `Matching metadata: ${x * 1000 + y} / ${count} ${( + ((x * 1000 + y) / count) * + 100 + ).toFixed(2)}% (${timer.elapsed()}) Total Matches: ${found}` + ); + let game = games.rows[y]; + let metadata = await Metadata.searchByText( + this.normalizeName(game.filename), + game.category + ); + if (metadata.length) { + let md = await Metadata.findByPk(metadata[0].id); + await game.setDetails(md); + await md.addFile(game); + found++; + } } } + console.log(`Completed matching metadata to files in ${timer.elapsed()}`) } async syncAllMetadata(retrying = false) { @@ -179,6 +209,8 @@ export default class MetadataSearch { } retryCount = 0; } + console.log(`Finished syncing metadata in ${timer.elapsed()}`); + this.matchAllMetadata() } catch (error) { if (error.code === "ERR_BAD_REQUEST" && !retrying) { this.setupClient(); @@ -197,9 +229,7 @@ export default class MetadataSearch { id: metadata.id, }, { - returning: true, - updateOnDuplicate: ["id"], - include: File, + include: File } ); } @@ -213,7 +243,9 @@ export default class MetadataSearch { : null; md.genre = metadata.genres?.map((genre) => genre.name); md.gamemodes = metadata.game_modes?.map((gm) => gm.name); - md.platforms = metadata.platforms?.map((platform) => this.platformMap[platform.name] || platform.name); + md.platforms = metadata.platforms?.map( + (platform) => this.platformMap[platform.name] || platform.name + ); md.screenshots = metadata.screenshots?.map((ss) => ss.image_id); md.videos = metadata.videos?.map((v) => v.video_id); md.developers = metadata.involved_companies @@ -240,7 +272,7 @@ export default class MetadataSearch { ); } //this needs to remain json as we want the keys to be retained - md.alternatetiles = JSON.stringify(alternates); + md.alternatetitles = alternates.length ? JSON.stringify(alternates) : null; await md.save(); if (game) { await game.setDetails(md); diff --git a/lib/models/file.js b/lib/models/file.js index 754ba21..a514da2 100644 --- a/lib/models/file.js +++ b/lib/models/file.js @@ -51,11 +51,6 @@ export default function (sequelize) { group: { type: DataTypes.TEXT }, - blockmetadata: { - type: DataTypes.BOOLEAN, - defaultValue: false, - allowNull: false - }, nongame: { type: DataTypes.BOOLEAN, defaultValue: false, diff --git a/lib/models/metadata.js b/lib/models/metadata.js index a4d5b97..2dc5416 100644 --- a/lib/models/metadata.js +++ b/lib/models/metadata.js @@ -14,7 +14,7 @@ export default function (sequelize) { allowNull: false, }, alternatetitles: { - type: DataTypes.STRING(1024), + type: DataTypes.STRING(4096), }, description: { type: DataTypes.STRING(16384), @@ -68,11 +68,15 @@ export default function (sequelize) { Metadata.beforeSave("addVector", async (instance) => { const title = instance.title || ""; + const alternateTitles = + JSON.parse(instance.alternatetitles || "[]") + .map((title) => title.name) + .join(", ") || ""; const query = ` - SELECT to_tsvector('english', $1) + SELECT to_tsvector('english', $1 || ', ' || $2) `; const [results] = await sequelize.query(query, { - bind: [title], + bind: [title, alternateTitles], raw: true, }); instance.searchVector = results[0].to_tsvector; @@ -81,22 +85,18 @@ export default function (sequelize) { // Add a class method for full-text search Metadata.searchByText = async function (searchQuery, platform, limit = 1) { let platformClause = ""; - let limitClause = `limit ${limit}`; + let limitClause = `LIMIT ${limit}`; if (platform) { platformClause = `AND '${platform}' = ANY(platforms)`; } const query = ` - SELECT * FROM "Metadata" - WHERE "searchVector" @@ plainto_tsquery('english', :search) :platformClause - ORDER BY ts_rank("searchVector", plainto_tsquery('english', :search)) DESC :limit + SELECT id FROM "Metadata" + WHERE "searchVector" @@ plainto_tsquery('english', $1) ${platformClause} + ORDER BY ts_rank("searchVector", plainto_tsquery('english', $1 )) ${limitClause} `; return await sequelize.query(query, { model: Metadata, - replacements: { - search: searchQuery, - platformClause: platformClause, - limit: limitClause, - }, + bind: [searchQuery], type: sequelize.QueryTypes.SELECT, }); }; diff --git a/lib/services/elasticsearch.js b/lib/services/elasticsearch.js index f63858f..cbbaf5b 100644 --- a/lib/services/elasticsearch.js +++ b/lib/services/elasticsearch.js @@ -168,7 +168,8 @@ export async function search(query, options) { // Fetch full records from PostgreSQL for the search results const ids = response.hits.hits.map((hit) => hit._id); const fullRecords = await File.findAll({ - where: { id: ids } + where: { id: ids }, + include: {model: Metadata, as: "details"}, }); // Create a map of full records by id @@ -192,7 +193,6 @@ export async function search(query, options) { const elapsed = timer.elapsedSeconds(); return { items: results, - db: fullRecords, count: response.hits.total.value || 0, elapsed, }; diff --git a/server.js b/server.js index c141caf..56b86bb 100644 --- a/server.js +++ b/server.js @@ -1,4 +1,5 @@ import getAllFiles from "./lib/dircrawl.js"; +import { optimizeDatabaseKws } from "./dboptimize.js"; import FileHandler from "./lib/filehandler.js"; import Searcher from "./lib/search.js"; import cron from "node-cron"; @@ -16,11 +17,10 @@ import { isNonGameContent, } from "./lib/emulatorConfig.js"; import fetch from "node-fetch"; -import { initDB, File, QueryCount } from "./lib/database.js"; +import { initDB, File, QueryCount, Metadata } from "./lib/database.js"; import { initElasticsearch } from "./lib/services/elasticsearch.js"; import i18n, { locales } from "./config/i18n.js"; import { v4 as uuidv4 } from "uuid"; -import { optimizeDatabaseKws } from "./lib/dboptimize.js"; import MetadataSearch from "./lib/metadatasearch.js"; import Flag from "./lib/flag.js"; import ConsoleIcons from "./lib/consoleicons.js"; @@ -81,6 +81,10 @@ async function getFilesJob() { } crawlTime = Date.now(); console.log(`Finished updating file list. ${fileCount} found.`); + if(Metadata.count() > metadataSearch.getIGDBGamesCount()){ + await metadataSearch.syncAllMetadata(); + } + optimizeDatabaseKws(); } function buildOptions(page, options) { @@ -205,16 +209,12 @@ app.get("/search", async function (req, res) { delete settings.combineWith; } let loadOldResults = - req.query.old === "true" || !metadataSearch.authorized ? true : false; + req.query.old === "true" || !Metadata.count() ? true : false; settings.pageSize = loadOldResults ? 100 : 10; settings.page = pageNum - 1; settings.sort = req.query.o || ""; let results = await search.findAllMatches(query, settings); debugPrint(results); - let metas = []; - if (!loadOldResults) { - metas = await metadataSearch.queueGetGamesMetadata(results.db); - } if (results.count && pageNum == 1) { queryCount += 1; await QueryCount.update({ count: queryCount }, { where: { id: 1 } }); @@ -222,7 +222,7 @@ app.get("/search", async function (req, res) { } let options = { query: query, - results: metas?.length ? metas : results.items, + results: results.items, count: results.count, elapsed: results.elapsed, pageNum: pageNum, @@ -266,7 +266,7 @@ app.get("/lucky", async function (req, res) { app.get("/settings", function (req, res) { let options = { defaultSettings: defaultSettings }; let page = "settings"; - options.oldSettingsAvailable = metadataSearch.authorized; + options.oldSettingsAvailable = Metadata.count() ? true : false; options = buildOptions(page, options); res.render(indexPage, options); }); diff --git a/views/pages/info.ejs b/views/pages/info.ejs index 69d13a6..b7b5cd8 100644 --- a/views/pages/info.ejs +++ b/views/pages/info.ejs @@ -78,6 +78,9 @@

<%= __('search.filename') %> <%= file.filename %>

+
+

<%= __('search.size') %> <%= file.size %>

+

<%= __('search.upload_date') %> <%= file.date %>

diff --git a/views/partials/result.ejs b/views/partials/result.ejs index 45db1da..30de365 100644 --- a/views/partials/result.ejs +++ b/views/partials/result.ejs @@ -22,7 +22,7 @@

<%= __('search.no_metadata') %>

<% } %> <% if(metadata.title) {%> -

<%= __('search.filename') %> <%= file.filename %> | <%= __('search.upload_date')%> <%= file.date %>

+

<%= __('search.filename') %> <%= file.filename %> | <%= __('search.size')%> <%= file.size %> | <%= __('search.upload_date')%> <%= file.date %>

<% } %>

<%= __('search.release_group') %> <%= file.group %>