From d0d8ba01aa525cd5b50ea6e2e46fd8a7443e52a7 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 30 May 2025 17:15:08 -0600 Subject: [PATCH] Remove sequelize pagination in full db tasks cause uh... it has a lot of problems that make no sense to me. --- lib/dboptimize.js | 45 ++++++++++++++------ lib/metadatasearch.js | 96 ++++++++++++++++++------------------------ lib/models/metadata.js | 6 +-- server.js | 4 +- 4 files changed, 80 insertions(+), 71 deletions(-) diff --git a/lib/dboptimize.js b/lib/dboptimize.js index 0ab9d3c..95a0b22 100644 --- a/lib/dboptimize.js +++ b/lib/dboptimize.js @@ -34,22 +34,38 @@ export async function optimizeDatabaseKws() { let dbLength = await File.count(); let optimizeTasks = []; let resolvedTasks = []; + let promiseIndex = 0; + let currentIndex = 0; + let result = await File.findAll({ + order: ["id", "filename"], + attributes: [ + "id", + "filename", + "filenamekws", + "category", + "categorykws", + "subcategories", + "subcategorieskws", + "region", + "regionkws", + "type", + "nongame" + ], + }); for (let i = 0; i < dbLength; ) { + let loopIndexStart = i; singleLineStatus( `Optimizing Keywords: ${i} / ${dbLength} ${((i / dbLength) * 100).toFixed( 2 )}% (${proctime.elapsed()})` ); - let result = await File.findAndCountAll({ - limit: BATCH_SIZE, - offset: i, - order: ["id", "filename"], - }); - for (let x = 0; x < result.rows.length; x++) { - debugPrint(`Submitting job for: ${result.rows[x]["filename"]}`); + + for (let x = i; x < currentIndex + BATCH_SIZE; x++) { + if(x >= dbLength) break; //Abort abandon ship, otherwise we sink + debugPrint(`Submitting job for: ${result[x].filename}`); let data = []; for (let column in keywords) { - data[column] = result.rows[x][column]; + data[column] = result[x][column]; } optimizeTasks.push( piscina @@ -66,23 +82,28 @@ export async function optimizeDatabaseKws() { ); i++; } + currentIndex = i; let settledTasks = await Promise.all(optimizeTasks); resolvedTasks.push(...settledTasks); debugPrint(`Resolving ${resolvedTasks.length} optimization tasks.`); for (let y = 0; y < resolvedTasks.length; y++) { let changed = false; for (let column in keywords) { - if (result.rows[y][column + "kws"] == resolvedTasks[y][column + "kws"]) + if ( + result[promiseIndex][column + "kws"] == + resolvedTasks[y][column + "kws"] + ) continue; - result.rows[y][column + "kws"] = resolvedTasks[y][column + "kws"]; + result[promiseIndex][column + "kws"] = resolvedTasks[y][column + "kws"]; changed = true; } if (changed) { - result.rows[y].save(); + result[promiseIndex].save(); changes++; } + promiseIndex++; } - await bulkIndexFiles(result.rows); + await bulkIndexFiles(result.slice(loopIndexStart, currentIndex)); optimizeTasks = []; resolvedTasks = []; } diff --git a/lib/metadatasearch.js b/lib/metadatasearch.js index bc7bc9c..a6d8bd7 100644 --- a/lib/metadatasearch.js +++ b/lib/metadatasearch.js @@ -115,82 +115,70 @@ export default class MetadataSearch { } async matchAllMetadata(fuzzy = false) { - let games = await File.findAndCountAll({ + let games = await File.findAll({ where: { nongame: false, - //detailsId: null, + detailsId: null, }, - limit: 1000, + attributes: ["id", "filename"], order: ["id", "filename"], + include: { model: Metadata, as: "details" }, }); - let count = games.count; - let pages = Math.ceil(games.count / 1000); + let count = games.length; let timer = new Timer(); let found = 0; console.log(`Matching ${count} games to metadata.`); - for (let x = 0; x < pages; x++) { - games = await File.findAndCountAll({ - where: { - nongame: false, - //detailsId: null, - }, - limit: 1000, - offset: x * 1000 - found, - order: ["id", "filename"], - include: { model: Metadata, as: "details" }, - }); - for (let y = 0; y < games.rows.length; y++) { - singleLineStatus( - `Matching metadata: ${x * 1000 + y} / ${count} ${( - ((x * 1000 + y) / count) * - 100 - ).toFixed(2)}% (${timer.elapsed()}) Total Matches: ${found}` - ); - let game = games.rows[y]; - let metadata = await Metadata.searchByText( - "title", + for (let x = 0; x < count; x++) { + singleLineStatus( + `Matching metadata: ${x} / ${count} ${( + ((x) / count) * + 100 + ).toFixed(2)}% (${timer.elapsed()}) Total Matches: ${found}` + ); + let game = games[x]; + let metadata = await Metadata.searchByText( + "title", + this.normalizeName(game.filename), + game.category + ); + if (metadata?.length == 0) { + // repeat the search under one of the alternate titles + metadata = await Metadata.searchByText( + "alternatetitles", this.normalizeName(game.filename), game.category ); - if (!metadata.length) { - // repeat the search under one of the alternate titles - metadata = await Metadata.searchByText( + } + if (metadata?.length >= 1) { + let md = await Metadata.findByPk(metadata[0].id); + await game.setDetails(md); + await md.addFile(game); + await game.save(); + await md.save(); + found++; + } else if (fuzzy) { + //this is much slower and should only be used if the faster full text search can't find it. + let metadata = await Metadata.fuzzySearchByText( + "title", + this.normalizeName(game.filename), + 0.8, + game.category + ); + if (!metadata?.length == 0) { + metadata = await Metadata.fuzzySearchByText( "alternatetitles", this.normalizeName(game.filename), + 0.8, game.category ); } - if (metadata.length) { + if (metadata?.length >= 1) { let md = await Metadata.findByPk(metadata[0].id); await game.setDetails(md); await md.addFile(game); await game.save(); await md.save(); found++; - } else if (fuzzy) { - //this is much slower and should only be used if the faster full text search can't find it. - let metadata = await Metadata.fuzzySearchByText( - "title", - this.normalizeName(game.filename), - 0.8, - game.category - ); - if (!metadata.length) { - metadata = await Metadata.fuzzySearchByText( - "alternatetitles", - this.normalizeName(game.filename), - 0.8, - game.category - ); - } - if (metadata.length) { - let md = await Metadata.findByPk(metadata[0].id); - await game.setDetails(md); - await md.addFile(game); - await game.save(); - await md.save(); - found++; - } } } } diff --git a/lib/models/metadata.js b/lib/models/metadata.js index 6a10fed..e914c4f 100644 --- a/lib/models/metadata.js +++ b/lib/models/metadata.js @@ -105,14 +105,14 @@ export default function (sequelize) { if (platform && platform != "Others") { platformClause = `AND '${platform}' = ANY(platforms)`; } + let fieldName = field + 'Vector' const query = ` SELECT id FROM "Metadata" - WHERE $1 @@ plainto_tsquery('english', $2) ${platformClause} + WHERE "${fieldName}" @@ plainto_tsquery('english', $1) ${platformClause} ORDER BY length(title) ${limitClause} `; return await sequelize.query(query, { - model: Metadata, - bind: [(field + 'Vector'), searchQuery], + bind: [searchQuery], type: sequelize.QueryTypes.SELECT, }); }; diff --git a/server.js b/server.js index 7fef03c..9a69495 100644 --- a/server.js +++ b/server.js @@ -40,6 +40,7 @@ let indexPage = "pages/index"; let flags = new Flag(); let consoleIcons = new ConsoleIcons(emulatorsData); + // Initialize databases await initDB(); await initElasticsearch(); @@ -579,5 +580,4 @@ if ( cron.schedule("0 30 2 * * *", getFilesJob); -await metadataSearch.syncAllMetadata() -await metadataSearch.matchAllMetadata() \ No newline at end of file +await optimizeDatabaseKws() \ No newline at end of file