it works but it's a mess

This commit is contained in:
Alexandra
2025-05-24 02:40:43 -06:00
parent 2f2f358ea2
commit bb743933bd
18 changed files with 519 additions and 378 deletions

View File

@@ -18,7 +18,7 @@ export const File = defineFile(sequelize);
export const QueryCount = defineQueryCount(sequelize);
export const Metadata = defineMetadata(sequelize)
Metadata.hasMany(File)
File.belongsTo(Metadata)
File.belongsTo(Metadata, {as: "details"})
export async function initDB() {
try {

View File

@@ -5,7 +5,7 @@ import { readFileSync } from "fs";
import { fileURLToPath } from "url";
import { dirname, resolve } from "path";
import { Piscina, FixedQueue } from "piscina";
import { timer } from "./time.js";
import { Timer } from "./time.js";
let piscina = new Piscina({
filename: resolve("./lib", "dbkwworker.js"),
@@ -28,7 +28,7 @@ const keywords = {
};
export async function optimizeDatabaseKws() {
let proctime = new timer();
let proctime = new Timer();
let changes = 0;
console.log("Optimizing DB Keywords...");
let dbLength = await File.count();

View File

@@ -5,7 +5,7 @@ import debugPrint from "./debugprint.js";
import { File } from './models/index.js';
import { bulkIndexFiles } from './services/elasticsearch.js';
import { optimizeDatabaseKws } from "./dboptimize.js";
import { timer } from "./time.js";
import { Timer } from "./time.js";
let piscina = new Piscina({
filename: resolve("./lib", "fileworker.js"),
@@ -15,7 +15,7 @@ let piscina = new Piscina({
const BATCH_SIZE = 1000; // Process files in batches for better performance
export default async function getAllFiles(catList) {
var proctime = new timer()
var proctime = new Timer()
const url = "https://myrient.erista.me/files/";
let parentRows = await getTableRows({ url: url, base: "" });
let parents = [];
@@ -161,7 +161,8 @@ async function processBatch(files) {
type: file.type,
date: file.date,
region: file.region,
group: file.group
group: file.group,
nongame: file.nongame
})),
{
returning: true,

View File

@@ -1,5 +1,8 @@
import innertext from "innertext";
import HTMLParse from "node-html-parser";
import { fileURLToPath } from 'url';
import { dirname, resolve } from "path";
import { readFileSync } from "fs";
export async function getTableRows(data) {
let retryLeft = 5;
@@ -55,7 +58,8 @@ export async function parseOutFile(data) {
type: findType(fullName, data.catList),
date: innertext(file.querySelector(".date").innerHTML).trim(),
region: findRegion(fullName, data.catList),
group: findGroup(fullName)
group: findGroup(fullName),
nongame: checkNonGame(name)
};
return processedFile;
}
@@ -166,6 +170,29 @@ function findGroup(str){
}
}
// Cache for nonGameTerms
let nonGameTermsCache = null;
function getNonGameTerms() {
if (nonGameTermsCache) {
return nonGameTermsCache;
}
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const nonGameTermsPath = resolve(__dirname, 'nonGameTerms.json');
nonGameTermsCache = JSON.parse(readFileSync(nonGameTermsPath, 'utf8'));
return nonGameTermsCache;
}
function checkNonGame(str){
const nonGameTerms = getNonGameTerms();
const termPatterns = nonGameTerms.terms.map(term => new RegExp(term, 'i'));
return termPatterns.some(pattern => pattern.test(str));
}
class HTTPResponseError extends Error {
constructor(response) {
super(`HTTP Error Response: ${response.status} ${response.statusText}`);

223
lib/metadatasearch.js Normal file
View File

@@ -0,0 +1,223 @@
import {
twitchAccessToken,
igdb,
request,
multi,
} from "@phalcode/ts-igdb-client";
import {
fields,
or,
and,
where,
whereIn,
WhereFlags,
WhereInFlags,
sort,
limit,
offset,
} from "@phalcode/ts-igdb-client";
import { File, Metadata } from "./database.js";
import { Sequelize } from "sequelize";
export default class MetadataSearch {
constructor() {
this.twitchSecrets = {
client_id: process.env.TWITCH_CLIENT_ID,
client_secret: process.env.TWITCH_CLIENT_SECRET,
};
this.setupClient();
}
gameFields = [
"name",
"alternative_names.comment",
"alternative_names.name",
"cover.image_id",
"total_rating",
"first_release_date",
"summary",
"genres.name",
"involved_companies.company.name",
"involved_companies.developer",
"involved_companies.publisher",
"involved_companies.supporting",
"game_modes.name",
"game_localizations.name",
"game_localizations.region",
"game_localizations.region.name",
"platforms.name",
"game_type.type",
];
async setupClient() {
try {
if (this.twitchSecrets.client_id && this.twitchSecrets.client_secret) {
this.accessToken = await twitchAccessToken(this.twitchSecrets);
this.client = igdb(this.twitchSecrets.client_id, this.accessToken);
if (this.accessToken) {
this.authorized = true;
return;
}
}
this.authorized = false; //disable
} catch (error) {
this.authorized = false;
}
}
async getMetadata(query, retrying = false) {
try {
if (!this.authorized) return;
const { data } = await this.client
.multi(...this.buildGameMultiQuery(query))
.execute();
return data;
} catch (error) {
if (error === "ERR_BAD_REQUEST" && !retrying) {
this.setupClient();
return this.getMetadata(query, true);
}
console.error("Failed to retrieve metadata:", error);
}
}
buildGameMultiQuery(query) {
let multiQuery = [];
for (let x in query) {
multiQuery.push(
request("games")
.alias(x)
.pipe(
fields(this.gameFields),
and(
...this.buildAndClauses("name", "~", query[x].name),
where("game_type.type", "=", "Main Game"),
where("platforms.name", "~", query[x].platform)
),
limit(1)
)
);
}
return multiQuery;
}
buildAndClauses(field, op, string) {
let andClauses = [];
let name = [...new Set(string.split(" "))].filter((n) => n); //dedupe;
for (let x in name) {
andClauses.push(where(field, op, name[x], WhereFlags.CONTAINS));
}
return andClauses;
}
normalizeName(filename) {
if (!filename) return;
return filename
.replace(/\.[A-z]{3,3}|\.|&|-|,|v[0-9]+\.[0-9]+|\[.*?\]|\(.*?\)/g, "")
.trim();
}
async getGamesMetadata(games) {
try {
if (!this.authorized || !games.length) return [];
let gameQuery = [];
for (let x in games) {
if (!(await games[x].getDetails()))
if (!games[x].nongame) {
if (!games[x].blockmetadata) {
gameQuery.push({
name: this.normalizeName(games[x].filename),
platform: games[x].category,
id: x,
});
}
}
}
if (!gameQuery.length) return [];
let gameMetas = await this.getMetadata(gameQuery);
if (!gameMetas.length) return [];
for (let x in gameMetas) {
if (gameMetas[x].result.length) {
await this.addMetadataToDb(
gameMetas[x].result[0],
games[gameQuery[x].id]
);
}
}
let details = await Promise.all(games.map((game) => game.getDetails()));
return details.map((details) => details?.dataValues);
} catch (error) {
console.error("Error getting metadata:", error);
}
}
async addMetadataToDb(metadata, game) {
try {
let md = await Metadata.findOne({
where: {
id: metadata.id,
},
});
if (!md) {
md = await Metadata.build(
{
id: metadata.id,
title: metadata.name,
description: metadata.summary,
rating: metadata.total_rating,
coverartid: metadata.cover?.image_id,
releasedate: metadata.first_release_date
? new Date(metadata.first_release_date * 1000)
: null,
genre: JSON.stringify(metadata.genres?.map((genre) => genre.name)),
gamemodes: JSON.stringify(
metadata.game_modes?.map((gm) => gm.name)
),
platforms: JSON.stringify(
metadata.platforms?.map((platform) => platform.name)
),
},
{
returning: true,
updateOnDuplicate: ["id"],
include: File,
}
);
}
//these don't work right unless I do them after the fact.
md.developers = JSON.stringify(
metadata.involved_companies
?.filter((ic) => ic.developer)
?.map((ic) => ic.company.name)
);
md.publishers = JSON.stringify(
metadata.involved_companies
?.filter((ic) => ic.publisher)
?.map((ic) => ic.company.name)
);
let alternates = [];
if (metadata.alternative_names) {
alternates.push(
metadata.alternative_names.map((an) => ({
type: an.comment,
name: an.name,
}))
);
}
if (metadata.game_localizations) {
alternates.push(
metadata.game_localizations.map((gn) => ({
type: gn.region.name,
name: gn.name,
}))
);
}
md.alternatetiles = JSON.stringify(alternates);
await md.save();
await game.setDetails(md);
await md.addFile(game);
} catch (error) {
console.error("Error adding metadata:", error);
}
}
}

View File

@@ -1,113 +0,0 @@
import { twitchAccessToken, igdb, request } from "@phalcode/ts-igdb-client";
import {
fields,
or,
and,
where,
whereIn,
WhereFlags,
WhereInFlags,
sort,
limit,
offset,
} from "@phalcode/ts-igdb-client";
const twitchSecrets = {
client_id: process.env.TWITCH_CLIENT_ID,
client_secret: process.env.TWITCH_CLIENT_SECRET,
};
const accessToken = await twitchAccessToken(twitchSecrets);
const client = igdb(twitchSecrets.client_id, accessToken);
const gameFields = [
"name",
"alternative_names.comment",
"alternative_names.name",
"cover.image_id",
"total_rating",
"first_release_date",
"summary",
"genres.name",
"involved_companies.company.name",
"involved_companies.developer",
"involved_companies.publisher",
"involved_companies.supporting",
"multiplayer_modes.*",
"game_localizations.name",
"game_localizations.region",
"platforms.name",
];
export async function getMetadata(query) {
const data = await client
.request("games")
.pipe(
fields(gameFields),
or(...buildOrAndClauses("name", "~", query)),
sort("name", "asc")
)
.execute();
return data;
}
function buildOrClauses(field, op, queries) {
let orClauses = [];
for (let x in queries) {
orClauses.push(where(field, op, queries[x], WhereFlags.CONTAINS));
}
return orClauses;
}
function buildOrAndClauses(field, op, queries) {
let orClauses = [];
for (let x in queries) {
let name = [...new Set(queries[x].split(" "))]; //dedupe;
let andClauses = [];
for (let y in name) {
andClauses.push(where(field, op, name[y], WhereFlags.CONTAINS));
}
orClauses.push(and(...andClauses));
}
return orClauses;
}
function normalizeName(filename) {
if (!filename) return;
return filename
.replace(/\.[A-z]{3,3}|\.|&|-|,|v[0-9]+\.[0-9]+|\[.*?\]|\(.*?\)/g, "")
.trim();
}
function getBestMatch(filename, data) {
const words = filename.split(" ");
let bestIndex = null;
let bestMatchCount = 0;
let lengthDifference = 0;
for (let x in data) {
let matchingWords = 0;
for (let y in words) {
if (data[x].name.toLowerCase().includes(words[y].toLowerCase()))
matchingWords++;
}
let diff = matchingWords - dataWords.length;
if (matchingWords > bestMatchCount && diff < lengthDifference) {
bestIndex = x;
bestMatchCount = matchingWords;
lengthDifference = diff;
if (lengthDifference < 0) lengthDifference = 0;
}
}
if (bestIndex != null) {
return data[bestIndex];
}
return;
}
let games = await getMetadata([
"The Legend of Zelda A Link to the Past",
"Super Mario Sunshine",
]);
console.log(JSON.stringify(games.data, null, 2));
//console.log(await getMetadata(games))

View File

@@ -55,6 +55,11 @@ export default function (sequelize) {
type: DataTypes.BOOLEAN,
defaultValue: false,
allowNull: false
},
nongame: {
type: DataTypes.BOOLEAN,
defaultValue: false,
allowNull: false
}
}, {
indexes: [

View File

@@ -2,10 +2,9 @@ import { DataTypes } from "sequelize"
export default function (sequelize) {
const Metadata = sequelize.define('Metadata', {
id: {
id: {//these will match the igdbid to make things a little easier
type: DataTypes.INTEGER,
primaryKey: true,
autoIncrement: true
},
title: {
type: DataTypes.STRING,
@@ -15,23 +14,17 @@ export default function (sequelize) {
type: DataTypes.STRING
},
description: {
type: DataTypes.STRING
type: DataTypes.TEXT
},
rating: {
type: DataTypes.STRING
},
coverarturl: {
coverartid: {
type: DataTypes.STRING
},
releasedate: {
type: DataTypes.DATE
},
igdbid: {
type: DataTypes.INTEGER
},
timetobeat: {
type: DataTypes.STRING
},
genre: {
type: DataTypes.STRING
},
@@ -43,11 +36,14 @@ export default function (sequelize) {
},
gamemodes:{
type: DataTypes.STRING
},
platforms: {
type: DataTypes.STRING
}
}, {
indexes: [
{ fields: ['title'] },
{ fields: ['description'] }//If this slows down the db may want to not index this.
{ fields: ['description'] },//If this slows down the db may want to not index this.
]
})

View File

@@ -7,6 +7,7 @@
"beta",
"box",
"boxart",
"cbr",
"cheat",
"config",
"cfg",
@@ -32,6 +33,7 @@
"mod",
"movie",
"music",
"mp4",
"ost",
"overlay",
"patch",

View File

@@ -1,32 +1,13 @@
import { Client } from '@elastic/elasticsearch';
import debugPrint from '../debugprint.js';
import { File } from '../models/index.js';
import { readFileSync } from 'fs';
import { fileURLToPath } from 'url';
import { dirname, resolve } from 'path';
import { Client } from "@elastic/elasticsearch";
import debugPrint from "../debugprint.js";
import { File } from "../models/index.js";
import { Timer } from "../time.js";
const client = new Client({
node: process.env.ELASTICSEARCH_URL || 'http://localhost:9200'
node: process.env.ELASTICSEARCH_URL || "http://localhost:9200",
});
const INDEX_NAME = 'myrient_files';
// Cache for nonGameTerms
let nonGameTermsCache = null;
function getNonGameTerms() {
if (nonGameTermsCache) {
return nonGameTermsCache;
}
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const nonGameTermsPath = resolve(__dirname, '../../lib/nonGameTerms.json');
nonGameTermsCache = JSON.parse(readFileSync(nonGameTermsPath, 'utf8'));
return nonGameTermsCache;
}
const INDEX_NAME = "myrient_files";
export async function initElasticsearch() {
try {
@@ -40,56 +21,59 @@ export async function initElasticsearch() {
analysis: {
analyzer: {
filename_analyzer: {
type: 'custom',
tokenizer: 'standard',
filter: ['lowercase', 'word_delimiter_graph']
}
}
}
type: "custom",
tokenizer: "standard",
filter: ["lowercase", "word_delimiter_graph"],
},
},
},
},
mappings: {
properties: {
filename: {
type: 'text',
analyzer: 'filename_analyzer'
type: "text",
analyzer: "filename_analyzer",
},
category: {
type: 'text',
analyzer: 'standard',
type: "text",
analyzer: "standard",
fields: {
keyword: {
type: 'keyword'
}
}
type: "keyword",
},
},
},
type: {
type: 'text',
analyzer: 'standard'
type: "text",
analyzer: "standard",
},
region: {
type: 'text',
analyzer: 'standard'
type: "text",
analyzer: "standard",
},
filenamekws: {
type: 'text',
analyzer: 'standard'
type: "text",
analyzer: "standard",
},
categorykws: {
type: 'text',
analyzer: 'standard'
type: "text",
analyzer: "standard",
},
regionkws: {
type: 'text',
analyzer: 'standard'
}
}
}
}
type: "text",
analyzer: "standard",
},
nongame: {
type: "boolean",
},
},
},
},
});
console.log('Elasticsearch index created');
console.log("Elasticsearch index created");
}
} catch (error) {
console.error('Elasticsearch init error:', error);
console.error("Elasticsearch init error:", error);
process.exit(1);
}
}
@@ -99,16 +83,16 @@ export async function indexFile(file) {
await client.index({
index: INDEX_NAME,
id: file.id.toString(),
document: file
document: file,
});
debugPrint(`Indexed file: ${file.filename}`);
} catch (error) {
console.error('Error indexing file:', error);
console.error("Error indexing file:", error);
}
}
export async function bulkIndexFiles(files) {
const operations = files.flatMap(file => [
const operations = files.flatMap((file) => [
{ index: { _index: INDEX_NAME, _id: file.id.toString() } },
{
filename: file.filename,
@@ -117,19 +101,20 @@ export async function bulkIndexFiles(files) {
region: file.region,
filenamekws: file.filenamekws,
categorykws: file.categorykws,
regionkws: file.regionkws
}
regionkws: file.regionkws,
nongame: file.nongame
},
]);
try {
const { errors, items } = await client.bulk({
refresh: true,
operations
operations,
});
if (errors) {
console.error('Bulk indexing had errors');
items.forEach(item => {
console.error("Bulk indexing had errors");
items.forEach((item) => {
if (item.index.error) {
console.error(item.index.error);
}
@@ -138,46 +123,52 @@ export async function bulkIndexFiles(files) {
debugPrint(`Bulk indexed ${files.length} files`);
} catch (error) {
console.error('Bulk indexing error:', error);
console.error("Bulk indexing error:", error);
}
}
export async function search(query, options) {
//add kws for selected fields
let builtFields = []
for(let field in options.fields){
builtFields.push(options.fields[field])
builtFields.push(options.fields[field] + 'kws')
let builtFields = [];
for (let field in options.fields) {
builtFields.push(options.fields[field]);
builtFields.push(options.fields[field] + "kws");
}
const searchQuery = {
index: INDEX_NAME,
body: {
size: 1500,
size: options.pageSize,
from: options.pageSize * options.page,
query: {
bool: {
must: buildMustClauses(query, options, builtFields),
should: buildShouldClauses(query, options, builtFields)
}
should: buildShouldClauses(query, options, builtFields),
},
},
highlight: {
fields: {
filename: {},
category: {},
type: {},
region: {}
}
}
}
region: {},
},
},
},
};
if (options.hideNonGame) {
searchQuery.body.query.bool["filter"] = {
term: { nongame: false },
};
}
try {
const startTime = process.hrtime();
let timer = new Timer();
const response = await client.search(searchQuery);
// Fetch full records from PostgreSQL for the search results
const ids = response.hits.hits.map(hit => hit._id);
const ids = response.hits.hits.map((hit) => hit._id);
const fullRecords = await File.findAll({
where: { id: ids }
where: { id: ids },
});
// Create a map of full records by id
@@ -187,49 +178,44 @@ export async function search(query, options) {
}, {});
// Build results with full PostgreSQL records
let results = response.hits.hits.map(hit => ({
...recordMap[hit._id].dataValues,
let results = response.hits.hits.map((hit) => ({
...recordMap[hit._id]?.dataValues,
score: hit._score,
highlights: hit.highlight
highlights: hit.highlight,
}));
// Apply non-game content filtering in JavaScript if the option is enabled
if (options.hideNonGame) {
const nonGameTerms = getNonGameTerms();
const termPatterns = nonGameTerms.terms.map(term => new RegExp(term, 'i'));
//Filter out anything that couldn't be found in postgres
results = results.filter(result => result.filename)
// Filter results in JavaScript (much faster than complex Elasticsearch queries)
results = results.filter(item => {
// Check if filename contains any of the non-game terms
return !termPatterns.some(pattern => pattern.test(item.filename));
});
}
const elapsed = parseHrtimeToSeconds(process.hrtime(startTime));
const elapsed = timer.elapsedSeconds();
return {
items: results,
elapsed
db: fullRecords,
count: response.hits.total.value || 0,
elapsed,
};
} catch (error) {
console.error('Search error:', error);
return { items: [], elapsed: 0 };
console.error("Search error:", error);
return { items: [], elapsed: 0, count: 0 };
}
}
function buildMustClauses(query, options, builtFields) {
const clauses = [];
if (options.combineWith === 'AND') {
query.split(' ').forEach(term => {
if (options.combineWith === "AND") {
query.split(" ").forEach((term) => {
clauses.push({
multi_match: {
query: term,
fields: builtFields.map(field =>
field === 'filename' || 'filenamekws' ? `${field}^2` : field
fields: builtFields.map((field) =>
field === "filename" || field === "filenamekws"
? `${field}^2`
: field
),
fuzziness: options.fuzzy || 0,
type: 'best_fields'
}
type: "best_fields",
},
});
});
}
@@ -240,26 +226,22 @@ function buildMustClauses(query, options, builtFields) {
function buildShouldClauses(query, options, builtFields) {
const clauses = [];
if (options.combineWith !== 'AND') {
if (options.combineWith !== "AND") {
clauses.push({
multi_match: {
query,
fields: builtFields.map(field =>
field === 'filename' || 'filenamekws' ? `${field}^2` : field
fields: builtFields.map((field) =>
field === "filename" || field === "filenamekws" ? `${field}^2` : field
),
fuzziness: options.fuzzy || 0,
type: 'best_fields'
}
type: "best_fields",
},
});
}
return clauses;
}
function parseHrtimeToSeconds(hrtime) {
return (hrtime[0] + (hrtime[1] / 1e9)).toFixed(3);
}
export async function getSuggestions(query, options) {
try {
const response = await client.search({
@@ -268,26 +250,26 @@ export async function getSuggestions(query, options) {
query: {
multi_match: {
query,
fields: ['filename^2', 'filenamekws^2', 'category', 'categorykws'],
fuzziness: 'AUTO',
type: 'best_fields'
}
fields: ["filename^2", "filenamekws^2", "category", "categorykws"],
fuzziness: "AUTO",
type: "best_fields",
},
},
_source: ['filename', 'category'],
size: 10
}
_source: ["filename", "category"],
size: 10,
},
});
return response.hits.hits.map(hit => ({
suggestion: hit._source.filename
return response.hits.hits.map((hit) => ({
suggestion: hit._source.filename,
}));
} catch (error) {
console.error('Suggestion error:', error);
console.error("Suggestion error:", error);
return [];
}
}
export async function getSample(query, options){
export async function getSample(query, options) {
try {
const response = await client.search({
index: INDEX_NAME,
@@ -295,18 +277,18 @@ export async function getSample(query, options){
query: {
match: {
filename: query,
}
},
},
_source: ['filename'],
size: 30
}
_source: ["filename"],
size: 30,
},
});
return response.hits.hits.map(hit => ({
sample: hit._source.filename
return response.hits.hits.map((hit) => ({
sample: hit._source.filename,
}));
} catch (error) {
console.error('Sample error:', error);
console.error("Sample error:", error);
return [];
}
}
}

View File

@@ -1,4 +1,4 @@
export class timer {
export class Timer {
constructor() {
this.startTime = process.hrtime();
}
@@ -13,4 +13,7 @@ export class timer {
let s = Math.floor(elapsed % 60);
return `${h ? h + "h" : ""}${m ? m + "m" : ""}${s + "s"}`;
}
}
elapsedSeconds(){
return this.parseHrtimetoSeconds(process.hrtime(this.startTime));
}
}

8
package-lock.json generated
View File

@@ -24,7 +24,7 @@
"pg-hstore": "^2.3.4",
"piscina": "^4.7.0",
"sanitize": "^2.1.2",
"sequelize": "^6.37.1",
"sequelize": "^6.37.7",
"sequelize-cli": "^6.6.2",
"to-words": "^4.5.1",
"uuid": "^11.1.0"
@@ -3175,9 +3175,9 @@
"license": "MIT"
},
"node_modules/sequelize": {
"version": "6.37.5",
"resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.37.5.tgz",
"integrity": "sha512-10WA4poUb3XWnUROThqL2Apq9C2NhyV1xHPMZuybNMCucDsbbFuKg51jhmyvvAUyUqCiimwTZamc3AHhMoBr2Q==",
"version": "6.37.7",
"resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.37.7.tgz",
"integrity": "sha512-mCnh83zuz7kQxxJirtFD7q6Huy6liPanI67BSlbzSYgVNl5eXVdE2CN1FuAeZwG1SNpGsNRCV+bJAVVnykZAFA==",
"funding": [
{
"type": "opencollective",

View File

@@ -19,7 +19,7 @@
"pg-hstore": "^2.3.4",
"piscina": "^4.7.0",
"sanitize": "^2.1.2",
"sequelize": "^6.37.1",
"sequelize": "^6.37.7",
"sequelize-cli": "^6.6.2",
"to-words": "^4.5.1",
"uuid": "^11.1.0"

View File

@@ -21,6 +21,7 @@ import { initElasticsearch } from "./lib/services/elasticsearch.js";
import i18n, { locales } from "./config/i18n.js";
import { v4 as uuidv4 } from "uuid";
import { optimizeDatabaseKws } from "./lib/dboptimize.js";
import MetadataSearch from "./lib/metadatasearch.js";
let categoryListPath = "./lib/categories.json";
let nonGameTermsPath = "./lib/nonGameTerms.json";
@@ -51,6 +52,7 @@ let defaultSettings = {
fuzzy: 0,
prefix: true,
hideNonGame: true,
useOldResults: false,
};
//programmatically set the default boosts while reducing overhead when adding another search field
@@ -64,6 +66,7 @@ for (let field in searchFields) {
}
let search = new Searcher(searchFields);
let metadataSearch = new MetadataSearch();
async function getFilesJob() {
console.log("Updating the file list.");
@@ -106,7 +109,7 @@ app.use((req, res, next) => {
});
//static files
app.use('/public', express.static('views/public'))
app.use("/public", express.static("views/public"));
//middleware
app.use(sanitize.middleware);
@@ -193,22 +196,35 @@ app.get("/search", async function (req, res) {
if (settings.combineWith != "AND") {
delete settings.combineWith;
}
settings.pageSize = settings.useOldResults ? 100 : 10;
settings.page = pageNum - 1;
let results = await search.findAllMatches(query, settings);
debugPrint(results);
if (results.items.length && pageNum == 1) {
let metas = await metadataSearch.getGamesMetadata(results.db);
if (results.count && pageNum == 1) {
queryCount += 1;
await QueryCount.update({ count: queryCount }, { where: { id: 1 } });
updateDefaults();
}
let resultOutput = [];
for (let x in results.items) {
resultOutput.push({
file: results.items[x],
metadata: metas[x] || [],
});
}
let options = {
query: query,
results: results,
results: resultOutput,
count: results.count,
elapsed: results.elapsed,
pageNum: pageNum,
pageCount: Math.ceil(results.count / settings.pageSize),
indexing: search.indexing,
urlPrefix: urlPrefix,
settings: settings,
};
let page = "resultsnew";
let page = settings.useOldResults ? "resultsold" : "results";
options = buildOptions(page, options);
res.render(indexPage, options);
});
@@ -511,4 +527,4 @@ if (
await getFilesJob();
}
cron.schedule("0 30 2 * * *", getFilesJob);
cron.schedule("0 30 2 * * *", getFilesJob);

View File

@@ -1,12 +1,8 @@
<%
let pageCount = Math.ceil(results.items.length / 100)
pageCount = pageCount ? pageCount : 1 //always ensure 1 page
if(pageNum > pageCount){
pageNum = 1
}
let entryStart = Math.floor((pageNum - 1) * 100)
let entryEnd = entryStart + 100
entryEnd = entryEnd > results.items.length ? results.items.length : entryEnd
%>
<script src='https://code.jquery.com/jquery-3.7.1.js' crossorigin="anonymous"></script>
<script src='https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js' crossorigin="anonymous"></script>
@@ -31,7 +27,7 @@
<ul class="SuggestionList col-sm-12" id="suggestionList" style="width: 50%;left: 195px;"></ul>
</div>
<p class="m-2">
<%= __('search.found_plural', { count: results.items.length }) %> <%= __('search.in_seconds', { seconds: results.elapsed }) %>.
<%= __('search.found_plural', { count: count }) %> <%= __('search.in_seconds', { seconds: elapsed }) %>.
<%= indexing ? __('search.indexing') : "" %>
<% if (settings.hideNonGame) { %>
<span class="badge badge-info" data-toggle="tooltip" data-placement="top" title="<%= __('settings.extras.hide_non_game.tooltip') %>">
@@ -43,63 +39,11 @@
</form>
<div class="col-sm-12 w-100 mt-3">
<p><%= __('search.displaying_results', { start: entryStart, end: entryEnd }) %></p>
<table class="table text-white table-bordered" id="results">
<thead>
<tr>
<th class="dt-orderable"><span><%= __('results.table.name') %></span><span class="dt-column-order"></span></th>
<th><%= __('results.table.group') %></th>
<th><%= __('results.table.category') %></th>
<th><%= __('results.table.region') %></th>
<th><%= __('results.table.type') %></th>
<th><%= __('results.table.size') %></th>
<th><%= __('results.table.date') %></th>
<th><%= __('results.table.score') %></th>
<% if (process.env.EMULATOR_ENABLED === 'true') { %>
<th><%= __('results.table.play') %></th>
<% } %>
</tr>
</thead>
<% for (let x = entryStart; x < entryEnd; x++) { %>
<tr>
<td>
<a href="<%= results.items[x].path %>">
<%= results.items[x].filename %>
</a>
</td>
<td>
<%= results.items[x].group %>
</td>
<td>
<%= results.items[x].category %>
</td>
<td>
<%= results.items[x].region %>
</td>
<td>
<%= results.items[x].type %>
</td>
<td>
<%= results.items[x].size %>
</td>
<td>
<%= results.items[x].date %>
</td>
<td>
<%= results.items[x].score.toFixed(2) %>
</td>
<% if (process.env.EMULATOR_ENABLED === 'true') { %>
<td>
<% if (isEmulatorCompatible(results.items[x].category)) { %>
<a href="/play/<%= results.items[x].id %>" class="btn btn-sm btn-secondary"><%= __('emulator.play') %></a>
<% } else { %>
<button class="btn btn-sm btn-secondary" disabled><%= __('emulator.not_available') %></button>
<% } %>
</td>
<% } %>
</tr>
<div>
<% for (let x = 0; x < results.length; x++) { %>
<%- include("../partials/result", {result: results[x]}) %>
<% } %>
</table>
</div>
<%
if(pageCount > 1) {
%>
@@ -150,33 +94,4 @@
</div>
<% } %>
</div>
</div>
<script defer>
/*resultTable = new DataTable('#results', {
"order": [[7, 'desc']],
"columns": [
{ "data": "name" }, // Name
{ "data": "group" }, // Group
{ "data": "category" }, // Category
{ "data": "region" }, // Region
{ "data": "type" }, // Type
{ "data": "size" }, // Size
{ "data": "date" }, // Date
{ "data": "score" }, // Search Score
<% if (process.env.EMULATOR_ENABLED === 'true') { %>
{ "data": "play", "orderable": false } // Play button column
<% } %>
],
"lengthMenu": [100, { label: 'All', value: -1 }, 50, 25, 10],
"paging": false,
"filter": false,
"layout": {
"bottomStart": ''
}
});*/
// Initialize tooltips
$(function () {
$('[data-toggle="tooltip"]').tooltip()
});
</script>
</div>

View File

@@ -44,11 +44,62 @@
<div class="col-sm-12 w-100 mt-3">
<p><%= __('search.displaying_results', { start: entryStart, end: entryEnd }) %></p>
<div>
<table class="table text-white table-bordered" id="results">
<thead>
<tr>
<th class="dt-orderable"><span><%= __('results.table.name') %></span><span class="dt-column-order"></span></th>
<th><%= __('results.table.group') %></th>
<th><%= __('results.table.category') %></th>
<th><%= __('results.table.region') %></th>
<th><%= __('results.table.type') %></th>
<th><%= __('results.table.size') %></th>
<th><%= __('results.table.date') %></th>
<th><%= __('results.table.score') %></th>
<% if (process.env.EMULATOR_ENABLED === 'true') { %>
<th><%= __('results.table.play') %></th>
<% } %>
</tr>
</thead>
<% for (let x = entryStart; x < entryEnd; x++) { %>
<%- include("../partials/result", {result: results.items[x]}) %>
<tr>
<td>
<a href="<%= results.items[x].path %>">
<%= results.items[x].filename %>
</a>
</td>
<td>
<%= results.items[x].group %>
</td>
<td>
<%= results.items[x].category %>
</td>
<td>
<%= results.items[x].region %>
</td>
<td>
<%= results.items[x].type %>
</td>
<td>
<%= results.items[x].size %>
</td>
<td>
<%= results.items[x].date %>
</td>
<td>
<%= results.items[x].score.toFixed(2) %>
</td>
<% if (process.env.EMULATOR_ENABLED === 'true') { %>
<td>
<% if (isEmulatorCompatible(results.items[x].category)) { %>
<a href="/play/<%= results.items[x].id %>" class="btn btn-sm btn-secondary"><%= __('emulator.play') %></a>
<% } else { %>
<button class="btn btn-sm btn-secondary" disabled><%= __('emulator.not_available') %></button>
<% } %>
</td>
<% } %>
</tr>
<% } %>
</div>
</table>
<%
if(pageCount > 1) {
%>
@@ -99,4 +150,33 @@
</div>
<% } %>
</div>
</div>
</div>
<script defer>
/*resultTable = new DataTable('#results', {
"order": [[7, 'desc']],
"columns": [
{ "data": "name" }, // Name
{ "data": "group" }, // Group
{ "data": "category" }, // Category
{ "data": "region" }, // Region
{ "data": "type" }, // Type
{ "data": "size" }, // Size
{ "data": "date" }, // Date
{ "data": "score" }, // Search Score
<% if (process.env.EMULATOR_ENABLED === 'true') { %>
{ "data": "play", "orderable": false } // Play button column
<% } %>
],
"lengthMenu": [100, { label: 'All', value: -1 }, 50, 25, 10],
"paging": false,
"filter": false,
"layout": {
"bottomStart": ''
}
});*/
// Initialize tooltips
$(function () {
$('[data-toggle="tooltip"]').tooltip()
});
</script>

View File

@@ -1,9 +1,8 @@
<%
let resultStart = Math.floor((pageNum - 1) * 100)
let length = results.items.length > 5 + resultStart ? 5 + resultStart : results.items.length
let length = results.length > 5 ? 5 : results.length
let resultString = ''
for(let x = resultStart ; x < length; x++){
resultString += `${x + 1}: ${results.items[x].filename}\n\n`
for(let x = 0 ; x < length; x++){
resultString += `${x + 1}: ${results[x].filename}\n\n`
}
resultString = resultString.trim()
%>

View File

@@ -1,14 +1,19 @@
<%
const metadata = result.metadata || new Object()
const file = result.file || new Object()
const coverUrl = metadata.coverartid ? `/proxy-image?url=https://images.igdb.com/igdb/image/upload/t_cover_big/${metadata.coverartid}.webp` : "/public/images/coverart/nocoverart.png"
%>
<div class="col-md-auto row align-items-start searchresult">
<div class="">
<img class="coverart" src="<%= result.image || "/public/images/coverart/nocoverart.png"%>">
<img class="coverart" src="<%= coverUrl %>">
</div>
<div class="col-md">
<p class="title"><%= result.title || result.filename %></p>
<p class="info"<span class="infoitem">Released: <%= result.releaseDate || result.date %></span> <span class="infoitem">Region: <%= result.region %></span></p>
<p class="description"><%= result.description || "No description was found." %></p>
<% if(result.title) {%>
<p class="file">Filename: <%= result.filename %></p>
<p class="title"><%= metadata.title || file.filename %></p>
<p class="info"<span class="infoitem">Released: <%= metadata.releasedate || file.date %></span> <span class="infoitem">Region: <%= file.region %></span> <span class="infoitem">Platform: <%= file.category %></span></p>
<p class="description"><%= metadata.description || "No description was found." %></p>
<% if(metadata.title) {%>
<p class="file">Filename: <%= file.filename %></p>
<% } %>
<p class="actions"><button class="btn btn-sm btn-secondary" href="">More Info</button> <button class="btn btn-sm btn-secondary" href="<%= result.path %>">Download</button> <button class="btn btn-sm btn-secondary" href="/play/<%= result.id %>">Play In Browser</a></button>
<p class="actions"><button class="btn btn-sm btn-secondary" href="">More Info</button> <button class="btn btn-sm btn-secondary" href="<%= file.path %>">Download</button> <button class="btn btn-sm btn-secondary" href="/play/<%= file.id %>">Play In Browser</a></button>
</div>
</div>