mirror of
https://github.com/alexankitty/Myrient-Search-Engine.git
synced 2026-01-15 08:23:18 -03:00
Merge pull request #22 from alexankitty/roman-numeral
Search Keywords, Roman Numeral Processing, etc.
This commit is contained in:
Alexandra
GitHub
@@ -9,7 +9,7 @@ services:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 1GB
|
||||
memory: 2GB
|
||||
ports:
|
||||
- "9200:9200"
|
||||
volumes:
|
||||
@@ -28,4 +28,4 @@ services:
|
||||
|
||||
volumes:
|
||||
elasticsearch_data:
|
||||
postgres_data:
|
||||
postgres_data:
|
||||
|
||||
152
lib/dbkwworker.js
Normal file
152
lib/dbkwworker.js
Normal file
@@ -0,0 +1,152 @@
|
||||
import { ToWords } from "to-words";
|
||||
import { getSample } from "./services/elasticsearch.js";
|
||||
|
||||
const toWords = new ToWords({
|
||||
localeCode: "en-US",
|
||||
converterOptions: {
|
||||
ignoreDecimal: false,
|
||||
doNotAddOnly: true,
|
||||
},
|
||||
});
|
||||
|
||||
function stringToWordArray(string) {
|
||||
let symbolRegex =
|
||||
/_|\+|=|\)|\(|\[|{|}|]|;|:|"|'|<|>|\.|,|\/|\?|\||\\|!|@|#|\$|%|\^|&|\*/g;
|
||||
let workingString = string.replaceAll("-", " ");
|
||||
workingString = workingString.replaceAll(symbolRegex, " ");
|
||||
let stringArray = workingString.split(" ");
|
||||
return stringArray.filter((entry) => entry.trim() != "");
|
||||
}
|
||||
|
||||
function kwProcessor(terms, kwArr) {
|
||||
for (let term in terms) {
|
||||
terms[term] = terms[term].toLowerCase();
|
||||
}
|
||||
let foundKws = [];
|
||||
|
||||
for (let word in terms) {
|
||||
for (let group in kwArr) {
|
||||
let currentGroup = kwArr[group];
|
||||
for (let index in currentGroup) {
|
||||
if (currentGroup[index] == terms[word]) {
|
||||
foundKws.push(...currentGroup);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (foundKws) return [...new Set(foundKws)];
|
||||
}
|
||||
|
||||
async function getNumerals(stringArr) {
|
||||
let numerals = [];
|
||||
let nameWordLen = 0;
|
||||
for (let word in stringArr) {
|
||||
let curWord = stringArr[word];
|
||||
if (validateRomanNumeral(curWord)) {
|
||||
nameWordLen = word;
|
||||
let numeral = parseNumeral(curWord);
|
||||
if (numeral) numerals.push(numeral);
|
||||
}
|
||||
}
|
||||
//Guard clause, exits when we didn't find a valid numeral
|
||||
if (!nameWordLen) return;
|
||||
let searchQuery = stringArr.slice(0, nameWordLen).join(" ").trim();
|
||||
//Check if this is a series
|
||||
let results = await getSample(searchQuery);
|
||||
let series = false;
|
||||
//always return ii if it's available
|
||||
for (let x in numerals) {
|
||||
if (numerals[x] == 2) return [...new Set(numerals)];
|
||||
}
|
||||
if (results.length > 1) {
|
||||
for (let x in results) {
|
||||
let seriesNumeral = [];
|
||||
let words = stringToWordArray(results[x].sample);
|
||||
for (let word in words) {
|
||||
let numeral = parseNumeral(words[word]);
|
||||
if (numeral) seriesNumeral.push(numeral);
|
||||
}
|
||||
if (seriesNumeral > 0) {
|
||||
for (let x in numerals) {
|
||||
for (let y in seriesNumeral) {
|
||||
if (numerals[x] != seriesNumeral[y]) {
|
||||
series = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!series) return;
|
||||
numerals.push(getNumberNames(numerals));
|
||||
return [...new Set(numerals)];
|
||||
}
|
||||
}
|
||||
|
||||
function parseNumeral(string) {
|
||||
//Keep these upper case to reduce the number of false positives. Make sure the input isn't tolower
|
||||
const romanNumerals = {
|
||||
/*M: 1000,
|
||||
CM: 900,
|
||||
D: 500,
|
||||
CD: 400,
|
||||
C: 100,
|
||||
XC: 90,
|
||||
L: 50,
|
||||
XL: 40,*/
|
||||
X: 10,
|
||||
IX: 9,
|
||||
V: 5,
|
||||
IV: 4,
|
||||
I: 1,
|
||||
};
|
||||
if (validateRomanNumeral(string)) {
|
||||
let numeralSum = 0;
|
||||
string = string.toUpperCase();
|
||||
for (let numeral in romanNumerals) {
|
||||
while (string.startsWith(numeral)) {
|
||||
numeralSum += romanNumerals[numeral];
|
||||
string = string.substring(numeral.length);
|
||||
}
|
||||
}
|
||||
if (string.length > 0) return 0;
|
||||
return numeralSum;
|
||||
}
|
||||
}
|
||||
|
||||
function getNumberNames(stringArr) {
|
||||
let numbers = [];
|
||||
for (let number in stringArr) {
|
||||
let curNum = stringArr[number];
|
||||
if (/^\d+$/.test(curNum)) {
|
||||
let numberName = toWords.convert(parseInt(curNum));
|
||||
if (numberName) numbers.push(numberName.trim());
|
||||
}
|
||||
}
|
||||
return [...new Set(numbers)];
|
||||
}
|
||||
|
||||
function validateRomanNumeral(string) {
|
||||
if (!string) return false;
|
||||
if (string == "vim") return false;
|
||||
let romanRegex = /i|v|x|l|c|d|m/gi;
|
||||
return !string.replaceAll(romanRegex, "");
|
||||
}
|
||||
|
||||
export async function optimizeKws(object) {
|
||||
for (let column in object.keywords) {
|
||||
if (!object.data[column]) continue;
|
||||
let wordArr = stringToWordArray(object.data[column]);
|
||||
let workKws = kwProcessor(wordArr, object.keywords[column]);
|
||||
//special case for filenames
|
||||
if (column == "filename") {
|
||||
let numerals = await getNumerals(wordArr);
|
||||
if (numerals) {
|
||||
workKws.push(...numerals);
|
||||
}
|
||||
workKws.push(...getNumberNames(wordArr));
|
||||
}
|
||||
object.data[column + "kws"] = workKws.join(" ").trim();
|
||||
}
|
||||
return object.data;
|
||||
}
|
||||
99
lib/dboptimize.js
Normal file
99
lib/dboptimize.js
Normal file
@@ -0,0 +1,99 @@
|
||||
import debugPrint from "./debugprint.js";
|
||||
import { bulkIndexFiles } from "./services/elasticsearch.js";
|
||||
import { File } from "./models/index.js";
|
||||
import { readFileSync } from "fs";
|
||||
import { fileURLToPath } from "url";
|
||||
import { dirname, resolve } from "path";
|
||||
import { Piscina, FixedQueue } from "piscina";
|
||||
import { timer } from "./time.js";
|
||||
|
||||
let piscina = new Piscina({
|
||||
filename: resolve("./lib", "dbkwworker.js"),
|
||||
taskQueue: new FixedQueue(),
|
||||
});
|
||||
|
||||
const BATCH_SIZE = 1000;
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const relatedKwRoot = "../lib/json/relatedkeywords/";
|
||||
const catKwPath = resolve(__dirname, relatedKwRoot + "categories.json");
|
||||
const nameKwpath = resolve(__dirname, relatedKwRoot + "names.json");
|
||||
const regionKwpath = resolve(__dirname, relatedKwRoot + "regions.json");
|
||||
//make sure the child object matches the column in the file db model
|
||||
const keywords = {
|
||||
filename: JSON.parse(readFileSync(nameKwpath, "utf8")),
|
||||
category: JSON.parse(readFileSync(catKwPath, "utf8")),
|
||||
subcategories: JSON.parse(readFileSync(catKwPath, "utf8")),
|
||||
region: JSON.parse(readFileSync(regionKwpath, "utf8")),
|
||||
};
|
||||
|
||||
export async function optimizeDatabaseKws() {
|
||||
let proctime = new timer();
|
||||
let changes = 0;
|
||||
console.log("Optimizing DB Keywords...");
|
||||
let dbLength = await File.count();
|
||||
let optimizeTasks = [];
|
||||
let resolvedTasks = [];
|
||||
for (let i = 0; i < dbLength; ) {
|
||||
singleLineStatus(`Optimizing Keywords: ${i} / ${dbLength}`);
|
||||
let result = await File.findAndCountAll({
|
||||
limit: BATCH_SIZE,
|
||||
offset: i,
|
||||
});
|
||||
for (let x = 0; x < result.rows.length; x++) {
|
||||
debugPrint(`Submitting job for: ${result.rows[x]["filename"]}`);
|
||||
let data = [];
|
||||
for (let column in keywords) {
|
||||
data[column] = result.rows[x][column];
|
||||
}
|
||||
optimizeTasks.push(
|
||||
piscina
|
||||
.run(
|
||||
{
|
||||
data: data,
|
||||
keywords: keywords,
|
||||
},
|
||||
{ name: "optimizeKws" }
|
||||
)
|
||||
.catch((err) => {
|
||||
console.error(err);
|
||||
})
|
||||
);
|
||||
i++;
|
||||
}
|
||||
let settledTasks = await Promise.all(optimizeTasks);
|
||||
resolvedTasks.push(...settledTasks);
|
||||
debugPrint(`Resolving ${resolvedTasks.length} optimization tasks.`);
|
||||
for (let y = 0; y < resolvedTasks.length; y++) {
|
||||
let changed = false;
|
||||
for (let column in keywords) {
|
||||
if (result.rows[y][column + "kws"] == resolvedTasks[y][column + "kws"])
|
||||
continue;
|
||||
result.rows[y][column + "kws"] = resolvedTasks[y][column + "kws"];
|
||||
changed = true;
|
||||
}
|
||||
if (changed) {
|
||||
result.rows[y].save();
|
||||
changes++;
|
||||
}
|
||||
}
|
||||
await bulkIndexFiles(result.rows);
|
||||
optimizeTasks = [];
|
||||
resolvedTasks = [];
|
||||
}
|
||||
console.log(
|
||||
`\nCompleted Keyword Optimization for ${changes} row${
|
||||
changes > 1 || changes == 0 ? "s" : ""
|
||||
} in ${proctime.elapsed()}.`
|
||||
);
|
||||
}
|
||||
|
||||
function singleLineStatus(str) {
|
||||
if (process.stdout.isTTY && process.env.DEBUG != "1") {
|
||||
process.stdout.clearLine(0);
|
||||
process.stdout.cursorTo(0);
|
||||
process.stdout.write(str);
|
||||
} else {
|
||||
console.log(str);
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import { resolve } from "path";
|
||||
import debugPrint from "./debugprint.js";
|
||||
import { File } from './models/index.js';
|
||||
import { bulkIndexFiles } from './services/elasticsearch.js';
|
||||
import { optimizeDatabaseKws } from "./dboptimize.js";
|
||||
import { timer } from "./time.js";
|
||||
|
||||
let piscina = new Piscina({
|
||||
filename: resolve("./lib", "fileworker.js"),
|
||||
@@ -13,7 +15,7 @@ let piscina = new Piscina({
|
||||
const BATCH_SIZE = 1000; // Process files in batches for better performance
|
||||
|
||||
export default async function getAllFiles(catList) {
|
||||
var startTime = process.hrtime();
|
||||
var proctime = new timer()
|
||||
const url = "https://myrient.erista.me/files/";
|
||||
let parentRows = await getTableRows({ url: url, base: "" });
|
||||
let parents = [];
|
||||
@@ -28,7 +30,7 @@ export default async function getAllFiles(catList) {
|
||||
);
|
||||
}
|
||||
let dirWork = splitFilesAndFolders(parents);
|
||||
let files = dirWork.files;
|
||||
// First run should only have directories. Is there a reason this could change in the future?
|
||||
let dirs = dirWork.directories;
|
||||
let fetchTasks = [];
|
||||
let resolvedFetchTasks = [];
|
||||
@@ -141,8 +143,9 @@ export default async function getAllFiles(catList) {
|
||||
var elapsed = parseHrtimeToSeconds(process.hrtime(startTime));
|
||||
var m = Math.floor(elapsed / 60);
|
||||
var s = Math.floor(elapsed % 60);
|
||||
console.log(`\nFinished crawling Myrient in ${m}m${s}s.`);
|
||||
console.log(`\nFinished crawling Myrient in ${proctime.elapsed()}.`);
|
||||
await piscina.close();
|
||||
await optimizeDatabaseKws();
|
||||
return fileCount;
|
||||
}
|
||||
|
||||
@@ -204,9 +207,4 @@ function singleLineStatus(str) {
|
||||
} else {
|
||||
console.log(str);
|
||||
}
|
||||
}
|
||||
|
||||
function parseHrtimeToSeconds(hrtime) {
|
||||
var seconds = (hrtime[0] + hrtime[1] / 1e9).toFixed(3);
|
||||
return seconds;
|
||||
}
|
||||
}
|
||||
@@ -51,7 +51,7 @@ export async function parseOutFile(data) {
|
||||
path: data.url + path,
|
||||
size: size,
|
||||
category: category,
|
||||
hidden: `${category.replaceAll(' ', '')} ${cats.subCat.replaceAll(' ', '')}`,
|
||||
subcategories: `${cats.subCat.replaceAll(' ', '')}`,
|
||||
type: findType(fullName, data.catList),
|
||||
date: innertext(file.querySelector(".date").innerHTML).trim(),
|
||||
region: findRegion(fullName, data.catList),
|
||||
|
||||
41
lib/json/relatedkeywords/categories.json
Normal file
41
lib/json/relatedkeywords/categories.json
Normal file
@@ -0,0 +1,41 @@
|
||||
[
|
||||
[
|
||||
"supernintendoentertainmentsystem",
|
||||
"snes",
|
||||
"super famicom",
|
||||
"family computer"
|
||||
],
|
||||
["nintendoentertainmentsystem", "nes", "famicom", "family computer"],
|
||||
["playstation", "ps", "play station"],
|
||||
["playstation1", "ps1", "playstation 1", "psone", "one"],
|
||||
["playstation2", "ps2", "playstation 2"],
|
||||
["playstation3", "ps3", "playstation 3"],
|
||||
["playstationmobile", "psm", "mobile"],
|
||||
["xbox360", "x360", "xbox"],
|
||||
["xbox", "xb"],
|
||||
["famicomdisksystem", "fds", "famicom disk system"],
|
||||
["gameboyadvance", "gba", "gameboy advance", "game boy"],
|
||||
["gameboycolor", "gbc", "gameboy color", "game boy"],
|
||||
["gameboy", "gb", "gameboy", "game boy"],
|
||||
["gamecube", "gc", "game cube", "dolphin"],
|
||||
["megadrive", "md", "megadrive", "mega drive", "genesis"],
|
||||
["dreamcast", "dc", "dream cast"],
|
||||
["playstationvita", "psv", "playstation vita"],
|
||||
["playstationnetwork", "psn", "playstation network"],
|
||||
["nintendo switch", "switch", "nx"],
|
||||
["nintendo 3ds", "3ds", "three ds", "3d dual screen"],
|
||||
["nintendo ds", "ds", "dual screen"],
|
||||
["nintendo 64", "n64", "ultra 64"],
|
||||
["wiiu", "wii u"],
|
||||
["atari 2600", "vcs", "video computer system"],
|
||||
["playstationvr", "psvr", "playstation vr"],
|
||||
["pc engine", "pcengine", "turbografx", "turbografx-16"],
|
||||
["neogeo", "neo geo", "mvs"],
|
||||
["xboxone", "xone", "xbox one"],
|
||||
["xboxseriesx", "xsx", "xbox series x"],
|
||||
["xboxseriess", "xss", "xbox series s"],
|
||||
["amiibo", "nfc figure"],
|
||||
["mastersystem", "ms", "master system"],
|
||||
["wii", "revolution"],
|
||||
["appple II", "apple 2"]
|
||||
]
|
||||
14
lib/json/relatedkeywords/names.json
Normal file
14
lib/json/relatedkeywords/names.json
Normal file
@@ -0,0 +1,14 @@
|
||||
[
|
||||
["vmu", "visual memory unit"],
|
||||
["cd", "compact disc"],
|
||||
["bd", "blu-ray", "blu ray"],
|
||||
["hd", "high definition"],
|
||||
["pdf", "portable document format"],
|
||||
["dlc", "downloadable content"],
|
||||
["byteswapped", "byte swapped"],
|
||||
["bigendian", "big endian"],
|
||||
["littleendian", "little endian"],
|
||||
["pc88", "pc-88", "pc 88"],
|
||||
["dvd", "digital video disc", "digital versatile disc"],
|
||||
["bros", "brothers", "bros."]
|
||||
]
|
||||
4
lib/json/relatedkeywords/regions.json
Normal file
4
lib/json/relatedkeywords/regions.json
Normal file
@@ -0,0 +1,4 @@
|
||||
[
|
||||
["uk", "united kingdom"],
|
||||
["usa", "united states of america"]
|
||||
]
|
||||
@@ -24,6 +24,21 @@ export default function (sequelize) {
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: false
|
||||
},
|
||||
subcategories: {
|
||||
type: DataTypes.TEXT
|
||||
},
|
||||
filenamekws: {
|
||||
type: DataTypes.TEXT,
|
||||
},
|
||||
categorykws: {
|
||||
type: DataTypes.TEXT,
|
||||
},
|
||||
subcategorieskws: {
|
||||
type: DataTypes.TEXT,
|
||||
},
|
||||
regionkws: {
|
||||
type: DataTypes.TEXT,
|
||||
},
|
||||
type: {
|
||||
type: DataTypes.TEXT
|
||||
},
|
||||
|
||||
@@ -1,40 +1,39 @@
|
||||
import debugPrint from './debugprint.js'
|
||||
import { search as elasticSearch, getSuggestions as elasticSuggestions } from './services/elasticsearch.js'
|
||||
import { File } from './models/index.js'
|
||||
import debugPrint from "./debugprint.js";
|
||||
import {
|
||||
search as elasticSearch,
|
||||
getSuggestions as elasticSuggestions,
|
||||
} from "./services/elasticsearch.js";
|
||||
import { File } from "./models/index.js";
|
||||
|
||||
export default class Searcher {
|
||||
constructor(fields) {
|
||||
this.fields = [...fields]
|
||||
this.indexing = false
|
||||
}
|
||||
constructor(fields) {
|
||||
this.fields = [...fields];
|
||||
this.indexing = false;
|
||||
}
|
||||
|
||||
async findAllMatches(query, options) {
|
||||
try {
|
||||
return await elasticSearch(query, options)
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return { items: [], elapsed: 0 }
|
||||
}
|
||||
async findAllMatches(query, options) {
|
||||
try {
|
||||
return await elasticSearch(query, options);
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
return { items: [], elapsed: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async getSuggestions(query, options) {
|
||||
try {
|
||||
return await elasticSuggestions(query, options)
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return []
|
||||
}
|
||||
async getSuggestions(query, options) {
|
||||
try {
|
||||
return await elasticSuggestions(query, options);
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
findIndex(id) {
|
||||
return File.findByPk(id)
|
||||
}
|
||||
findIndex(id) {
|
||||
return File.findByPk(id);
|
||||
}
|
||||
|
||||
async getIndexSize() {
|
||||
return await File.count()
|
||||
}
|
||||
|
||||
get termCount() {
|
||||
return 0 // Not applicable with Elasticsearch
|
||||
}
|
||||
}
|
||||
async getIndexSize() {
|
||||
return await File.count();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
{
|
||||
"StringAssoc": [
|
||||
["supernintendoentertainmentsystem", "snes", "super famicom", "family computer"],
|
||||
["nintendoentertainmentsystem", "nes", "famicom", "family computer"],
|
||||
["playstation", "ps", "play station"],
|
||||
["playstation1", "ps1", "playstation 1", "psone", "one"],
|
||||
["playstation2", "ps2", "playstation 2"],
|
||||
["playstation3", "ps3", "playstation 3"],
|
||||
["playstationmobile", "psm", "mobile"],
|
||||
["xbox360", "x360", "xbox"],
|
||||
["xbox", "xb"],
|
||||
["famicomdisksystem", "fds", "famicom disk system"],
|
||||
["gameboyadvance", "gba", "gameboy advance"],
|
||||
["gameboycolor", "gbc", "gameboy color"],
|
||||
["gameboy", "gb", "gameboy"],
|
||||
["gamecube", "gc", "game cube", "dolphin"],
|
||||
["megadrive", "md", "megadrive", "mega drive"],
|
||||
["dreamcast", "dc", "dream case"],
|
||||
["playstationvita", "psv", "playstation vita"],
|
||||
["playstationnetwork", "psn", "playstation network"],
|
||||
["uk", "united kingdom"],
|
||||
["usa", "united states of america"],
|
||||
["vmu", "visual memory unit"],
|
||||
["cd", "compact disc"],
|
||||
["bd", "blu-ray", "blu ray"],
|
||||
["hd", "high definition"],
|
||||
["pdf", "portable document format"],
|
||||
["dlc", "downloadable content"],
|
||||
["byteswapped", "byte swapped"],
|
||||
["bigendian", "big endian"],
|
||||
["littleendian", "little endian"],
|
||||
["pc88", "pc-88", "pc 88"],
|
||||
["dvd", "digital video disc", "digital versatile disc"],
|
||||
["mastersystem", "ms", "master system"],
|
||||
["wii", "revolution"],
|
||||
["bros", "brothers", "bros."],
|
||||
["bros.", "brothers", "bros"],
|
||||
["playstation4", "ps4", "playstation 4"],
|
||||
["playstation5", "ps5", "playstation 5"],
|
||||
["playstationportable", "psp", "playstation portable"],
|
||||
["nintendoswitch", "switch", "nx"],
|
||||
["nintendo3ds", "3ds", "three ds"],
|
||||
["nintendods", "ds", "dual screen"],
|
||||
["nintendo64", "n64", "ultra 64"],
|
||||
["wiiu", "wii u"],
|
||||
["atari2600", "vcs", "video computer system"],
|
||||
["segasaturn", "saturn"],
|
||||
["genesis", "mega drive", "sega genesis"],
|
||||
["virtualboy", "vb", "virtual boy"],
|
||||
["playstationvr", "psvr", "playstation vr"],
|
||||
["pcengine", "pc engine", "turbografx"],
|
||||
["neogeo", "neo geo", "mvs"],
|
||||
["xboxone", "xone", "xbox one"],
|
||||
["xboxseriesx", "xsx", "xbox series x"],
|
||||
["xboxseriess", "xss", "xbox series s"],
|
||||
["amiibo", "nfc figure"]
|
||||
]
|
||||
}
|
||||
@@ -69,6 +69,18 @@ export async function initElasticsearch() {
|
||||
region: {
|
||||
type: 'text',
|
||||
analyzer: 'standard'
|
||||
},
|
||||
filenamekws: {
|
||||
type: 'text',
|
||||
analyzer: 'standard'
|
||||
},
|
||||
categorykws: {
|
||||
type: 'text',
|
||||
analyzer: 'standard'
|
||||
},
|
||||
regionkws: {
|
||||
type: 'text',
|
||||
analyzer: 'standard'
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -102,7 +114,10 @@ export async function bulkIndexFiles(files) {
|
||||
filename: file.filename,
|
||||
category: file.category,
|
||||
type: file.type,
|
||||
region: file.region
|
||||
region: file.region,
|
||||
filenamekws: file.filenamekws,
|
||||
categorykws: file.categorykws,
|
||||
regionkws: file.regionkws
|
||||
}
|
||||
]);
|
||||
|
||||
@@ -128,6 +143,10 @@ export async function bulkIndexFiles(files) {
|
||||
}
|
||||
|
||||
export async function search(query, options) {
|
||||
//add kws for selected fields
|
||||
for(let field in options.fields){
|
||||
options.fields.push(options.fields[field] + 'kws')
|
||||
}
|
||||
const searchQuery = {
|
||||
index: INDEX_NAME,
|
||||
body: {
|
||||
@@ -204,7 +223,7 @@ function buildMustClauses(query, options) {
|
||||
multi_match: {
|
||||
query: term,
|
||||
fields: options.fields.map(field =>
|
||||
field === 'filename' ? `${field}^2` : field
|
||||
field === 'filename' || 'filenamekws' ? `${field}^2` : field
|
||||
),
|
||||
fuzziness: options.fuzzy || 0,
|
||||
type: 'best_fields'
|
||||
@@ -224,7 +243,7 @@ function buildShouldClauses(query, options) {
|
||||
multi_match: {
|
||||
query,
|
||||
fields: options.fields.map(field =>
|
||||
field === 'filename' ? `${field}^2` : field
|
||||
field === 'filename' || 'filenamekws' ? `${field}^2` : field
|
||||
),
|
||||
fuzziness: options.fuzzy || 0,
|
||||
type: 'best_fields'
|
||||
@@ -247,7 +266,7 @@ export async function getSuggestions(query, options) {
|
||||
query: {
|
||||
multi_match: {
|
||||
query,
|
||||
fields: ['filename^2', 'category'],
|
||||
fields: ['filename^2', 'filenamekws^2', 'category', 'categorykws'],
|
||||
fuzziness: 'AUTO',
|
||||
type: 'best_fields'
|
||||
}
|
||||
@@ -264,4 +283,28 @@ export async function getSuggestions(query, options) {
|
||||
console.error('Suggestion error:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export async function getSample(query, options){
|
||||
try {
|
||||
const response = await client.search({
|
||||
index: INDEX_NAME,
|
||||
body: {
|
||||
query: {
|
||||
match: {
|
||||
filename: query,
|
||||
}
|
||||
},
|
||||
_source: ['filename'],
|
||||
size: 30
|
||||
}
|
||||
});
|
||||
|
||||
return response.hits.hits.map(hit => ({
|
||||
sample: hit._source.filename
|
||||
}));
|
||||
} catch (error) {
|
||||
console.error('Sample error:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
16
lib/time.js
Normal file
16
lib/time.js
Normal file
@@ -0,0 +1,16 @@
|
||||
export class timer {
|
||||
constructor() {
|
||||
this.startTime = process.hrtime();
|
||||
}
|
||||
parseHrtimetoSeconds(hrtime) {
|
||||
var seconds = (hrtime[0] + hrtime[1] / 1e9).toFixed(3);
|
||||
return seconds;
|
||||
}
|
||||
elapsed() {
|
||||
let elapsed = this.parseHrtimetoSeconds(process.hrtime(this.startTime));
|
||||
let h = Math.floor(elapsed / 3600);
|
||||
let m = Math.floor(elapsed / 60);
|
||||
let s = Math.floor(elapsed % 60);
|
||||
return `${h ? h + "h" : ""}${m ? m + "m" : ""}${s + "s"}`;
|
||||
}
|
||||
}
|
||||
16
package-lock.json
generated
16
package-lock.json
generated
@@ -26,6 +26,7 @@
|
||||
"sanitize": "^2.1.2",
|
||||
"sequelize": "^6.37.1",
|
||||
"sequelize-cli": "^6.6.2",
|
||||
"to-words": "^4.5.1",
|
||||
"uuid": "^11.1.0"
|
||||
}
|
||||
},
|
||||
@@ -3576,6 +3577,15 @@
|
||||
"integrity": "sha512-bu9oCYYWC1iRjx+3UnAjqCsfrWNZV1ghNQf49b3w5xE8J/tNShHTzp5syWJfwGH+pxUgTTLUnzHnfuydW7wmbg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/to-words": {
|
||||
"version": "4.5.1",
|
||||
"resolved": "https://registry.npmjs.org/to-words/-/to-words-4.5.1.tgz",
|
||||
"integrity": "sha512-/Yp5UX72RzSaOk+KvUglc/uXgIVjWN3WKqeTouz/izWNkCDHjo1Tmhz9UA7VUtNMUBvkyt59NUGfjL5FBEEDqg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/toidentifier": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
|
||||
@@ -3656,9 +3666,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "6.21.1",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-6.21.1.tgz",
|
||||
"integrity": "sha512-q/1rj5D0/zayJB2FraXdaWxbhWiNKDvu8naDT2dl1yTlvJp4BLtOcp2a5BvgGNQpYYJzau7tf1WgKv3b+7mqpQ==",
|
||||
"version": "6.21.3",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-6.21.3.tgz",
|
||||
"integrity": "sha512-gBLkYIlEnSp8pFbT64yFgGE6UIB9tAkhukC23PmMDCe5Nd+cRqKxSjw5y54MK2AZMgZfJWMaNE4nYUHgi1XEOw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18.17"
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
"sanitize": "^2.1.2",
|
||||
"sequelize": "^6.37.1",
|
||||
"sequelize-cli": "^6.6.2",
|
||||
"to-words": "^4.5.1",
|
||||
"uuid": "^11.1.0"
|
||||
},
|
||||
"type": "module"
|
||||
|
||||
227
server.js
227
server.js
@@ -8,21 +8,24 @@ import http from "http";
|
||||
import sanitize from "sanitize";
|
||||
import debugPrint from "./lib/debugprint.js";
|
||||
import compression from "compression";
|
||||
import cookieParser from 'cookie-parser';
|
||||
import { generateAsciiArt } from './lib/asciiart.js';
|
||||
import { getEmulatorConfig, isEmulatorCompatible, isNonGameContent } from './lib/emulatorConfig.js';
|
||||
import fetch from 'node-fetch';
|
||||
import { initDB, File, QueryCount } from './lib/database.js';
|
||||
import { initElasticsearch } from './lib/services/elasticsearch.js';
|
||||
import i18n, { locales } from './config/i18n.js';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import cookieParser from "cookie-parser";
|
||||
import { generateAsciiArt } from "./lib/asciiart.js";
|
||||
import {
|
||||
getEmulatorConfig,
|
||||
isEmulatorCompatible,
|
||||
isNonGameContent,
|
||||
} from "./lib/emulatorConfig.js";
|
||||
import fetch from "node-fetch";
|
||||
import { initDB, File, QueryCount } from "./lib/database.js";
|
||||
import { initElasticsearch } from "./lib/services/elasticsearch.js";
|
||||
import i18n, { locales } from "./config/i18n.js";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { optimizeDatabaseKws } from "./lib/dboptimize.js";
|
||||
|
||||
let categoryListPath = "./lib/categories.json"
|
||||
let searchAlikesPath = './lib/searchalikes.json'
|
||||
let nonGameTermsPath = './lib/nonGameTerms.json'
|
||||
let emulatorsPath = './lib/emulators.json'
|
||||
let categoryListPath = "./lib/categories.json";
|
||||
let nonGameTermsPath = "./lib/nonGameTerms.json";
|
||||
let emulatorsPath = "./lib/emulators.json";
|
||||
let categoryList = await FileHandler.parseJsonFile(categoryListPath);
|
||||
global.searchAlikes = await FileHandler.parseJsonFile(searchAlikesPath)
|
||||
let nonGameTerms = await FileHandler.parseJsonFile(nonGameTermsPath);
|
||||
let emulatorsData = await FileHandler.parseJsonFile(emulatorsPath);
|
||||
let crawlTime = 0;
|
||||
@@ -36,7 +39,7 @@ await initElasticsearch();
|
||||
|
||||
// Get initial counts
|
||||
fileCount = await File.count();
|
||||
crawlTime = (await File.max('updatedAt'))?.getTime() || 0;
|
||||
crawlTime = (await File.max("updatedAt"))?.getTime() || 0;
|
||||
queryCount = (await QueryCount.findOne())?.count || 0;
|
||||
|
||||
let searchFields = ["filename", "category", "type", "region"];
|
||||
@@ -65,7 +68,7 @@ let search = new Searcher(searchFields);
|
||||
async function getFilesJob() {
|
||||
console.log("Updating the file list.");
|
||||
fileCount = await getAllFiles(categoryList);
|
||||
if(!fileCount) {
|
||||
if (!fileCount) {
|
||||
console.log("File update failed");
|
||||
return;
|
||||
}
|
||||
@@ -81,14 +84,13 @@ let defaultOptions = {
|
||||
crawlTime: crawlTime,
|
||||
queryCount: queryCount,
|
||||
fileCount: fileCount,
|
||||
termCount: 0,
|
||||
generateAsciiArt: generateAsciiArt,
|
||||
isEmulatorCompatible: isEmulatorCompatible,
|
||||
isNonGameContent: isNonGameContent,
|
||||
nonGameTerms: nonGameTerms
|
||||
nonGameTerms: nonGameTerms,
|
||||
};
|
||||
|
||||
function updateDefaults(){
|
||||
function updateDefaults() {
|
||||
defaultOptions.crawlTime = crawlTime;
|
||||
defaultOptions.queryCount = queryCount;
|
||||
defaultOptions.fileCount = fileCount;
|
||||
@@ -98,15 +100,15 @@ let app = express();
|
||||
let server = http.createServer(app);
|
||||
|
||||
app.use((req, res, next) => {
|
||||
res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
|
||||
res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');
|
||||
res.setHeader("Cross-Origin-Opener-Policy", "same-origin");
|
||||
res.setHeader("Cross-Origin-Embedder-Policy", "require-corp");
|
||||
next();
|
||||
});
|
||||
|
||||
app.use(sanitize.middleware);
|
||||
app.use(compression())
|
||||
app.use(express.json())
|
||||
app.use(cookieParser())
|
||||
app.use(compression());
|
||||
app.use(express.json());
|
||||
app.use(cookieParser());
|
||||
app.set("view engine", "ejs");
|
||||
|
||||
app.use((req, res, next) => {
|
||||
@@ -137,20 +139,20 @@ app.use((req, res, next) => {
|
||||
|
||||
// Fallback to English
|
||||
if (!lang) {
|
||||
lang = 'en';
|
||||
lang = "en";
|
||||
}
|
||||
req.setLocale(lang);
|
||||
res.locals.locale = lang;
|
||||
|
||||
res.locals.availableLocales = locales;
|
||||
|
||||
res.cookie('lang', lang, { maxAge: 365 * 24 * 60 * 60 * 1000 }); // 1 year
|
||||
res.cookie("lang", lang, { maxAge: 365 * 24 * 60 * 60 * 1000 }); // 1 year
|
||||
|
||||
next();
|
||||
});
|
||||
|
||||
// Add helper function to all templates
|
||||
app.locals.__ = function() {
|
||||
app.locals.__ = function () {
|
||||
return i18n.__.apply(this, arguments);
|
||||
};
|
||||
|
||||
@@ -161,9 +163,9 @@ app.get("/", function (req, res) {
|
||||
|
||||
app.get("/search", async function (req, res) {
|
||||
let query = req.query.q ? req.query.q : "";
|
||||
let pageNum = parseInt(req.query.p)
|
||||
let urlPrefix = encodeURI(`/search?s=${req.query.s}&q=${req.query.q}&p=`)
|
||||
pageNum = pageNum ? pageNum : 1
|
||||
let pageNum = parseInt(req.query.p);
|
||||
let urlPrefix = encodeURI(`/search?s=${req.query.s}&q=${req.query.q}&p=`);
|
||||
pageNum = pageNum ? pageNum : 1;
|
||||
let settings = {};
|
||||
try {
|
||||
settings = req.query.s ? JSON.parse(atob(req.query.s)) : defaultSettings;
|
||||
@@ -189,13 +191,10 @@ app.get("/search", async function (req, res) {
|
||||
}
|
||||
let results = await search.findAllMatches(query, settings);
|
||||
debugPrint(results);
|
||||
if(results.items.length && pageNum == 1){
|
||||
if (results.items.length && pageNum == 1) {
|
||||
queryCount += 1;
|
||||
await QueryCount.update(
|
||||
{ count: queryCount },
|
||||
{ where: { id: 1 } }
|
||||
);
|
||||
updateDefaults()
|
||||
await QueryCount.update({ count: queryCount }, { where: { id: 1 } });
|
||||
updateDefaults();
|
||||
}
|
||||
let options = {
|
||||
query: query,
|
||||
@@ -203,7 +202,7 @@ app.get("/search", async function (req, res) {
|
||||
pageNum: pageNum,
|
||||
indexing: search.indexing,
|
||||
urlPrefix: urlPrefix,
|
||||
settings: settings
|
||||
settings: settings,
|
||||
};
|
||||
let page = "results";
|
||||
options = buildOptions(page, options);
|
||||
@@ -213,7 +212,9 @@ app.get("/search", async function (req, res) {
|
||||
app.get("/lucky", async function (req, res) {
|
||||
let results = { items: [] };
|
||||
if (req.query.q) {
|
||||
let settings = req.query.s ? JSON.parse(atob(req.query.s)) : defaultSettings;
|
||||
let settings = req.query.s
|
||||
? JSON.parse(atob(req.query.s))
|
||||
: defaultSettings;
|
||||
results = await search.findAllMatches(req.query.q, settings);
|
||||
debugPrint(results);
|
||||
}
|
||||
@@ -223,17 +224,14 @@ app.get("/lucky", async function (req, res) {
|
||||
const count = await File.count();
|
||||
const randomId = Math.floor(Math.random() * count);
|
||||
const luckyFile = await File.findOne({
|
||||
offset: randomId
|
||||
offset: randomId,
|
||||
});
|
||||
debugPrint(`${randomId}: ${luckyFile?.path}`);
|
||||
res.redirect(luckyFile?.path || '/');
|
||||
res.redirect(luckyFile?.path || "/");
|
||||
}
|
||||
queryCount += 1;
|
||||
await QueryCount.update(
|
||||
{ count: queryCount },
|
||||
{ where: { id: 1 } }
|
||||
);
|
||||
updateDefaults()
|
||||
await QueryCount.update({ count: queryCount }, { where: { id: 1 } });
|
||||
updateDefaults();
|
||||
});
|
||||
|
||||
app.get("/settings", function (req, res) {
|
||||
@@ -243,18 +241,21 @@ app.get("/settings", function (req, res) {
|
||||
res.render(indexPage, options);
|
||||
});
|
||||
|
||||
app.post("/suggest", async function(req, res){
|
||||
if(!req.body){
|
||||
return
|
||||
app.post("/suggest", async function (req, res) {
|
||||
if (!req.body) {
|
||||
return;
|
||||
}
|
||||
if(typeof req.body.query == 'undefined'){
|
||||
return
|
||||
if (typeof req.body.query == "undefined") {
|
||||
return;
|
||||
}
|
||||
let suggestions = await search.getSuggestions(req.body.query, defaultSettings)
|
||||
debugPrint(suggestions)
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
let suggestions = await search.getSuggestions(
|
||||
req.body.query,
|
||||
defaultSettings
|
||||
);
|
||||
debugPrint(suggestions);
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
res.end(JSON.stringify({ suggestions }));
|
||||
})
|
||||
});
|
||||
|
||||
app.get("/about", function (req, res) {
|
||||
let page = "about";
|
||||
@@ -263,8 +264,8 @@ app.get("/about", function (req, res) {
|
||||
|
||||
app.get("/play/:id", async function (req, res) {
|
||||
// Block access if emulator is disabled
|
||||
if (process.env.EMULATOR_ENABLED !== 'true') {
|
||||
res.redirect('/');
|
||||
if (process.env.EMULATOR_ENABLED !== "true") {
|
||||
res.redirect("/");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -272,14 +273,14 @@ app.get("/play/:id", async function (req, res) {
|
||||
let romFile = await search.findIndex(fileId);
|
||||
|
||||
if (!romFile) {
|
||||
res.redirect('/');
|
||||
res.redirect("/");
|
||||
return;
|
||||
}
|
||||
|
||||
let options = {
|
||||
romFile: romFile,
|
||||
emulatorConfig: getEmulatorConfig(romFile.category),
|
||||
isNonGame: isNonGameContent(romFile.filename, nonGameTerms)
|
||||
isNonGame: isNonGameContent(romFile.filename, nonGameTerms),
|
||||
};
|
||||
|
||||
let page = "emulator";
|
||||
@@ -289,48 +290,51 @@ app.get("/play/:id", async function (req, res) {
|
||||
|
||||
app.get("/proxy-rom/:id", async function (req, res, next) {
|
||||
// Block access if emulator is disabled
|
||||
if (process.env.EMULATOR_ENABLED !== 'true') {
|
||||
return next(new Error('Emulator feature is disabled'));
|
||||
if (process.env.EMULATOR_ENABLED !== "true") {
|
||||
return next(new Error("Emulator feature is disabled"));
|
||||
}
|
||||
|
||||
let fileId = parseInt(req.params.id);
|
||||
let romFile = await search.findIndex(fileId);
|
||||
|
||||
if (!romFile) {
|
||||
return next(new Error('ROM not found'));
|
||||
return next(new Error("ROM not found"));
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(romFile.path);
|
||||
const contentLength = response.headers.get('content-length');
|
||||
const contentLength = response.headers.get("content-length");
|
||||
|
||||
res.setHeader('Content-Type', 'application/zip');
|
||||
res.setHeader('Content-Length', contentLength);
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${romFile.filename}"`);
|
||||
res.setHeader("Content-Type", "application/zip");
|
||||
res.setHeader("Content-Length", contentLength);
|
||||
res.setHeader(
|
||||
"Content-Disposition",
|
||||
`attachment; filename="${romFile.filename}"`
|
||||
);
|
||||
|
||||
// Add all required cross-origin headers
|
||||
res.setHeader('Cross-Origin-Resource-Policy', 'same-origin');
|
||||
res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');
|
||||
res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
|
||||
res.setHeader("Cross-Origin-Resource-Policy", "same-origin");
|
||||
res.setHeader("Cross-Origin-Embedder-Policy", "require-corp");
|
||||
res.setHeader("Cross-Origin-Opener-Policy", "same-origin");
|
||||
|
||||
response.body.pipe(res);
|
||||
} catch (error) {
|
||||
console.error('Error proxying ROM:', error);
|
||||
console.error("Error proxying ROM:", error);
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/proxy-bios", async function (req, res, next) {
|
||||
// Block access if emulator is disabled
|
||||
if (process.env.EMULATOR_ENABLED !== 'true') {
|
||||
return next(new Error('Emulator feature is disabled'));
|
||||
if (process.env.EMULATOR_ENABLED !== "true") {
|
||||
return next(new Error("Emulator feature is disabled"));
|
||||
}
|
||||
|
||||
const biosUrl = req.query.url;
|
||||
|
||||
// Validate that URL is from GitHub
|
||||
if (!biosUrl || !biosUrl.startsWith('https://github.com')) {
|
||||
return next(new Error('Invalid BIOS URL - only GitHub URLs are allowed'));
|
||||
if (!biosUrl || !biosUrl.startsWith("https://github.com")) {
|
||||
return next(new Error("Invalid BIOS URL - only GitHub URLs are allowed"));
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -340,37 +344,38 @@ app.get("/proxy-bios", async function (req, res, next) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const contentLength = response.headers.get('content-length');
|
||||
const contentType = response.headers.get('content-type');
|
||||
const contentLength = response.headers.get("content-length");
|
||||
const contentType = response.headers.get("content-type");
|
||||
|
||||
res.setHeader('Content-Type', contentType || 'application/octet-stream');
|
||||
res.setHeader('Content-Length', contentLength);
|
||||
res.setHeader("Content-Type", contentType || "application/octet-stream");
|
||||
res.setHeader("Content-Length", contentLength);
|
||||
|
||||
// Add all required cross-origin headers
|
||||
res.setHeader('Cross-Origin-Resource-Policy', 'same-origin');
|
||||
res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');
|
||||
res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
|
||||
res.setHeader("Cross-Origin-Resource-Policy", "same-origin");
|
||||
res.setHeader("Cross-Origin-Embedder-Policy", "require-corp");
|
||||
res.setHeader("Cross-Origin-Opener-Policy", "same-origin");
|
||||
|
||||
response.body.pipe(res);
|
||||
} catch (error) {
|
||||
console.error('Error proxying BIOS:', error);
|
||||
console.error("Error proxying BIOS:", error);
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
// Proxy route for EmulatorJS content
|
||||
app.get('/emulatorjs/*', async function (req, res, next) {
|
||||
app.get("/emulatorjs/*", async function (req, res, next) {
|
||||
try {
|
||||
// Extract the path after /emulatorjs/
|
||||
const filePath = req.path.replace(/^\/emulatorjs\//, '');
|
||||
const filePath = req.path.replace(/^\/emulatorjs\//, "");
|
||||
|
||||
// Support both stable and latest paths
|
||||
const emulatorJsUrl = `https://cdn.emulatorjs.org/stable/${filePath}`;
|
||||
|
||||
const response = await fetch(emulatorJsUrl, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36'
|
||||
}
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -378,33 +383,33 @@ app.get('/emulatorjs/*', async function (req, res, next) {
|
||||
}
|
||||
|
||||
// Copy content type and length
|
||||
const contentType = response.headers.get('content-type');
|
||||
const contentType = response.headers.get("content-type");
|
||||
if (contentType) {
|
||||
res.setHeader('Content-Type', contentType);
|
||||
res.setHeader("Content-Type", contentType);
|
||||
}
|
||||
|
||||
const contentLength = response.headers.get('content-length');
|
||||
const contentLength = response.headers.get("content-length");
|
||||
if (contentLength) {
|
||||
res.setHeader('Content-Length', contentLength);
|
||||
res.setHeader("Content-Length", contentLength);
|
||||
}
|
||||
|
||||
// Set special headers for WASM files
|
||||
if (filePath.endsWith('.wasm')) {
|
||||
res.setHeader('Content-Type', 'application/wasm');
|
||||
if (filePath.endsWith(".wasm")) {
|
||||
res.setHeader("Content-Type", "application/wasm");
|
||||
}
|
||||
|
||||
// Special handling for JavaScript files
|
||||
if (filePath.endsWith('.js')) {
|
||||
res.setHeader('Content-Type', 'application/javascript');
|
||||
if (filePath.endsWith(".js")) {
|
||||
res.setHeader("Content-Type", "application/javascript");
|
||||
}
|
||||
|
||||
res.setHeader('Cross-Origin-Resource-Policy', 'same-origin');
|
||||
res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');
|
||||
res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
|
||||
res.setHeader("Cross-Origin-Resource-Policy", "same-origin");
|
||||
res.setHeader("Cross-Origin-Embedder-Policy", "require-corp");
|
||||
res.setHeader("Cross-Origin-Opener-Policy", "same-origin");
|
||||
|
||||
response.body.pipe(res);
|
||||
} catch (error) {
|
||||
console.error('Error proxying EmulatorJS content:', error);
|
||||
console.error("Error proxying EmulatorJS content:", error);
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
@@ -423,14 +428,15 @@ app.get("/proxy-image", async function (req, res, next) {
|
||||
const imageUrl = req.query.url;
|
||||
|
||||
if (!imageUrl) {
|
||||
return next(new Error('No image URL provided'));
|
||||
return next(new Error("No image URL provided"));
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(imageUrl, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36'
|
||||
}
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -438,26 +444,26 @@ app.get("/proxy-image", async function (req, res, next) {
|
||||
}
|
||||
|
||||
// Copy content type
|
||||
const contentType = response.headers.get('content-type');
|
||||
const contentType = response.headers.get("content-type");
|
||||
if (contentType) {
|
||||
res.setHeader('Content-Type', contentType);
|
||||
res.setHeader("Content-Type", contentType);
|
||||
}
|
||||
|
||||
const contentLength = response.headers.get('content-length');
|
||||
const contentLength = response.headers.get("content-length");
|
||||
if (contentLength) {
|
||||
res.setHeader('Content-Length', contentLength);
|
||||
res.setHeader("Content-Length", contentLength);
|
||||
}
|
||||
|
||||
response.body.pipe(res);
|
||||
} catch (error) {
|
||||
console.error('Error proxying image:', error);
|
||||
console.error("Error proxying image:", error);
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
// 404 handler
|
||||
app.use((req, res, next) => {
|
||||
const err = new Error('Page Not Found');
|
||||
const err = new Error("Page Not Found");
|
||||
err.status = 404;
|
||||
next(err);
|
||||
});
|
||||
@@ -465,20 +471,20 @@ app.use((req, res, next) => {
|
||||
// Error handling middleware
|
||||
app.use((err, req, res, next) => {
|
||||
const status = err.status || 500;
|
||||
const message = err.message || 'An unexpected error occurred';
|
||||
const message = err.message || "An unexpected error occurred";
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
if (process.env.NODE_ENV !== "production") {
|
||||
console.error(err);
|
||||
}
|
||||
|
||||
res.status(status);
|
||||
|
||||
res.render('pages/error', {
|
||||
res.render("pages/error", {
|
||||
status,
|
||||
message,
|
||||
stack: process.env.NODE_ENV !== 'production' ? err.stack : null,
|
||||
stack: process.env.NODE_ENV !== "production" ? err.stack : null,
|
||||
req,
|
||||
requestId: req.requestId
|
||||
requestId: req.requestId,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -502,3 +508,4 @@ if (
|
||||
}
|
||||
|
||||
cron.schedule("0 30 2 * * *", getFilesJob);
|
||||
optimizeDatabaseKws();
|
||||
|
||||
@@ -4,8 +4,6 @@
|
||||
<div class="stats"> | </div>
|
||||
<div id="file-count" class="stats"><%= __('footer.files') %></div>
|
||||
<div class="stats"> | </div>
|
||||
<div id="term-count" class="stats"><%= __('footer.terms') %></div>
|
||||
<div class="stats"> | </div>
|
||||
<div id="crawl-time" class="stats"><%= __('footer.last_crawl') %></div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -20,5 +18,4 @@
|
||||
document.getElementById('crawl-time').innerText += ` ${timeConverter('<%= crawlTime %>')}`
|
||||
document.getElementById('file-count').innerText += ` ${(<%= fileCount %>).toLocaleString(undefined)}`
|
||||
document.getElementById('query-count').innerText += ` ${(<%= queryCount %>).toLocaleString(undefined)}`
|
||||
document.getElementById('term-count').innerText += ` ${(<%= termCount %>).toLocaleString(undefined)}`
|
||||
</script>
|
||||
|
||||
Reference in New Issue
Block a user