Firehawk52-backup/Myrient-Search-Engine
28
0
Fork 0
mirror of https://github.com/alexankitty/Myrient-Search-Engine.git synced 2026-01-15 08:23:18 -03:00
Code Issues Packages Projects Releases Wiki Activity

idk man I did a ton of stuff. plus prelim categories wooh

Browse Source
This commit is contained in:
Alexandra
2024-10-16 03:09:31 -06:00
parent 69757f3ff7
commit 50d45b041b
7 changed files with 504 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.env
View File

@@ -1,2 +1,3 @@
PORT=8062
FUZZY_LENIENCY=0.2
FUZZY_LENIENCY=0.2
DEBUG=0

394
lib/categories.json Normal file
View File

@@ -0,0 +1,394 @@
{
"Categories": {
"Acorn": [
"Archimedes",
"Atom",
"RISC"
],
"ACT": [
"Apricot PC Xi"
],
"Amstrad": [
"CPC"
],
"Analogue": [
"Pocket"
],
"APF": [
"Imagination Machine",
"MP-1000"
],
"Apple": [
"I",
"II Plus",
"II",
"IIGS",
"IIe",
"Macintosh"
],
"Apple-Bandai": [
"Pippin",
"Playdia"
],
"Arcade": [
"PC-based"
],
"Arduboy Inc": [
"Arduboy"
],
"Atari": [
"2600",
"5200",
"7800",
"8-bit",
"Jaguar",
"Lynx",
"ST"
],
"Bally": [
"Astrocade"
],
"Bandai": [
"Design Master Denshi Mangajuku",
"Gundam RX-78",
"WonderSwan Color",
"WonderSwan"
],
"Benesse": [
"Pocket Challenge V2",
"Pocket Challenge W"
],
"Bit Corporation": [
"Gamate"
],
"Casio": [
"Loopy",
"PV-1000"
],
"Coleco": [
"ColecoVision"
],
"Commodore": [
"Amiga",
"Commodore 64",
"Plus-4",
"VIC-20"
],
"Digital Media Cartridge": [
"Firecore"
],
"Emerson": [
"Arcadia 2001"
],
"Entex": [
"Adventure Vision"
],
"Epoch": [
"Game Pocket Computer",
"Super Cassette Vision"
],
"Fairchild": [
"Channel F"
],
"Fujitsu": [
"FM Towns",
"FM-7",
"FM-Towns",
"FMR50"
],
"Fukutake Publishing": [
"StudyBox"
],
"Funtech": [
"Super Acan"
],
"funworld": [
"Photo Play"
],
"GamePark": [
"GP2X",
"GP32"
],
"GCE": [
"Vectrex"
],
"Google": [
"Android"
],
"Hartung": [
"Game Master"
],
"Hasbro": [
"VideoNow Color",
"VideoNow Jr",
"VideoNow XP",
"VideoNow",
"iON"
],
"Hitachi": [
"S1"
],
"IBM": [
"PC Compatible",
"PC and Compatibles"
],
"Incredible Technologies": [
"Eagle"
],
"Interton": [
"VC 4000"
],
"iQue": [
"iQue"
],
"Konami": [
"FireBeat",
"M2",
"Picno",
"System 573",
"System GV",
"e-Amusement"
],
"LeapFrog": [
"Explorer",
"LeapPad",
"Leapster"
],
"Luxor": [
"ABC 800"
],
"Magnavox": [
"Odyssey 2"
],
"Mattel": [
"Fisher-Price iXL",
"HyperScan",
"Intellivision"
],
"Memorex": [
"Visual Information System"
],
"Merit": [
"Megatouch"
],
"Microsoft": [
"MSX",
"MSX2",
"Pocket PC",
"Xbox 360",
"Xbox"
],
"Milton-Bradley": [
"Omni"
],
"Mobile": [
"J2ME",
"Palm OS",
"Pocket PC",
"Symbian"
],
"Namco": [
"System 246"
],
"Navisoft": [
"Naviken 2.1"
],
"NEC": [
"PC Engine",
"PC-88",
"PC-98",
"PC-FX"
],
"Nichibutsu": [
"My Vision"
],
"Nintendo": [
"3DS",
"64",
"64DD",
"DS",
"DSi",
"Family BASIC",
"Family Computer Disk System",
"Family Computer Network System",
"Game & Watch",
"Game Boy Advance",
"Game Boy Color",
"Game Boy",
"Gamecube",
"Kiosk Video Compact Flash",
"Misc",
"New Nintendo 3DS",
"Nintendo Entertainment System",
"Pokemon Mini",
"SDKs",
"Satellaview",
"Sufami Turbo",
"Super Nintendo Entertainment System",
"Triforce",
"VirtualBoy",
"Wallpapers",
"Wii U",
"Wii",
"amiibo"
],
"Nokia": [
"N-Gage"
],
"Others": [
"Audio",
"BD-Video",
"CD-ROM",
"DVD-ROM",
"DVD-Video",
"HD DVD",
"HD DVD-Video",
"HumblePlay",
"Palm",
"Photo CD",
"Pocket PC",
"Project EGG",
"Video CD",
"Video Game Documents",
"Video Game Magazine Scans",
"Video Game OSTs",
"Video Game Scans",
"itch.io"
],
"Ouya": [
"Ouya"
],
"Panasonic": [
"3DO",
"M2"
],
"Philips": [
"CD-i",
"videopac+"
],
"RCA": [
"Studio II"
],
"Sanyo": [
"MBC-550"
],
"Sega": [
"32X",
"Beena",
"CD",
"Chihiro",
"Dreamcast",
"Game Gear",
"Lindbergh",
"Master System",
"Mega Drive",
"Naomi 2",
"Naomi",
"PICO",
"Prologue 21",
"RingEdge 2",
"RingEdge",
"SG-1000",
"Saturn"
],
"Seta": [
"Aleck64"
],
"Sharp": [
"MZ-2200",
"MZ-700",
"X1",
"X68000"
],
"Sinclair": [
"ZX Spectrum"
],
"SNK": [
"NeoGeo CD",
"NeoGeo Pocket Color",
"NeoGeo Pocket"
],
"Sony": [
"Electronic Book",
"PlayStation 2",
"PlayStation 3",
"PlayStation Mobile",
"PlayStation Portable",
"PlayStation Vita",
"PlayStation"
],
"Tab-Austria": [
"Quizard"
],
"TeleNova": [
"Compis"
],
"Texas Instruments": [
"TI-99-4A"
],
"Tiger": [
"Game.com",
"Gizmondo"
],
"Tomy": [
"Kiss-Site"
],
"Toshiba": [
"Pasopia",
"Visicom"
],
"VM Labs": [
"NUON"
],
"VTech": [
"CreatiVision",
"V.Flash & V.Smile Pro",
"V.Smile"
],
"Watara": [
"SuperVision"
],
"Welback": [
"Mega Duck"
],
"Yamaha": [
"Copera"
],
"ZAPiT Games": [
"Game Wave Family Entertainment System"
],
"Zeebo": [
"Zeebo"
]
},
"Types": [
"BIOS",
"BigEndian",
"ByteSwapped",
"CBZ",
"DLC",
"Decrypted",
"Development Kit",
"Digital",
"Disc Keys",
"GameShark",
"Header",
"Headerless",
"LittleEndian",
"NKit",
"NoNpDrm",
"PDF",
"PS one Classics",
"PSN",
"PSX2PSP",
"RAW",
"Source Code",
"UMD Music",
"UMD Video",
"Updates and DLC",
"Updates",
"VPK",
"Visual Memory Unit",
"WUX"
]
}

28
lib/dirwalk.js
View File

@@ -1,5 +1,6 @@
import fetch from 'node-fetch'
import jsdom from "jsdom"
import innertext from 'innertext'
const { JSDOM } = jsdom
global.DOMParser = new JSDOM().window.DOMParser
@@ -11,13 +12,21 @@ export default async function getAllFiles(){
let files = dirWork.files
let dirs = dirWork.directories
while(dirs.length > 0) {
console.log(`Working on: ${dirs[0].name}`)
if(process.env.DEBUG == '1'){
console.log(`Working on: ${dirs[0].name}`)
}
let results = await getFilesAndFolders(dirs[0].path, dirs[0].name)
let working = splitFilesAndFolders(results)
if(working.files.length > 0) files.push(...working.files)
if(working.directories.length > 0) dirs.push(...working.directories)
dirs.shift()
console.log(`Directories Remaining: ${dirs.length}, Files Found: ${files.length}`)
let dirStatus = `Directories Remaining: ${dirs.length}, Files Found: ${files.length}`
if(process.env.DEBUG == '1'){
console.log(dirStatus)
}
else{
singleLineStatus(dirStatus)
}
}
return files
}
@@ -34,8 +43,8 @@ async function getFilesAndFolders(url, base = ""){
const fileList = fileHtml.getElementById('list').rows
for(let x = 1; x < fileList.length; x++){
let path = fileList[x].querySelector('.link').firstChild.getAttribute('href')
let name = fileList[x].querySelector('.link').innerText.trim()
let size = fileList[x].querySelector('.size').innerText.trim()
let name = innertext(fileList[x].querySelector('.link').innerHTML).trim()
let size = innertext(fileList[x].querySelector('.size').innerHTML).trim()
fileArray.push({
name: base + name,
filename: name,
@@ -69,3 +78,14 @@ function splitFilesAndFolders(dirArray){
files: files
}
}
function singleLineStatus(str){
if(process.stdout.isTTY){
process.stdout.clearLine(0)
process.stdout.cursorTo(0)
process.stdout.write(str + "\n")
}
else{
console.log(str)
}
}

31
lib/loadfiles.js
View File

@@ -1,6 +1,29 @@
import { readFile } from 'fs/promises';
import { readFile, writeFile } from 'fs/promises';
import fs from 'fs'
export default async function parseJsonFile(filePath) {
let data = JSON.parse(await readFile(filePath, "utf8"));
return data
export async function parseJsonFile(filePath) {
try{
let data = JSON.parse(await readFile(filePath, "utf8"));
return data
}
catch(err){
console.error(err)
}
}
export async function saveJsonFile(filePath, fileArr){
let data = await JSON.stringify(fileArr)
await writeFile(filePath, data, err => {
if(err){
console.error(err)
}
else{
console.log(`Successfully saved file list to ${filePath}.`)
}
}
)
}
export async function fileExists(filePath){
return fs.existsSync(filePath)
}

32
package-lock.json generated
View File

@@ -8,7 +8,9 @@
"dotenv": "^16.4.5",
"ejs": "^3.1.10",
"express": "^4.21.1",
"file-older-than": "^1.0.0",
"fuse.js": "^7.0.0",
"innertext": "^1.0.3",
"jsdom": "^25.0.1",
"node-cron": "^3.0.3",
"node-fetch": "^3.3.2"
@@ -357,6 +359,12 @@
"url": "https://dotenvx.com"
}
},
"node_modules/duration-js": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/duration-js/-/duration-js-4.0.0.tgz",
"integrity": "sha512-qoXjOsH97r+NrOa6sK5V2cwBOouVG/LI9jwgwKvjVkyqGpZ72yilWjjzFJYPqqbvNZDwpRMaLEUFE+PTefvOEA==",
"license": "MIT"
},
"node_modules/ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -500,6 +508,15 @@
"node": "^12.20 || >= 14.13"
}
},
"node_modules/file-older-than": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/file-older-than/-/file-older-than-1.0.0.tgz",
"integrity": "sha512-OUYyochwjcAdahP6NPc/dMhBjkR9lUjMoJL+J9/woJljCGye7Io4R0Er4scguzZFYb2/tnxp12eqqPRusxZ6DQ==",
"license": "MIT",
"dependencies": {
"duration-js": "^4.0.0"
}
},
"node_modules/filelist": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz",
@@ -710,6 +727,12 @@
"node": ">=18"
}
},
"node_modules/html-entities": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/html-entities/-/html-entities-1.4.0.tgz",
"integrity": "sha512-8nxjcBcd8wovbeKx7h3wTji4e6+rhaVuPNpMqwWgnHh+N9ToqsCs6XztWRBPQ+UtzsoMAdKZtUENoVzU/EMtZA==",
"license": "MIT"
},
"node_modules/http-errors": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -816,6 +839,15 @@
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"license": "ISC"
},
"node_modules/innertext": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/innertext/-/innertext-1.0.3.tgz",
"integrity": "sha512-ZC410b7IbrTrmt8bQb27xUOJgXkJu+XL6MVncb9FGyxjRIHyQqNjpSDY20zvSUttkAiYj0dait/67/sXyWvwYg==",
"license": "ISC",
"dependencies": {
"html-entities": "^1.2.0"
}
},
"node_modules/ipaddr.js": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",

2
package.json
View File

@@ -3,7 +3,9 @@
"dotenv": "^16.4.5",
"ejs": "^3.1.10",
"express": "^4.21.1",
"file-older-than": "^1.0.0",
"fuse.js": "^7.0.0",
"innertext": "^1.0.3",
"jsdom": "^25.0.1",
"node-cron": "^3.0.3",
"node-fetch": "^3.3.2"

26
server.js
View File

@@ -1,10 +1,28 @@
import getAllFiles from './lib/dirwalk.js'
import parseJsonFile from './lib/loadfiles.js'
import {parseJsonFile, saveJsonFile, fileExists} from './lib/loadfiles.js'
import fuzzySearch from './lib/search.js'
import cron from 'node-cron'
import FileOlderThan from 'file-older-than'
import 'dotenv/config'
import express from 'express'
var fileList = await parseJsonFile('./filelist.json')
var fileListPath = './filelist.json'
var fileList = []
async function getFilesJob(){
console.log('Updating the file list.')
fileList = await getAllFiles()
saveJsonFile(fileListPath, fileList)
console.log(`Finished updating file list. ${fileList.length} found.`)
}
if(!fileExists(fileListPath) || FileOlderThan(fileListPath, '1d')){
await getFilesJob()
}
else{
fileList = await parseJsonFile(fileListPath)
}
var app = express();
app.set('view engine', 'ejs')
@@ -27,4 +45,6 @@ app.get('/search', function(req, res) {
app.listen(process.env.PORT)
console.log(`Listening on ${process.env.PORT}.`)
console.log(`Loaded ${fileList.length} known files.`)
console.log(`Loaded ${fileList.length} known files.`)
cron.schedule('0 0 0 * * *', getFilesJob)