diff --git a/.env b/.env index 0e114c9..e7560c4 100644 --- a/.env +++ b/.env @@ -1,2 +1,3 @@ PORT=8062 -FUZZY_LENIENCY=0.2 \ No newline at end of file +FUZZY_LENIENCY=0.2 +DEBUG=0 \ No newline at end of file diff --git a/lib/categories.json b/lib/categories.json new file mode 100644 index 0000000..b1e31bf --- /dev/null +++ b/lib/categories.json @@ -0,0 +1,394 @@ +{ + "Categories": { + "Acorn": [ + "Archimedes", + "Atom", + "RISC" + ], + "ACT": [ + "Apricot PC Xi" + ], + "Amstrad": [ + "CPC" + ], + "Analogue": [ + "Pocket" + ], + "APF": [ + "Imagination Machine", + "MP-1000" + ], + "Apple": [ + "I", + "II Plus", + "II", + "IIGS", + "IIe", + "Macintosh" + ], + "Apple-Bandai": [ + "Pippin", + "Playdia" + ], + "Arcade": [ + "PC-based" + ], + "Arduboy Inc": [ + "Arduboy" + ], + "Atari": [ + "2600", + "5200", + "7800", + "8-bit", + "Jaguar", + "Lynx", + "ST" + ], + "Bally": [ + "Astrocade" + ], + "Bandai": [ + "Design Master Denshi Mangajuku", + "Gundam RX-78", + "WonderSwan Color", + "WonderSwan" + ], + "Benesse": [ + "Pocket Challenge V2", + "Pocket Challenge W" + ], + "Bit Corporation": [ + "Gamate" + ], + "Casio": [ + "Loopy", + "PV-1000" + ], + "Coleco": [ + "ColecoVision" + ], + "Commodore": [ + "Amiga", + "Commodore 64", + "Plus-4", + "VIC-20" + ], + "Digital Media Cartridge": [ + "Firecore" + ], + "Emerson": [ + "Arcadia 2001" + ], + "Entex": [ + "Adventure Vision" + ], + "Epoch": [ + "Game Pocket Computer", + "Super Cassette Vision" + ], + "Fairchild": [ + "Channel F" + ], + "Fujitsu": [ + "FM Towns", + "FM-7", + "FM-Towns", + "FMR50" + ], + "Fukutake Publishing": [ + "StudyBox" + ], + "Funtech": [ + "Super Acan" + ], + "funworld": [ + "Photo Play" + ], + "GamePark": [ + "GP2X", + "GP32" + ], + "GCE": [ + "Vectrex" + ], + "Google": [ + "Android" + ], + "Hartung": [ + "Game Master" + ], + "Hasbro": [ + "VideoNow Color", + "VideoNow Jr", + "VideoNow XP", + "VideoNow", + "iON" + ], + "Hitachi": [ + "S1" + ], + "IBM": [ + "PC Compatible", + "PC and Compatibles" + ], + "Incredible Technologies": [ + "Eagle" + ], + "Interton": [ + "VC 4000" + ], + "iQue": [ + "iQue" + ], + "Konami": [ + "FireBeat", + "M2", + "Picno", + "System 573", + "System GV", + "e-Amusement" + ], + "LeapFrog": [ + "Explorer", + "LeapPad", + "Leapster" + ], + "Luxor": [ + "ABC 800" + ], + "Magnavox": [ + "Odyssey 2" + ], + "Mattel": [ + "Fisher-Price iXL", + "HyperScan", + "Intellivision" + ], + "Memorex": [ + "Visual Information System" + ], + "Merit": [ + "Megatouch" + ], + "Microsoft": [ + "MSX", + "MSX2", + "Pocket PC", + "Xbox 360", + "Xbox" + ], + "Milton-Bradley": [ + "Omni" + ], + "Mobile": [ + "J2ME", + "Palm OS", + "Pocket PC", + "Symbian" + ], + "Namco": [ + "System 246" + ], + "Navisoft": [ + "Naviken 2.1" + ], + "NEC": [ + "PC Engine", + "PC-88", + "PC-98", + "PC-FX" + ], + "Nichibutsu": [ + "My Vision" + ], + "Nintendo": [ + "3DS", + "64", + "64DD", + "DS", + "DSi", + "Family BASIC", + "Family Computer Disk System", + "Family Computer Network System", + "Game & Watch", + "Game Boy Advance", + "Game Boy Color", + "Game Boy", + "Gamecube", + "Kiosk Video Compact Flash", + "Misc", + "New Nintendo 3DS", + "Nintendo Entertainment System", + "Pokemon Mini", + "SDKs", + "Satellaview", + "Sufami Turbo", + "Super Nintendo Entertainment System", + "Triforce", + "VirtualBoy", + "Wallpapers", + "Wii U", + "Wii", + "amiibo" + ], + "Nokia": [ + "N-Gage" + ], + "Others": [ + "Audio", + "BD-Video", + "CD-ROM", + "DVD-ROM", + "DVD-Video", + "HD DVD", + "HD DVD-Video", + "HumblePlay", + "Palm", + "Photo CD", + "Pocket PC", + "Project EGG", + "Video CD", + "Video Game Documents", + "Video Game Magazine Scans", + "Video Game OSTs", + "Video Game Scans", + "itch.io" + ], + "Ouya": [ + "Ouya" + ], + "Panasonic": [ + "3DO", + "M2" + ], + "Philips": [ + "CD-i", + "videopac+" + ], + "RCA": [ + "Studio II" + ], + "Sanyo": [ + "MBC-550" + ], + "Sega": [ + "32X", + "Beena", + "CD", + "Chihiro", + "Dreamcast", + "Game Gear", + "Lindbergh", + "Master System", + "Mega Drive", + "Naomi 2", + "Naomi", + "PICO", + "Prologue 21", + "RingEdge 2", + "RingEdge", + "SG-1000", + "Saturn" + ], + "Seta": [ + "Aleck64" + ], + "Sharp": [ + "MZ-2200", + "MZ-700", + "X1", + "X68000" + ], + "Sinclair": [ + "ZX Spectrum" + ], + "SNK": [ + "NeoGeo CD", + "NeoGeo Pocket Color", + "NeoGeo Pocket" + ], + "Sony": [ + "Electronic Book", + "PlayStation 2", + "PlayStation 3", + "PlayStation Mobile", + "PlayStation Portable", + "PlayStation Vita", + "PlayStation" + ], + "Tab-Austria": [ + "Quizard" + ], + "TeleNova": [ + "Compis" + ], + "Texas Instruments": [ + "TI-99-4A" + ], + "Tiger": [ + "Game.com", + "Gizmondo" + ], + "Tomy": [ + "Kiss-Site" + ], + "Toshiba": [ + "Pasopia", + "Visicom" + ], + "VM Labs": [ + "NUON" + ], + "VTech": [ + "CreatiVision", + "V.Flash & V.Smile Pro", + "V.Smile" + ], + "Watara": [ + "SuperVision" + ], + "Welback": [ + "Mega Duck" + ], + "Yamaha": [ + "Copera" + ], + "ZAPiT Games": [ + "Game Wave Family Entertainment System" + ], + "Zeebo": [ + "Zeebo" + ] + }, + "Types": [ + "BIOS", + "BigEndian", + "ByteSwapped", + "CBZ", + "DLC", + "Decrypted", + "Development Kit", + "Digital", + "Disc Keys", + "GameShark", + "Header", + "Headerless", + "LittleEndian", + "NKit", + "NoNpDrm", + "PDF", + "PS one Classics", + "PSN", + "PSX2PSP", + "RAW", + "Source Code", + "UMD Music", + "UMD Video", + "Updates and DLC", + "Updates", + "VPK", + "Visual Memory Unit", + "WUX" + ] +} \ No newline at end of file diff --git a/lib/dirwalk.js b/lib/dirwalk.js index 1e5426a..74915ff 100644 --- a/lib/dirwalk.js +++ b/lib/dirwalk.js @@ -1,5 +1,6 @@ import fetch from 'node-fetch' import jsdom from "jsdom" +import innertext from 'innertext' const { JSDOM } = jsdom global.DOMParser = new JSDOM().window.DOMParser @@ -11,13 +12,21 @@ export default async function getAllFiles(){ let files = dirWork.files let dirs = dirWork.directories while(dirs.length > 0) { - console.log(`Working on: ${dirs[0].name}`) + if(process.env.DEBUG == '1'){ + console.log(`Working on: ${dirs[0].name}`) + } let results = await getFilesAndFolders(dirs[0].path, dirs[0].name) let working = splitFilesAndFolders(results) if(working.files.length > 0) files.push(...working.files) if(working.directories.length > 0) dirs.push(...working.directories) dirs.shift() - console.log(`Directories Remaining: ${dirs.length}, Files Found: ${files.length}`) + let dirStatus = `Directories Remaining: ${dirs.length}, Files Found: ${files.length}` + if(process.env.DEBUG == '1'){ + console.log(dirStatus) + } + else{ + singleLineStatus(dirStatus) + } } return files } @@ -34,8 +43,8 @@ async function getFilesAndFolders(url, base = ""){ const fileList = fileHtml.getElementById('list').rows for(let x = 1; x < fileList.length; x++){ let path = fileList[x].querySelector('.link').firstChild.getAttribute('href') - let name = fileList[x].querySelector('.link').innerText.trim() - let size = fileList[x].querySelector('.size').innerText.trim() + let name = innertext(fileList[x].querySelector('.link').innerHTML).trim() + let size = innertext(fileList[x].querySelector('.size').innerHTML).trim() fileArray.push({ name: base + name, filename: name, @@ -69,3 +78,14 @@ function splitFilesAndFolders(dirArray){ files: files } } + +function singleLineStatus(str){ + if(process.stdout.isTTY){ + process.stdout.clearLine(0) + process.stdout.cursorTo(0) + process.stdout.write(str + "\n") + } + else{ + console.log(str) + } +} \ No newline at end of file diff --git a/lib/loadfiles.js b/lib/loadfiles.js index f715bf5..638e794 100644 --- a/lib/loadfiles.js +++ b/lib/loadfiles.js @@ -1,6 +1,29 @@ -import { readFile } from 'fs/promises'; +import { readFile, writeFile } from 'fs/promises'; +import fs from 'fs' -export default async function parseJsonFile(filePath) { - let data = JSON.parse(await readFile(filePath, "utf8")); - return data +export async function parseJsonFile(filePath) { + try{ + let data = JSON.parse(await readFile(filePath, "utf8")); + return data + } + catch(err){ + console.error(err) + } +} + +export async function saveJsonFile(filePath, fileArr){ + let data = await JSON.stringify(fileArr) + await writeFile(filePath, data, err => { + if(err){ + console.error(err) + } + else{ + console.log(`Successfully saved file list to ${filePath}.`) + } + } + ) +} + +export async function fileExists(filePath){ + return fs.existsSync(filePath) } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index a388dc6..8194076 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,9 @@ "dotenv": "^16.4.5", "ejs": "^3.1.10", "express": "^4.21.1", + "file-older-than": "^1.0.0", "fuse.js": "^7.0.0", + "innertext": "^1.0.3", "jsdom": "^25.0.1", "node-cron": "^3.0.3", "node-fetch": "^3.3.2" @@ -357,6 +359,12 @@ "url": "https://dotenvx.com" } }, + "node_modules/duration-js": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/duration-js/-/duration-js-4.0.0.tgz", + "integrity": "sha512-qoXjOsH97r+NrOa6sK5V2cwBOouVG/LI9jwgwKvjVkyqGpZ72yilWjjzFJYPqqbvNZDwpRMaLEUFE+PTefvOEA==", + "license": "MIT" + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -500,6 +508,15 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/file-older-than": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-older-than/-/file-older-than-1.0.0.tgz", + "integrity": "sha512-OUYyochwjcAdahP6NPc/dMhBjkR9lUjMoJL+J9/woJljCGye7Io4R0Er4scguzZFYb2/tnxp12eqqPRusxZ6DQ==", + "license": "MIT", + "dependencies": { + "duration-js": "^4.0.0" + } + }, "node_modules/filelist": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", @@ -710,6 +727,12 @@ "node": ">=18" } }, + "node_modules/html-entities": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-1.4.0.tgz", + "integrity": "sha512-8nxjcBcd8wovbeKx7h3wTji4e6+rhaVuPNpMqwWgnHh+N9ToqsCs6XztWRBPQ+UtzsoMAdKZtUENoVzU/EMtZA==", + "license": "MIT" + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -816,6 +839,15 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/innertext": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/innertext/-/innertext-1.0.3.tgz", + "integrity": "sha512-ZC410b7IbrTrmt8bQb27xUOJgXkJu+XL6MVncb9FGyxjRIHyQqNjpSDY20zvSUttkAiYj0dait/67/sXyWvwYg==", + "license": "ISC", + "dependencies": { + "html-entities": "^1.2.0" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", diff --git a/package.json b/package.json index 9125775..d558ec2 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,9 @@ "dotenv": "^16.4.5", "ejs": "^3.1.10", "express": "^4.21.1", + "file-older-than": "^1.0.0", "fuse.js": "^7.0.0", + "innertext": "^1.0.3", "jsdom": "^25.0.1", "node-cron": "^3.0.3", "node-fetch": "^3.3.2" diff --git a/server.js b/server.js index 3b7eaa6..f80dd3e 100644 --- a/server.js +++ b/server.js @@ -1,10 +1,28 @@ import getAllFiles from './lib/dirwalk.js' -import parseJsonFile from './lib/loadfiles.js' +import {parseJsonFile, saveJsonFile, fileExists} from './lib/loadfiles.js' import fuzzySearch from './lib/search.js' +import cron from 'node-cron' +import FileOlderThan from 'file-older-than' import 'dotenv/config' import express from 'express' -var fileList = await parseJsonFile('./filelist.json') + +var fileListPath = './filelist.json' +var fileList = [] + +async function getFilesJob(){ + console.log('Updating the file list.') + fileList = await getAllFiles() + saveJsonFile(fileListPath, fileList) + console.log(`Finished updating file list. ${fileList.length} found.`) +} + +if(!fileExists(fileListPath) || FileOlderThan(fileListPath, '1d')){ + await getFilesJob() +} +else{ + fileList = await parseJsonFile(fileListPath) +} var app = express(); app.set('view engine', 'ejs') @@ -27,4 +45,6 @@ app.get('/search', function(req, res) { app.listen(process.env.PORT) console.log(`Listening on ${process.env.PORT}.`) -console.log(`Loaded ${fileList.length} known files.`) \ No newline at end of file +console.log(`Loaded ${fileList.length} known files.`) + +cron.schedule('0 0 0 * * *', getFilesJob) \ No newline at end of file