mirror of
https://github.com/alexankitty/Myrient-Search-Engine.git
synced 2026-01-15 08:23:18 -03:00
make the file handler a class and change the name. update the footer to include a query counter
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,2 +1,3 @@
|
||||
node_modules
|
||||
filelist.json
|
||||
queries.txt
|
||||
52
lib/filehandler.js
Normal file
52
lib/filehandler.js
Normal file
@@ -0,0 +1,52 @@
|
||||
import { readFile, writeFile } from "fs/promises";
|
||||
import fs from "fs";
|
||||
|
||||
export default class FileHandler {
|
||||
static async parseJsonFile(filePath) {
|
||||
try {
|
||||
let data = await JSON.parse(await readFile(filePath, "utf8"));
|
||||
return data;
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
}
|
||||
}
|
||||
|
||||
static async saveJsonFile(filePath, fileArr) {
|
||||
let data = await JSON.stringify(fileArr);
|
||||
await writeFile(filePath, data, (err) => {
|
||||
if (err) {
|
||||
console.error(err);
|
||||
} else {
|
||||
console.log(`Successfully saved file list to ${filePath}.`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
static fileExists(filePath) {
|
||||
return fs.existsSync(filePath);
|
||||
}
|
||||
|
||||
static async fileTime(filePath) {
|
||||
try {
|
||||
return fs.statSync(filePath).mtimeMs;
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
}
|
||||
}
|
||||
|
||||
static async readFile(filePath) {
|
||||
try {
|
||||
return await readFile(filePath, "utf8");
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
}
|
||||
}
|
||||
|
||||
static async writeFile(filePath, data) {
|
||||
await writeFile(filePath, data, (err) => {
|
||||
if (err) {
|
||||
console.error(err);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
import { readFile, writeFile } from 'fs/promises';
|
||||
import fs from 'fs'
|
||||
|
||||
export async function parseJsonFile(filePath) {
|
||||
try{
|
||||
let data = await JSON.parse(await readFile(filePath, "utf8"));
|
||||
return data
|
||||
}
|
||||
catch(err){
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveJsonFile(filePath, fileArr){
|
||||
let data = await JSON.stringify(fileArr)
|
||||
await writeFile(filePath, data, err => {
|
||||
if(err){
|
||||
console.error(err)
|
||||
}
|
||||
else{
|
||||
console.log(`Successfully saved file list to ${filePath}.`)
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
export async function fileExists(filePath){
|
||||
return fs.existsSync(filePath)
|
||||
}
|
||||
|
||||
export async function fileTime(filePath){
|
||||
try{
|
||||
return fs.statSync(filePath).mtimeMs
|
||||
}
|
||||
catch(err){
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
|
||||
35
server.js
35
server.js
@@ -1,5 +1,5 @@
|
||||
import getAllFiles from './lib/dircrawl.js'
|
||||
import {parseJsonFile, saveJsonFile, fileExists, fileTime} from './lib/loadfiles.js'
|
||||
import FileHandler from './lib/filehandler.js'
|
||||
import Searcher from './lib/search.js'
|
||||
import cron from 'node-cron'
|
||||
import FileOlderThan from 'file-older-than'
|
||||
@@ -8,11 +8,19 @@ import express from 'express'
|
||||
import http from 'http'
|
||||
import sanitize from 'sanitize'
|
||||
|
||||
|
||||
let fileListPath = './filelist.json'
|
||||
let queryCountFile = './queries.txt'
|
||||
let categoryListPath = './lib/categories.json'
|
||||
let categoryList = await parseJsonFile(categoryListPath)
|
||||
//TO DO: add if exist to suppress an error
|
||||
let crawlTime = await fileTime(fileListPath)
|
||||
let categoryList = await FileHandler.parseJsonFile(categoryListPath)
|
||||
let crawlTime = 0
|
||||
let queryCount = 0
|
||||
if(FileHandler.fileExists(fileListPath)){
|
||||
crawlTime = await FileHandler.fileTime(fileListPath)
|
||||
}
|
||||
if(FileHandler.fileExists(queryCountFile)){
|
||||
queryCount = parseInt(await FileHandler.readFile(queryCountFile))
|
||||
}
|
||||
|
||||
let searchFields = ['filename', 'category', 'type', 'region']
|
||||
|
||||
@@ -42,19 +50,19 @@ let search //cheat so we can check before assignment
|
||||
async function getFilesJob(){
|
||||
console.log('Updating the file list.')
|
||||
fileList = await getAllFiles(categoryList)
|
||||
saveJsonFile(fileListPath, fileList)
|
||||
await FileHandler.saveJsonFile(fileListPath, fileList)
|
||||
if(typeof search !== 'undefined'){
|
||||
search.createIndex(fileList, searchFields) //recreate the search index
|
||||
}
|
||||
crawlTime = await fileTime(fileListPath)
|
||||
crawlTime = await FileHandler.fileTime(fileListPath)
|
||||
console.log(`Finished updating file list. ${fileList.length} found.`)
|
||||
}
|
||||
|
||||
if(process.env.FORCE_FILE_REBUILD == "1" || !fileExists(fileListPath) || FileOlderThan(fileListPath, '1w')){
|
||||
if(process.env.FORCE_FILE_REBUILD == "1" || !FileHandler.fileExists(fileListPath) || FileOlderThan(fileListPath, '1w')){
|
||||
await getFilesJob()
|
||||
}
|
||||
else{
|
||||
fileList = await parseJsonFile(fileListPath)
|
||||
fileList = await FileHandler.parseJsonFile(fileListPath)
|
||||
}
|
||||
|
||||
search = new Searcher(fileList, searchFields)
|
||||
@@ -67,7 +75,8 @@ app.set('view engine', 'ejs')
|
||||
app.get('/', function(req, res) {
|
||||
res.render('pages/index', {
|
||||
page: 'search',
|
||||
crawlTime: crawlTime
|
||||
crawlTime: crawlTime,
|
||||
queryCount: queryCount.toLocaleString()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -86,8 +95,11 @@ app.get('/search', async function(req, res) {
|
||||
query: query,
|
||||
results: results,
|
||||
crawlTime: crawlTime,
|
||||
indexing: search.indexing
|
||||
indexing: search.indexing,
|
||||
queryCount: queryCount.toLocaleString()
|
||||
})
|
||||
queryCount += 1
|
||||
FileHandler.writeFile(queryCountFile, String(queryCount))
|
||||
})
|
||||
|
||||
app.get("/lucky", async function(req, res) {
|
||||
@@ -109,7 +121,8 @@ app.get("/settings", function(req, res) {
|
||||
res.render('pages/index', {
|
||||
page: 'settings',
|
||||
crawlTime: crawlTime,
|
||||
defaultSettings: defaultSettings
|
||||
defaultSettings: defaultSettings,
|
||||
queryCount: queryCount.toLocaleString()
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<div class="mb-2">
|
||||
<p class="text-center text-secondary footer-text">© 2024 Alexankitty <a href='https://ko-fi.com/Q5Q4IFNAO' target='_blank'><img height='36' style='border:0px;height:36px;vertical-align: bottom;' src='https://storage.ko-fi.com/cdn/kofi5.png?v=3' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a></p>
|
||||
<p class="text-center text-secondary footer-text">Not affiliated with Myrient/Erista!</p>
|
||||
<p id="crawl-time" class="text-center text-secondary footer-text">Time of Last Crawl: </p>
|
||||
<p id="crawl-time" class="text-center text-secondary footer-text">Number of Queries: <%= queryCount %> | Time of Last Crawl: </p>
|
||||
</div>
|
||||
|
||||
<script defer>
|
||||
@@ -10,7 +10,7 @@
|
||||
var date = new Date(timestamp);
|
||||
var options = { hour12: false };
|
||||
//var time = date + ' ' + month + ' ' + year + ' ' + hour + ':' + min + ':' + sec ;
|
||||
return date.toLocaleString('en-GB', options)
|
||||
return date.toLocaleString(undefined, options)
|
||||
}
|
||||
document.getElementById('crawl-time').innerText += ` ${timeConverter('<%= crawlTime %>')}`
|
||||
</script>
|
||||
Reference in New Issue
Block a user