make the file handler a class and change the name. update the footer to include a query counter

This commit is contained in:
Alexandra
2024-10-21 23:39:54 -06:00
parent 3aa382a140
commit 6a39d100a7
5 changed files with 80 additions and 53 deletions

3
.gitignore vendored
View File

@@ -1,2 +1,3 @@
node_modules
filelist.json
filelist.json
queries.txt

52
lib/filehandler.js Normal file
View File

@@ -0,0 +1,52 @@
import { readFile, writeFile } from "fs/promises";
import fs from "fs";
export default class FileHandler {
static async parseJsonFile(filePath) {
try {
let data = await JSON.parse(await readFile(filePath, "utf8"));
return data;
} catch (err) {
console.error(err);
}
}
static async saveJsonFile(filePath, fileArr) {
let data = await JSON.stringify(fileArr);
await writeFile(filePath, data, (err) => {
if (err) {
console.error(err);
} else {
console.log(`Successfully saved file list to ${filePath}.`);
}
});
}
static fileExists(filePath) {
return fs.existsSync(filePath);
}
static async fileTime(filePath) {
try {
return fs.statSync(filePath).mtimeMs;
} catch (err) {
console.error(err);
}
}
static async readFile(filePath) {
try {
return await readFile(filePath, "utf8");
} catch (err) {
console.error(err);
}
}
static async writeFile(filePath, data) {
await writeFile(filePath, data, (err) => {
if (err) {
console.error(err);
}
});
}
}

View File

@@ -1,39 +0,0 @@
import { readFile, writeFile } from 'fs/promises';
import fs from 'fs'
export async function parseJsonFile(filePath) {
try{
let data = await JSON.parse(await readFile(filePath, "utf8"));
return data
}
catch(err){
console.error(err)
}
}
export async function saveJsonFile(filePath, fileArr){
let data = await JSON.stringify(fileArr)
await writeFile(filePath, data, err => {
if(err){
console.error(err)
}
else{
console.log(`Successfully saved file list to ${filePath}.`)
}
}
)
}
export async function fileExists(filePath){
return fs.existsSync(filePath)
}
export async function fileTime(filePath){
try{
return fs.statSync(filePath).mtimeMs
}
catch(err){
console.error(err)
}
}

View File

@@ -1,5 +1,5 @@
import getAllFiles from './lib/dircrawl.js'
import {parseJsonFile, saveJsonFile, fileExists, fileTime} from './lib/loadfiles.js'
import FileHandler from './lib/filehandler.js'
import Searcher from './lib/search.js'
import cron from 'node-cron'
import FileOlderThan from 'file-older-than'
@@ -8,11 +8,19 @@ import express from 'express'
import http from 'http'
import sanitize from 'sanitize'
let fileListPath = './filelist.json'
let queryCountFile = './queries.txt'
let categoryListPath = './lib/categories.json'
let categoryList = await parseJsonFile(categoryListPath)
//TO DO: add if exist to suppress an error
let crawlTime = await fileTime(fileListPath)
let categoryList = await FileHandler.parseJsonFile(categoryListPath)
let crawlTime = 0
let queryCount = 0
if(FileHandler.fileExists(fileListPath)){
crawlTime = await FileHandler.fileTime(fileListPath)
}
if(FileHandler.fileExists(queryCountFile)){
queryCount = parseInt(await FileHandler.readFile(queryCountFile))
}
let searchFields = ['filename', 'category', 'type', 'region']
@@ -42,19 +50,19 @@ let search //cheat so we can check before assignment
async function getFilesJob(){
console.log('Updating the file list.')
fileList = await getAllFiles(categoryList)
saveJsonFile(fileListPath, fileList)
await FileHandler.saveJsonFile(fileListPath, fileList)
if(typeof search !== 'undefined'){
search.createIndex(fileList, searchFields) //recreate the search index
}
crawlTime = await fileTime(fileListPath)
crawlTime = await FileHandler.fileTime(fileListPath)
console.log(`Finished updating file list. ${fileList.length} found.`)
}
if(process.env.FORCE_FILE_REBUILD == "1" || !fileExists(fileListPath) || FileOlderThan(fileListPath, '1w')){
if(process.env.FORCE_FILE_REBUILD == "1" || !FileHandler.fileExists(fileListPath) || FileOlderThan(fileListPath, '1w')){
await getFilesJob()
}
else{
fileList = await parseJsonFile(fileListPath)
fileList = await FileHandler.parseJsonFile(fileListPath)
}
search = new Searcher(fileList, searchFields)
@@ -67,7 +75,8 @@ app.set('view engine', 'ejs')
app.get('/', function(req, res) {
res.render('pages/index', {
page: 'search',
crawlTime: crawlTime
crawlTime: crawlTime,
queryCount: queryCount.toLocaleString()
})
})
@@ -86,8 +95,11 @@ app.get('/search', async function(req, res) {
query: query,
results: results,
crawlTime: crawlTime,
indexing: search.indexing
indexing: search.indexing,
queryCount: queryCount.toLocaleString()
})
queryCount += 1
FileHandler.writeFile(queryCountFile, String(queryCount))
})
app.get("/lucky", async function(req, res) {
@@ -109,7 +121,8 @@ app.get("/settings", function(req, res) {
res.render('pages/index', {
page: 'settings',
crawlTime: crawlTime,
defaultSettings: defaultSettings
defaultSettings: defaultSettings,
queryCount: queryCount.toLocaleString()
})
})

View File

@@ -1,7 +1,7 @@
<div class="mb-2">
<p class="text-center text-secondary footer-text">&copy; 2024 Alexankitty <a href='https://ko-fi.com/Q5Q4IFNAO' target='_blank'><img height='36' style='border:0px;height:36px;vertical-align: bottom;' src='https://storage.ko-fi.com/cdn/kofi5.png?v=3' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a></p>
<p class="text-center text-secondary footer-text">Not affiliated with Myrient/Erista!</p>
<p id="crawl-time" class="text-center text-secondary footer-text">Time of Last Crawl: </p>
<p id="crawl-time" class="text-center text-secondary footer-text">Number of Queries: <%= queryCount %> | Time of Last Crawl: </p>
</div>
<script defer>
@@ -10,7 +10,7 @@
var date = new Date(timestamp);
var options = { hour12: false };
//var time = date + ' ' + month + ' ' + year + ' ' + hour + ':' + min + ':' + sec ;
return date.toLocaleString('en-GB', options)
return date.toLocaleString(undefined, options)
}
document.getElementById('crawl-time').innerText += ` ${timeConverter('<%= crawlTime %>')}`
</script>