mirror of
https://github.com/alexankitty/Myrient-Search-Engine.git
synced 2026-01-15 16:33:15 -03:00
make the file handler a class and change the name. update the footer to include a query counter
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -1,2 +1,3 @@
|
|||||||
node_modules
|
node_modules
|
||||||
filelist.json
|
filelist.json
|
||||||
|
queries.txt
|
||||||
52
lib/filehandler.js
Normal file
52
lib/filehandler.js
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
import { readFile, writeFile } from "fs/promises";
|
||||||
|
import fs from "fs";
|
||||||
|
|
||||||
|
export default class FileHandler {
|
||||||
|
static async parseJsonFile(filePath) {
|
||||||
|
try {
|
||||||
|
let data = await JSON.parse(await readFile(filePath, "utf8"));
|
||||||
|
return data;
|
||||||
|
} catch (err) {
|
||||||
|
console.error(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static async saveJsonFile(filePath, fileArr) {
|
||||||
|
let data = await JSON.stringify(fileArr);
|
||||||
|
await writeFile(filePath, data, (err) => {
|
||||||
|
if (err) {
|
||||||
|
console.error(err);
|
||||||
|
} else {
|
||||||
|
console.log(`Successfully saved file list to ${filePath}.`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
static fileExists(filePath) {
|
||||||
|
return fs.existsSync(filePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
static async fileTime(filePath) {
|
||||||
|
try {
|
||||||
|
return fs.statSync(filePath).mtimeMs;
|
||||||
|
} catch (err) {
|
||||||
|
console.error(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static async readFile(filePath) {
|
||||||
|
try {
|
||||||
|
return await readFile(filePath, "utf8");
|
||||||
|
} catch (err) {
|
||||||
|
console.error(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static async writeFile(filePath, data) {
|
||||||
|
await writeFile(filePath, data, (err) => {
|
||||||
|
if (err) {
|
||||||
|
console.error(err);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
import { readFile, writeFile } from 'fs/promises';
|
|
||||||
import fs from 'fs'
|
|
||||||
|
|
||||||
export async function parseJsonFile(filePath) {
|
|
||||||
try{
|
|
||||||
let data = await JSON.parse(await readFile(filePath, "utf8"));
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
catch(err){
|
|
||||||
console.error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function saveJsonFile(filePath, fileArr){
|
|
||||||
let data = await JSON.stringify(fileArr)
|
|
||||||
await writeFile(filePath, data, err => {
|
|
||||||
if(err){
|
|
||||||
console.error(err)
|
|
||||||
}
|
|
||||||
else{
|
|
||||||
console.log(`Successfully saved file list to ${filePath}.`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function fileExists(filePath){
|
|
||||||
return fs.existsSync(filePath)
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function fileTime(filePath){
|
|
||||||
try{
|
|
||||||
return fs.statSync(filePath).mtimeMs
|
|
||||||
}
|
|
||||||
catch(err){
|
|
||||||
console.error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
35
server.js
35
server.js
@@ -1,5 +1,5 @@
|
|||||||
import getAllFiles from './lib/dircrawl.js'
|
import getAllFiles from './lib/dircrawl.js'
|
||||||
import {parseJsonFile, saveJsonFile, fileExists, fileTime} from './lib/loadfiles.js'
|
import FileHandler from './lib/filehandler.js'
|
||||||
import Searcher from './lib/search.js'
|
import Searcher from './lib/search.js'
|
||||||
import cron from 'node-cron'
|
import cron from 'node-cron'
|
||||||
import FileOlderThan from 'file-older-than'
|
import FileOlderThan from 'file-older-than'
|
||||||
@@ -8,11 +8,19 @@ import express from 'express'
|
|||||||
import http from 'http'
|
import http from 'http'
|
||||||
import sanitize from 'sanitize'
|
import sanitize from 'sanitize'
|
||||||
|
|
||||||
|
|
||||||
let fileListPath = './filelist.json'
|
let fileListPath = './filelist.json'
|
||||||
|
let queryCountFile = './queries.txt'
|
||||||
let categoryListPath = './lib/categories.json'
|
let categoryListPath = './lib/categories.json'
|
||||||
let categoryList = await parseJsonFile(categoryListPath)
|
let categoryList = await FileHandler.parseJsonFile(categoryListPath)
|
||||||
//TO DO: add if exist to suppress an error
|
let crawlTime = 0
|
||||||
let crawlTime = await fileTime(fileListPath)
|
let queryCount = 0
|
||||||
|
if(FileHandler.fileExists(fileListPath)){
|
||||||
|
crawlTime = await FileHandler.fileTime(fileListPath)
|
||||||
|
}
|
||||||
|
if(FileHandler.fileExists(queryCountFile)){
|
||||||
|
queryCount = parseInt(await FileHandler.readFile(queryCountFile))
|
||||||
|
}
|
||||||
|
|
||||||
let searchFields = ['filename', 'category', 'type', 'region']
|
let searchFields = ['filename', 'category', 'type', 'region']
|
||||||
|
|
||||||
@@ -42,19 +50,19 @@ let search //cheat so we can check before assignment
|
|||||||
async function getFilesJob(){
|
async function getFilesJob(){
|
||||||
console.log('Updating the file list.')
|
console.log('Updating the file list.')
|
||||||
fileList = await getAllFiles(categoryList)
|
fileList = await getAllFiles(categoryList)
|
||||||
saveJsonFile(fileListPath, fileList)
|
await FileHandler.saveJsonFile(fileListPath, fileList)
|
||||||
if(typeof search !== 'undefined'){
|
if(typeof search !== 'undefined'){
|
||||||
search.createIndex(fileList, searchFields) //recreate the search index
|
search.createIndex(fileList, searchFields) //recreate the search index
|
||||||
}
|
}
|
||||||
crawlTime = await fileTime(fileListPath)
|
crawlTime = await FileHandler.fileTime(fileListPath)
|
||||||
console.log(`Finished updating file list. ${fileList.length} found.`)
|
console.log(`Finished updating file list. ${fileList.length} found.`)
|
||||||
}
|
}
|
||||||
|
|
||||||
if(process.env.FORCE_FILE_REBUILD == "1" || !fileExists(fileListPath) || FileOlderThan(fileListPath, '1w')){
|
if(process.env.FORCE_FILE_REBUILD == "1" || !FileHandler.fileExists(fileListPath) || FileOlderThan(fileListPath, '1w')){
|
||||||
await getFilesJob()
|
await getFilesJob()
|
||||||
}
|
}
|
||||||
else{
|
else{
|
||||||
fileList = await parseJsonFile(fileListPath)
|
fileList = await FileHandler.parseJsonFile(fileListPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
search = new Searcher(fileList, searchFields)
|
search = new Searcher(fileList, searchFields)
|
||||||
@@ -67,7 +75,8 @@ app.set('view engine', 'ejs')
|
|||||||
app.get('/', function(req, res) {
|
app.get('/', function(req, res) {
|
||||||
res.render('pages/index', {
|
res.render('pages/index', {
|
||||||
page: 'search',
|
page: 'search',
|
||||||
crawlTime: crawlTime
|
crawlTime: crawlTime,
|
||||||
|
queryCount: queryCount.toLocaleString()
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -86,8 +95,11 @@ app.get('/search', async function(req, res) {
|
|||||||
query: query,
|
query: query,
|
||||||
results: results,
|
results: results,
|
||||||
crawlTime: crawlTime,
|
crawlTime: crawlTime,
|
||||||
indexing: search.indexing
|
indexing: search.indexing,
|
||||||
|
queryCount: queryCount.toLocaleString()
|
||||||
})
|
})
|
||||||
|
queryCount += 1
|
||||||
|
FileHandler.writeFile(queryCountFile, String(queryCount))
|
||||||
})
|
})
|
||||||
|
|
||||||
app.get("/lucky", async function(req, res) {
|
app.get("/lucky", async function(req, res) {
|
||||||
@@ -109,7 +121,8 @@ app.get("/settings", function(req, res) {
|
|||||||
res.render('pages/index', {
|
res.render('pages/index', {
|
||||||
page: 'settings',
|
page: 'settings',
|
||||||
crawlTime: crawlTime,
|
crawlTime: crawlTime,
|
||||||
defaultSettings: defaultSettings
|
defaultSettings: defaultSettings,
|
||||||
|
queryCount: queryCount.toLocaleString()
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<div class="mb-2">
|
<div class="mb-2">
|
||||||
<p class="text-center text-secondary footer-text">© 2024 Alexankitty <a href='https://ko-fi.com/Q5Q4IFNAO' target='_blank'><img height='36' style='border:0px;height:36px;vertical-align: bottom;' src='https://storage.ko-fi.com/cdn/kofi5.png?v=3' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a></p>
|
<p class="text-center text-secondary footer-text">© 2024 Alexankitty <a href='https://ko-fi.com/Q5Q4IFNAO' target='_blank'><img height='36' style='border:0px;height:36px;vertical-align: bottom;' src='https://storage.ko-fi.com/cdn/kofi5.png?v=3' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a></p>
|
||||||
<p class="text-center text-secondary footer-text">Not affiliated with Myrient/Erista!</p>
|
<p class="text-center text-secondary footer-text">Not affiliated with Myrient/Erista!</p>
|
||||||
<p id="crawl-time" class="text-center text-secondary footer-text">Time of Last Crawl: </p>
|
<p id="crawl-time" class="text-center text-secondary footer-text">Number of Queries: <%= queryCount %> | Time of Last Crawl: </p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script defer>
|
<script defer>
|
||||||
@@ -10,7 +10,7 @@
|
|||||||
var date = new Date(timestamp);
|
var date = new Date(timestamp);
|
||||||
var options = { hour12: false };
|
var options = { hour12: false };
|
||||||
//var time = date + ' ' + month + ' ' + year + ' ' + hour + ':' + min + ':' + sec ;
|
//var time = date + ' ' + month + ' ' + year + ' ' + hour + ':' + min + ':' + sec ;
|
||||||
return date.toLocaleString('en-GB', options)
|
return date.toLocaleString(undefined, options)
|
||||||
}
|
}
|
||||||
document.getElementById('crawl-time').innerText += ` ${timeConverter('<%= crawlTime %>')}`
|
document.getElementById('crawl-time').innerText += ` ${timeConverter('<%= crawlTime %>')}`
|
||||||
</script>
|
</script>
|
||||||
Reference in New Issue
Block a user