Firehawk52-backup/Myrient-Search-Engine
28
0
Fork 0
mirror of https://github.com/alexankitty/Myrient-Search-Engine.git synced 2026-01-15 16:33:15 -03:00
Code Issues Packages Projects Releases Wiki Activity

Instance name from environment & About page

Browse Source
This commit is contained in:
ovosimpatico
2024-10-27 01:53:57 -03:00
parent 3417f45186
commit cdf6e274ab
13 changed files with 105 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.env
View File

@@ -3,9 +3,11 @@ BIND_ADDRESS=0.0.0.0
FORCE_FILE_REBUILD=0
DEBUG=0
NODE_ENV=production
# Memory Impacting Settings - Trades for threading efficiency. Much slower, but should be useful for limited memory environments like VPS
# Memory Impacting Settings - Trades for threading efficiency. Much slower, but should be useful for limited memory environments like VPS
# May also decrease 504 failure rates
# Changes the maximum number of jobs the crawler can queue. Setting it too high will cause a call stack overflow
MAX_JOB_QUEUE=1000
# Changes the maximum number of pages that can be fetched for parsing. Has a massive impact on memory usage. Setting to 12 results in about 1.1GiB memory usage
MAX_FETCH_JOBS=1000
MAX_FETCH_JOBS=1000
# Changes the name of your instance
INSTANCE_NAME=Myrient

9
lib/asciiart.js Normal file
View File

@@ -0,0 +1,9 @@
import figlet from 'figlet';
export function generateAsciiArt(text) {
return figlet.textSync(text || process.env.INSTANCE_NAME || 'Myrient', {
font: 'Slant',
horizontalLayout: 'default',
verticalLayout: 'default'
});
}

14
lib/search.js
View File

@@ -11,7 +11,7 @@ export default class Searcher{
}
termProcessor(term){
term = term.toLowerCase()
term = term.toLowerCase()
let stringArray = [term]
stringArray.push(...Searcher.stringBreakout(term))
for(let group in searchAlikes.StringAssoc){
@@ -25,7 +25,7 @@ export default class Searcher{
}
}
return [...new Set(stringArray)]
}
}
static stringBreakout(string){
let symbolRegex = /-|_|\+|=|\)|\(|\[|{|}|]|;|:|"|'|<|>|\.|,|\/|\?|\||\\|!|@|#|\$|%|\^|&|\*/g
@@ -62,7 +62,7 @@ export default class Searcher{
console.error(err)
}
}
async createIndex(fileArr){
if(!this.miniSearch){
this.miniSearch = new MiniSearch({
@@ -157,7 +157,7 @@ export default class Searcher{
else {
//bad result discard
continue
}
}
suggestions.push(`${query}${prediction}`)
continue
}
@@ -178,10 +178,10 @@ export default class Searcher{
else {
//bad result discard
continue
}
}
suggestions.push(`${query}${prediction}`)
continue
}
}
}
let dedupe = [...new Set(suggestions)]
let dedupeLimit = dedupe.length >= 10 ? 10 : dedupe.length
@@ -190,7 +190,7 @@ export default class Searcher{
arr.push({
suggestion: dedupe[x]
})
}
}
return arr
}
}

13
package-lock.json generated
View File

@@ -9,6 +9,7 @@
"dotenv": "^16.4.5",
"ejs": "^3.1.10",
"express": "^4.21.1",
"figlet": "^1.7.0",
"file-older-than": "^1.0.0",
"innertext": "^1.0.3",
"jsdom": "^25.0.1",
@@ -934,6 +935,18 @@
"node": "^12.20 || >= 14.13"
}
},
"node_modules/figlet": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/figlet/-/figlet-1.8.0.tgz",
"integrity": "sha512-chzvGjd+Sp7KUvPHZv6EXV5Ir3Q7kYNpCr4aHrRW79qFtTefmQZNny+W1pW9kf5zeE6dikku2W50W/wAH2xWgw==",
"license": "MIT",
"bin": {
"figlet": "bin/index.js"
},
"engines": {
"node": ">= 0.4.0"
}
},
"node_modules/file-older-than": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/file-older-than/-/file-older-than-1.0.0.tgz",

3
package.json
View File

@@ -12,7 +12,8 @@
"node-fetch": "^3.3.2",
"node-html-parser": "^6.1.13",
"piscina": "^4.7.0",
"sanitize": "^2.1.2"
"sanitize": "^2.1.2",
"figlet": "^1.7.0"
},
"type": "module"
}

13
server.js
View File

@@ -8,7 +8,8 @@ import express from "express";
import http from "http";
import sanitize from "sanitize";
import debugPrint from "./lib/debugprint.js";
import compression from "compression";
import compression from "compression";
import { generateAsciiArt } from './lib/asciiart.js';
let fileListPath = "./filelist.json";
let queryCountFile = "./queries.txt";
@@ -99,7 +100,8 @@ let defaultOptions = {
crawlTime: crawlTime,
queryCount: queryCount,
fileCount: fileCount,
termCount: search.miniSearch.termCount
termCount: search.miniSearch.termCount,
generateAsciiArt: generateAsciiArt
};
function updateDefaults(){
@@ -114,7 +116,7 @@ let server = http.createServer(app);
app.use(sanitize.middleware);
app.use(compression())
app.use(express.json())
app.set("view engine", "ejs");
app.set("view engine", "ejs");
app.get("/", function (req, res) {
let page = "search";
@@ -209,6 +211,11 @@ app.post("/suggest", async function(req, res){
res.end(JSON.stringify({ suggestions }));
})
app.get("/about", function (req, res) {
let page = "about";
res.render(indexPage, buildOptions(page));
});
server.listen(process.env.PORT, process.env.BIND_ADDRESS);
server.on("listening", function () {
console.log(

32
views/pages/about.ejs Normal file
View File

@@ -0,0 +1,32 @@
<div class="row h-50 w-100 m-0">
<div class="col-sm-12 my-auto text-center">
<pre style="font: 20px / 19px monospace; color: white; text-align: center; overflow: hidden;">
<%= generateAsciiArt() %>
About
</pre>
<div class="card w-auto mx-auto text-center d-inline-block p-3">
<div class="about-content">
<h4>About <%= process.env.INSTANCE_NAME || 'Myrient' %> Search</h4>
<p>A search engine for <a href="https://github.com/alexankitty/myrient-global-search">Myrient</a> -
a service by Erista dedicated to video game preservation.</p>
<p>Myrient offers organized and publicly available video game collections, keeping them from becoming
lost to time.</p>
<p class="text-secondary mb-4">Not affiliated with Myrient/Erista!</p>
<div class="mb-4">
<p>If you like this project, please consider supporting Myrient:</p>
<a href="https://myrient.erista.me/donate/" class="btn btn-secondary">Donate to Myrient</a>
</div>
<div class="border-top pt-3">
<p>Search engine created by <a href="https://github.com/alexankitty">Alexankitty</a></p>
<p><a href="https://github.com/alexankitty/myrient-global-search">View project on GitHub</a></p>
<a href='https://ko-fi.com/Q5Q4IFNAO' target='_blank'>
<img height='36' style='border:0px;height:36px;'
src='https://storage.ko-fi.com/cdn/kofi5.png?v=3' alt='Buy Me a Coffee at ko-fi.com' />
</a>
</div>
</div>
</div>
</div>
</div>

19
views/pages/results.ejs
View File

@@ -1,4 +1,4 @@
<%
<%
let pageCount = Math.ceil(results.items.length / 100)
pageCount = pageCount ? pageCount : 1 //always ensure 1 page
if(pageNum > pageCount){
@@ -13,31 +13,26 @@
<script src='https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.5.2/js/bootstrap.min.js'></script>
<script src='https://cdn.datatables.net/2.1.8/js/dataTables.js'></script>
<script src='https://cdn.datatables.net/2.1.8/js/dataTables.bootstrap4.js'></script>
<link rel="stylesheet" href="https://cdn.datatables.net/2.1.8/css/dataTables.bootstrap4.css">
<link rel="stylesheet" href="https://cdn.datatables.net/2.1.8/css/dataTables.bootstrap4.css">
<div class="row w-100 m-0">
<form class="ml-2 form-inline w-100" action="/search">
<div class="w-100 align-items-center">
<div class="form-group">
<a href="/">
<pre class="mt-4 ml-2" style="font: 6px / 5px monospace; color: white; text-align: center; overflow: hidden; display: inline-flex;">
______ ___ _____ _____
___ |/ /____ ___________(_)_____________ /_
__ /|_/ /__ / / /_ ___/_ /_ _ \_ __ \ __/
_ / / / _ /_/ /_ / _ / / __/ / / / /_
/_/ /_/ _\__, / /_/ /_/ \___//_/ /_/\__/
/____/
<%= generateAsciiArt() %>
</pre>
</a>
<input type="hidden" name="s" id="searchSettings">
<input id="search" type="text" class="w-50 form-control bg-dark text-white ml-2" name="q" value="<%= query %>" autocomplete="off">
<button type="submit" class="btn btn-secondary ml-2">Search</button>
</div>
<ul class="SuggestionList col-sm-12" id="suggestionList" style="width: 50%;left: 195px;"></ul>
</div>
<p class="m-2">Found <%= results.items.length %> result<%= results.items.length != 1 ? 's': '' %> in <%= results.elapsed %> seconds. <%= indexing ? "Indexing in progress, if the list is missing something please try reloading in a few minutes" : "" %></p>
</form>
<div class="col-sm-12 w-100 mt-3">
<p>Displaying results <%= entryStart %> through <%= entryEnd %>. </p>
<table class="table text-white table-bordered" id="results">
@@ -80,7 +75,7 @@ _ / / / _ /_/ /_ / _ / / __/ / / / /_
</tr>
<% } %>
</table>
<%
<%
if(pageCount > 1) {
%>
<div class="row justify-content-between">
@@ -88,7 +83,7 @@ _ / / / _ /_/ /_ / _ / / __/ / / / /_
<div class="dt-paging">
<nav aria-label="pagination">
<ul class="pagination">
<%
<%
let ellipsesElem = '<li class="dt-paging-button page-item disabled"><a class="page-link ellipsis" aria-controls="results" aria-disabled="true" data-dt-idx="ellipsis" tabindex="-1">…</a></li> '
let pageUrlPrefix = ''
const getPageRange = (page) => {

12
views/pages/search.ejs
View File

@@ -1,18 +1,14 @@
<div class="row h-50 w-100 m-0">
<div class="col-sm-12 my-auto">
<pre style="font: 20px / 19px monospace; color: white; text-align: center; overflow: hidden;">
______ ___ _____ _____
___ |/ /____ ___________(_)_____________ /_
__ /|_/ /__ / / /_ ___/_ /_ _ \_ __ \ __/
_ / / / _ /_/ /_ / _ / / __/ / / / /_
/_/ /_/ _\__, / /_/ /_/ \___//_/ /_/\__/
/____/
<%= generateAsciiArt() %>
Search!
</pre>
</pre>
<div class="text-center text-white">
<form>
<input type="hidden" name="s" id="searchSettings">
<input id="search" type="text" style="width: 80%;display: inline;" class="form-control bg-dark text-white mb-2" name="q" autocomplete="off">
<input id="search" type="text" style="width: 80%;display: inline;" class="form-control bg-dark text-white mb-2"
name="q" autocomplete="off">
<ul class="SuggestionList col-sm-12" id="suggestionList" style="width: 78%;left: 11%;"></ul>
<div>
<button type="submit" formaction="/search" class="btn btn-secondary">Myrient Search</button>

13
views/pages/settings.ejs
View File

@@ -4,14 +4,9 @@
<div class="row h-50 w-100 m-0">
<div class="col-sm-12 my-auto text-center">
<pre style="font: 20px / 19px monospace; color: white; text-align: center; overflow: hidden;">
______ ___ _____ _____
___ |/ /____ ___________(_)_____________ /_
__ /|_/ /__ / / /_ ___/_ /_ _ \_ __ \ __/
_ / / / _ /_/ /_ / _ / / __/ / / / /_
/_/ /_/ _\__, / /_/ /_/ \___//_/ /_/\__/
/____/
<%= generateAsciiArt() %>
Settings
</pre>
</pre>
<div class="card w-auto mx-auto text-center d-inline-block p-3">
<form>
<div class="form-group">
@@ -121,7 +116,7 @@ _ / / / _ /_/ /_ / _ / / __/ / / / /_
window.location.href = '/'
}
function loadSettings(){
function loadSettings(){
$(function () {
$('[data-toggle="tooltip"]').tooltip()
})
@@ -168,4 +163,4 @@ _ / / / _ /_/ /_ / _ / / __/ / / / /_
textBox.disabled = false
}
}
</script>
</script>

6
views/partials/footer.ejs
View File

@@ -1,7 +1,5 @@
<div class="mb-2">
<p class="text-center text-secondary footer-text">&copy; 2024 Alexankitty <a href='https://ko-fi.com/Q5Q4IFNAO' target='_blank'><img height='36' style='border:0px;height:36px;vertical-align: bottom;' src='https://storage.ko-fi.com/cdn/kofi5.png?v=3' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a></p>
<p class="text-center text-secondary footer-text">Not affiliated with Myrient/Erista!</p>
<div class="text-center text-secondary footer-text">
<div class="text-center text-secondary footer-text">
<div id="query-count" class="stats">Number of Queries:</div>
<div class="stats"> | </div>
<div id="file-count" class="stats">Known Files:</div>
@@ -23,4 +21,4 @@
document.getElementById('file-count').innerText += ` ${(<%= fileCount %>).toLocaleString(undefined)}`
document.getElementById('query-count').innerText += ` ${(<%= queryCount %>).toLocaleString(undefined)}`
document.getElementById('term-count').innerText += ` ${(<%= termCount %>).toLocaleString(undefined)}`
</script>
</script>

16
views/partials/head.ejs
View File

@@ -1,6 +1,6 @@
<meta charset="UTF-8">
<!-- HTML Meta Tags -->
<title>Myrient Search</title>
<title><%= process.env.INSTANCE_NAME || 'Myrient' %> Search</title>
<!-- CSS (load bootstrap from a CDN) -->
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.5.2/css/bootstrap.min.css">
@@ -47,7 +47,7 @@
td a {
display: block;
}
.footer-text{
margin: 0;
}
@@ -105,7 +105,7 @@
list-style-image: none;
padding: 0;
border: 1px solid #ccc;
margin: 0 0 0.2em 0;
margin: 0 0 0.2em 0;
border-radius: 3px;
box-shadow: 0 1px 1px rgba(0, 0, 0, 0.1);
background: #262c2c;
@@ -189,8 +189,8 @@
})
searchElem.addEventListener('keyup', function (e) {
if(e.key === 'Escape'){
return
}
return
}
if(e.key === 'ArrowUp'){
if(!totalSuggestions) return
if(typeof selectedSuggestion != 'number'){
@@ -219,7 +219,7 @@
selectSuggestion(selectedSuggestion)
return
}
if(e.key === 'Enter'){
if(e.key === 'Enter'){
if(selectedSuggestion != null){
enterSuggestion(selectedSuggestion)
return
@@ -231,7 +231,7 @@
clearTimeout(typingTimeout);
}
typingTimeout = setTimeout(function() {
typingTimeout = null;
typingTimeout = null;
if(!query){
let suggestionList = document.getElementById('suggestionList')
suggestionList.replaceChildren()
@@ -283,4 +283,4 @@
selectedItems[item].classList.remove('selected')
}
}
</script>
</script>

12
views/partials/header.ejs
View File

@@ -1,18 +1,12 @@
<nav class="navbar navbar-expand-sm navbar-light text-white">
<a id="brand-name" class="navbar-brand text-white" href="/">Myrient Search</a>
<a id="brand-name" class="navbar-brand text-white" href="/"><%= process.env.INSTANCE_NAME || 'Myrient' %> Search</a>
<a id="brand-name" class="navbar-brand text-white hidden" href="/search">Results</a>
<ul class="navbar-nav mr-auto">
<li class="nav-item">
<a class="nav-link text-white" href="/settings">Settings</a>
</li>
<li class="nav-item">
<a class="nav-link text-white" href="https://github.com/alexankitty/myrient-global-search/">Github Project</a>
</li>
<li class="nav-item">
<a class="nav-link text-white" href="https://myrient.erista.me/donate/">Donate</a>
</li>
<li class="nav-item">
<a class="nav-link text-white" href="https://myrient.erista.me/">Home</a>
<a class="nav-link text-white" href="/about">About</a>
</li>
</ul>
</nav>
@@ -25,4 +19,4 @@
aTag.classList.remove('hidden')
}
})
</script>
</script>