From 0c43711820ebbcf875293770d08504b759ce85fc Mon Sep 17 00:00:00 2001 From: Alexandra Date: Sat, 19 Oct 2024 00:08:34 -0600 Subject: [PATCH] the big fat settings update + asynchronous indexing --- .env | 2 - lib/search.js | 59 +++++++++----- server.js | 61 +++++++++++---- views/pages/results.ejs | 2 +- views/pages/search.ejs | 10 ++- views/pages/settings.ejs | 164 ++++++++++++++++++++++++++++++++++++++- views/partials/head.ejs | 7 ++ 7 files changed, 263 insertions(+), 42 deletions(-) diff --git a/.env b/.env index f80972f..2b85128 100644 --- a/.env +++ b/.env @@ -1,6 +1,4 @@ PORT=8062 BIND_ADDRESS=0.0.0.0 -FUZZY_DISTANCE=0.2 -MIN_MATCH=15 FORCE_FILE_REBUILD=0 DEBUG=0 \ No newline at end of file diff --git a/lib/search.js b/lib/search.js index 64d941f..b5a922c 100644 --- a/lib/search.js +++ b/lib/search.js @@ -1,35 +1,58 @@ import MiniSearch from 'minisearch' export default class Searcher{ - constructor(fileArr){ + constructor(fileArr, fields){ this.distance = parseFloat(process.env.FUZZY_DISTANCE) this.minMatch = parseFloat(process.env.MIN_MATCH) - this.createIndex(fileArr) + this.indexing = false + this.createIndex(fileArr, fields) } - findAllMatches(query){ - var startTime = process.hrtime(); - let results = this.miniSearch.search(query) - var elapsed = this.parseHrtimeToSeconds(process.hrtime(startTime)); - return { - items: results, - elapsed: elapsed + async findAllMatches(query, options){ + try{ + var startTime = process.hrtime(); + while(this.indexing){ + await this.sleep(1000) + } + if(process.env.DEBUG == "1"){ + console.log(options) + } + let results = this.miniSearch.search(query, options) + var elapsed = this.parseHrtimeToSeconds(process.hrtime(startTime)); + return { + items: results, + elapsed: elapsed + } + } + catch(err){ + console.error(err) } } - createIndex(fileArr){ - this.miniSearch = new MiniSearch({ - fields: ['filename', 'category', 'type'], - storeFields: ['filename', 'category', 'type', 'date', 'size', 'region', 'path'], - searchOptions: { - boost: { name: 2 }, - fuzzy: this.distance, - }, + async createIndex(fileArr, fields){ + if(!this.miniSearch){ + this.miniSearch = new MiniSearch({ + fields: fields, + storeFields: ['filename', 'category', 'type', 'date', 'size', 'region', 'path'], + }) + + } + else{ + this.miniSearch.removeAll() + } + this.indexing = false + this.miniSearch.addAllAsync(fileArr) + .then( result => { + console.log('File list indexing completed.') + console.log(`Total terms in index: ${this.miniSearch.termCount}`) + this.indexing = false }) - this.miniSearch.addAll(fileArr) } parseHrtimeToSeconds(hrtime){ var seconds = (hrtime[0] + (hrtime[1] / 1e9)).toFixed(3); return seconds; } + sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); + } } \ No newline at end of file diff --git a/server.js b/server.js index 2b876ad..3339e56 100644 --- a/server.js +++ b/server.js @@ -8,35 +8,58 @@ import express from 'express' import http from 'http' import sanitize from 'sanitize' -var fileListPath = './filelist.json' -var categoryListPath = './lib/categories.json' -var categoryList = await parseJsonFile(categoryListPath) -var crawlTime = await fileTime(fileListPath) +let fileListPath = './filelist.json' +let categoryListPath = './lib/categories.json' +let categoryList = await parseJsonFile(categoryListPath) +//TO DO: add if exist to suppress an error +let crawlTime = await fileTime(fileListPath) -var fileList = [] +let searchFields = ['filename', 'category', 'type', 'region'] + +let defaultSettings = { + boost: { + }, + combineWith: 'AND', + fields: searchFields, + fuzzy: 0.2, + prefix: true, +} + +//programmatically set the default boosts while reducing overhead when adding another search field +for(let field in searchFields){ + let fieldName = searchFields[field] + if(searchFields[field] == 'filename'){ + defaultSettings.boost[fieldName] = 2 + } + else { + defaultSettings.boost[fieldName] = 1 + } +} + +let fileList = [] async function getFilesJob(){ console.log('Updating the file list.') fileList = await getAllFiles(categoryList) saveJsonFile(fileListPath, fileList) if(search){ - search.createIndex(fileList) //recreate the search index + search.createIndex(fileList, searchFields) //recreate the search index } crawlTime = await fileTime(fileListPath) console.log(`Finished updating file list. ${fileList.length} found.`) } -if(process.env.FORCE_FILE_REBUILD == "1" || !fileExists(fileListPath) || FileOlderThan(fileListPath, '1d')){ +if(process.env.FORCE_FILE_REBUILD == "1" || !fileExists(fileListPath) || FileOlderThan(fileListPath, '1w')){ await getFilesJob() } else{ fileList = await parseJsonFile(fileListPath) } -var search = new Searcher(fileList) +let search = new Searcher(fileList, searchFields) -var app = express(); -var server = http.createServer(app); +let app = express(); +let server = http.createServer(app); app.use(sanitize.middleware) app.set('view engine', 'ejs') @@ -47,9 +70,13 @@ app.get('/', function(req, res) { }) }) -app.get('/search', function(req, res) { +app.get('/search', async function(req, res) { let query = req.query.q ? req.query.q : '' - let results = search.findAllMatches(query) + let settings = req.query.s ? JSON.parse(req.query.s) : defaultSettings + if(!settings.combineWith){ + delete settings.combineWith //remove if unset to avoid crashing + } + let results = await search.findAllMatches(query, settings) if(process.env.DEBUG == "1"){ console.log(results) } @@ -58,11 +85,12 @@ app.get('/search', function(req, res) { query: query, results: results, crawlTime: crawlTime - }) + }) }) -app.get("/lucky", function(req, res) { - let results = search.findAllMatches(req.query.q) +app.get("/lucky", async function(req, res) { + let settings = req.query.s ? JSON.parse(req.query.s) : defaultSettings + let results = await search.findAllMatches(req.query.q, settings) if(process.env.DEBUG == "1"){ console.log(results) } @@ -78,7 +106,8 @@ app.get("/lucky", function(req, res) { app.get("/settings", function(req, res) { res.render('pages/index', { page: 'settings', - crawlTime: crawlTime + crawlTime: crawlTime, + defaultSettings: defaultSettings }) }) diff --git a/views/pages/results.ejs b/views/pages/results.ejs index b3aac0a..a885aad 100644 --- a/views/pages/results.ejs +++ b/views/pages/results.ejs @@ -14,7 +14,7 @@ _ / / / _ /_/ /_ / _ / / __/ / / / /_ -

Found <%= results.items.length %> result<%= results.items.length != 1 ? 's': '' %> in <%= results.elapsed %> seconds.

+

Found <%= results.items.length %> result<%= results.items.length != 1 ? 's': '' %> in <%= results.elapsed %> seconds. <%= results.elapsed > 10 ? ' Lucky you, you caught the indexer while it was running.' : '' %>

diff --git a/views/pages/search.ejs b/views/pages/search.ejs index 2317c88..7704e13 100644 --- a/views/pages/search.ejs +++ b/views/pages/search.ejs @@ -11,6 +11,7 @@ _ / / / _ /_/ /_ / _ / / __/ / / / /_
+
@@ -19,4 +20,11 @@ _ / / / _ /_/ /_ / _ / / __/ / / / /_
-
\ No newline at end of file + + + \ No newline at end of file diff --git a/views/pages/settings.ejs b/views/pages/settings.ejs index db62eb1..6c4b9c3 100644 --- a/views/pages/settings.ejs +++ b/views/pages/settings.ejs @@ -1,6 +1,6 @@
-
- 
+  

+    
 ______  ___              _____            _____ 
 ___   |/  /____  ___________(_)_____________  /_
 __  /|_/ /__  / / /_  ___/_  /_  _ \_  __ \  __/
@@ -9,6 +9,162 @@ _  /  / / _  /_/ /_  /   _  / /  __/  / / / /_
           /____/                                
                                 Settings
         

-        
Under Construction.

+    

+      

+        

+          
Search Columns

+          

+            <% for(let field in defaultSettings.fields) { %>
+            
+            <% } %>
+          

+        

+        

+          
Search Score Multiplier

+          

+            <% for(let field in defaultSettings.boost) { %>
+            

+              
+              
+            

+            <% } %>
+          

+        

+        

+          
Extras

+          

+            

+              
+              
+            

+          

+          

+            

+              
+              
+            

+          

+        

+        
+      

     

-

\ No newline at end of file
+
+
+ + \ No newline at end of file diff --git a/views/partials/head.ejs b/views/partials/head.ejs index 59e2ada..37bb699 100644 --- a/views/partials/head.ejs +++ b/views/partials/head.ejs @@ -60,4 +60,11 @@ .nav-link, .navbar-brand{ transition: all 0.5s; } + .card { + background-color: #262c2c; + border: 1px solid rgba(255,255,255,.325) + } + .form-control:focus { + border-color: rgb(255, 189, 51)!important; + } \ No newline at end of file