the big fat settings update + asynchronous indexing

2026-01-15 16:33:15 -03:00 · 2024-10-19 00:08:34 -06:00
parent b39c5631dd
commit 0c43711820
7 changed files with 263 additions and 42 deletions
--- a/.env
+++ b/.env
@@ -1,6 +1,4 @@
 PORT=8062
 BIND_ADDRESS=0.0.0.0
-FUZZY_DISTANCE=0.2
-MIN_MATCH=15
 FORCE_FILE_REBUILD=0
 DEBUG=0
--- a/lib/search.js
+++ b/lib/search.js
@@ -1,35 +1,58 @@
 import MiniSearch from 'minisearch'

 export default class Searcher{
-    constructor(fileArr){
+    constructor(fileArr, fields){
        this.distance = parseFloat(process.env.FUZZY_DISTANCE)
        this.minMatch = parseFloat(process.env.MIN_MATCH)
-        this.createIndex(fileArr)
+        this.indexing = false
+        this.createIndex(fileArr, fields)
    }

-    findAllMatches(query){
-        var startTime = process.hrtime();
-        let results = this.miniSearch.search(query)
-        var elapsed = this.parseHrtimeToSeconds(process.hrtime(startTime));
-        return {
-            items: results,
-            elapsed: elapsed
+    async findAllMatches(query, options){
+        try{
+            var startTime = process.hrtime();
+            while(this.indexing){
+                await this.sleep(1000)
+            }
+            if(process.env.DEBUG == "1"){
+                console.log(options)
+            }
+            let results = this.miniSearch.search(query, options)
+            var elapsed = this.parseHrtimeToSeconds(process.hrtime(startTime));
+            return {
+                items: results,
+                elapsed: elapsed
+            }
+        }
+        catch(err){
+            console.error(err)
        }
    }
    
-    createIndex(fileArr){
-        this.miniSearch = new MiniSearch({
-            fields: ['filename', 'category', 'type'],
-            storeFields: ['filename', 'category', 'type', 'date', 'size', 'region', 'path'],
-            searchOptions: {
-                boost: { name: 2 },
-                fuzzy: this.distance,
-            },
+    async createIndex(fileArr, fields){
+        if(!this.miniSearch){
+            this.miniSearch = new MiniSearch({
+                fields: fields,
+                storeFields: ['filename', 'category', 'type', 'date', 'size', 'region', 'path'],
+            })
+            
+        }
+        else{
+            this.miniSearch.removeAll()
+        }
+        this.indexing = false
+        this.miniSearch.addAllAsync(fileArr)
+        .then( result => {
+            console.log('File list indexing completed.')
+            console.log(`Total terms in index: ${this.miniSearch.termCount}`)
+            this.indexing = false
        })
-        this.miniSearch.addAll(fileArr)
    }
    parseHrtimeToSeconds(hrtime){
        var seconds = (hrtime[0] + (hrtime[1] / 1e9)).toFixed(3);
        return seconds;
    }
+    sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
 }
--- a/server.js
+++ b/server.js
@@ -8,35 +8,58 @@ import express from 'express'
 import http from 'http'
 import sanitize from 'sanitize'

-var fileListPath = './filelist.json'
-var categoryListPath = './lib/categories.json'
-var categoryList = await parseJsonFile(categoryListPath)
-var crawlTime = await fileTime(fileListPath)
+let fileListPath = './filelist.json'
+let categoryListPath = './lib/categories.json'
+let categoryList = await parseJsonFile(categoryListPath)
+//TO DO: add if exist to suppress an error
+let crawlTime = await fileTime(fileListPath)

-var fileList = []
+let searchFields = ['filename', 'category', 'type', 'region']
+
+let defaultSettings = {
+  boost: {
+  },
+  combineWith: 'AND',
+  fields:  searchFields,
+  fuzzy: 0.2,
+  prefix: true,
+}
+
+//programmatically set the default boosts while reducing overhead when adding another search field
+for(let field in searchFields){
+  let fieldName = searchFields[field]
+  if(searchFields[field] == 'filename'){
+    defaultSettings.boost[fieldName] = 2
+  }
+  else {
+    defaultSettings.boost[fieldName] = 1
+  }
+}
+
+let fileList = []

 async function getFilesJob(){
  console.log('Updating the file list.')
  fileList = await getAllFiles(categoryList)
  saveJsonFile(fileListPath, fileList)
  if(search){
-    search.createIndex(fileList) //recreate the search index
+    search.createIndex(fileList, searchFields) //recreate the search index
  }
  crawlTime = await fileTime(fileListPath)
  console.log(`Finished updating file list. ${fileList.length} found.`)
 }

-if(process.env.FORCE_FILE_REBUILD == "1" || !fileExists(fileListPath) || FileOlderThan(fileListPath, '1d')){
+if(process.env.FORCE_FILE_REBUILD == "1" || !fileExists(fileListPath) || FileOlderThan(fileListPath, '1w')){
  await getFilesJob()
 }
 else{
  fileList = await parseJsonFile(fileListPath)
 }

-var search = new Searcher(fileList)
+let search = new Searcher(fileList, searchFields)

-var app = express();
-var server = http.createServer(app);
+let app = express();
+let server = http.createServer(app);
 app.use(sanitize.middleware)
 app.set('view engine', 'ejs')

@@ -47,9 +70,13 @@ app.get('/', function(req, res) {
  })  
 })

-app.get('/search', function(req, res) {
+app.get('/search', async function(req, res) {
  let query = req.query.q ? req.query.q : ''
-  let results = search.findAllMatches(query)
+  let settings = req.query.s ? JSON.parse(req.query.s) : defaultSettings
+  if(!settings.combineWith){
+    delete settings.combineWith //remove if unset to avoid crashing
+  }
+  let results = await search.findAllMatches(query, settings)
  if(process.env.DEBUG == "1"){
    console.log(results)
  }
@@ -58,11 +85,12 @@ app.get('/search', function(req, res) {
    query: query,
    results: results,
    crawlTime: crawlTime
-  })  
+    })  
 })

-app.get("/lucky", function(req, res) {
-  let results = search.findAllMatches(req.query.q)
+app.get("/lucky", async function(req, res) {
+  let settings = req.query.s ? JSON.parse(req.query.s) : defaultSettings
+  let results = await search.findAllMatches(req.query.q, settings)
  if(process.env.DEBUG == "1"){
    console.log(results)
  }
@@ -78,7 +106,8 @@ app.get("/lucky", function(req, res) {
 app.get("/settings", function(req, res) {
  res.render('pages/index', {
    page: 'settings',
-    crawlTime: crawlTime
+    crawlTime: crawlTime,
+    defaultSettings: defaultSettings
  })
 })

--- a/views/pages/results.ejs
+++ b/views/pages/results.ejs
@@ -14,7 +14,7 @@ _  /  / / _  /_/ /_  /   _  / /  __/  / / / /_
                <button type="submit" class="btn btn-secondary ml-2">Search</button>
            </div>
        </div>
-        <p class="m-2">Found <%= results.items.length %> result<%= results.items.length != 1 ? 's': '' %> in <%= results.elapsed %> seconds.</p>
+        <p class="m-2">Found <%= results.items.length %> result<%= results.items.length != 1 ? 's': '' %> in <%= results.elapsed %> seconds. <%= results.elapsed > 10 ? ' Lucky you, you caught the indexer while it was running.' : '' %></p>
    </form>
    
    <div class="col-sm-12 w-100 mt-3">
--- a/views/pages/search.ejs
+++ b/views/pages/search.ejs
@@ -11,6 +11,7 @@ _  /  / / _  /_/ /_  /   _  / /  __/  / / / /_
        </pre>
        <div class="text-center text-white">
            <form>
+                <input type="hidden" name="s" id="searchSettings"> 
                <input type="text" style="width: 80%;display: inline;" class="form-control bg-dark text-white mb-2" name="q">
                <div>
                    <button type="submit" formaction="/search" class="btn btn-secondary">Myrient Search</button>
@@ -19,4 +20,11 @@ _  /  / / _  /_/ /_  /   _  / /  __/  / / / /_
            </form>
        </div>
    </div>
-</div>
+</div>
+
+<script>
+    settingStore = localStorage.getItem('settings')
+    if(typeof settingStore == 'string'){
+        document.getElementById('searchSettings').value = settingStore
+    }
+</script>
--- a/views/pages/settings.ejs
+++ b/views/pages/settings.ejs
@@ -1,6 +1,6 @@
 <div class="row h-50 w-100 m-0">
-    <div class="col-sm-12 my-auto text-center">
-        <pre style="font: 20px / 19px monospace; color: white; text-align: center; overflow: hidden;">
+  <div class="col-sm-12 my-auto text-center">
+    <pre style="font: 20px / 19px monospace; color: white; text-align: center; overflow: hidden;">
 ______  ___              _____            _____ 
 ___   |/  /____  ___________(_)_____________  /_
 __  /|_/ /__  / / /_  ___/_  /_  _ \_  __ \  __/
@@ -9,6 +9,162 @@ _  /  / / _  /_/ /_  /   _  / /  __/  / / / /_
          /____/                                
                                Settings
        </pre>
-        <p>Under Construction.</p>
+    <div class="card w-auto mx-auto text-center d-inline-block p-3">
+      <form>
+        <div class="form-group">
+          <h4>Search Columns</h4>
+          <div class="">
+            <% for(let field in defaultSettings.fields) { %>
+            <label class="checkbox-inline p-1" for="<%= defaultSettings.fields[field] %>">
+              <input type="checkbox" name="<%= defaultSettings.fields[field] %>" id="<%= defaultSettings.fields[field] %>" value="true">
+              <%= defaultSettings.fields[field].charAt(0).toUpperCase() + defaultSettings.fields[field].slice(1) %>
+            </label>
+            <% } %>
+          </div>
+        </div>
+        <div class="form-group">
+          <h4>Search Score Multiplier</h4>
+          <div class="">
+            <% for(let field in defaultSettings.boost) { %>
+            <div class="d-inline-block">
+              <label for="<%= field %>boost"><%= field.charAt(0).toUpperCase() + field.slice(1) %></label>
+              <input type="number" class="form-control bg-dark text-white" name="<%= field %>boost" id="<%= field %>boost" min="1" max="5">
+            </div>
+            <% } %>
+          </div>
+        </div>
+        <div class="form-group">
+          <h4>Extras</h4>
+          <div class="form-group">
+            <div class="d-inline-block">
+              <label for="fuzzy">Fuzzy Value</label>
+              <input type="number" class="form-control bg-dark text-white" id="fuzzy" name="fuzzy" step="0.01" min="0" max="1">
+            </div>
+          </div>
+          <div class="form-group">
+            <div class="">
+              <label class="checkbox-inline p-1">
+                <input type="checkbox" id="prefix" value="true">
+                Allow Prefixes
+              </label>
+              <label class="checkbox-inline p-1">
+                <input type="checkbox" id="combineWith" value="AND">
+                Match All Words
+              </label>
+            </div>
+          </div>
+        </div>
+        <button type="button" class="btn btn-secondary mb-2" action="#" id="saveSettings">Save Settings</button>
+      </form>
    </div>
-</div>
+  </div>
+</div>
+
+<script defer>
+  defaults = <%-JSON.stringify(defaultSettings)%>
+
+  function handleSettings(saving = false) {
+    //You're more than welcome to clean this up if you want to.
+    for (let key in defaults) {
+      if (defaults[key] instanceof Object && !(defaults[key] instanceof Array)) {
+        for (let subkey in defaults[key]) {
+          //load default if it doesn't exist
+          if (!settings[key]) {
+            //create an empty object if for some reason it doesn't exist
+            settings[key] = {}
+          }
+          if (!settings[key][subkey]) {
+            settings[key][subkey] = defaults[key][subkey]
+          }
+
+          if (typeof defaults[key][subkey] == "boolean") {
+            if (saving) {
+              settings[key][subkey] = document.getElementById(subkey + key).checked
+            } else {
+              document.getElementById(subkey + key).checked = settings[key][subkey]
+            }
+          } else {
+            if (saving) {
+              settings[key][subkey] = document.getElementById(subkey + key).value
+            } else {
+              document.getElementById(subkey + key).value = settings[key][subkey]
+            }
+          }
+        }
+      } else {
+        //load in defaults if it doesn't exist and it's not a subkey
+        if (typeof settings[key] == 'undefined') {
+          settings[key] = defaults[key]
+        }
+        if (defaults[key] instanceof Array) {
+          if (saving) {
+            settings[key] = []
+            for (let item in defaults[key]) {
+              if (document.getElementById(defaults[key][item]).checked) {
+                settings[key].push(defaults[key][item])
+              }
+            }
+          } else {
+            for (let item in settings[key]) {
+              document.getElementById(settings[key][item]).checked = true
+            }
+          }
+          continue
+        } else {
+          if (typeof defaults[key] == "boolean") {
+            if (saving) {
+              settings[key] = document.getElementById(key).checked
+            } else {
+              document.getElementById(key).checked = settings[key]
+            }
+          } else {
+            let inputElem = document.getElementById(key)
+            if (saving) {
+              if (inputElem.getAttribute('type') == 'checkbox') {
+                if (inputElem.checked) {
+                  settings[key] = document.getElementById(key).value
+                } else {
+                  settings[key] = ''
+                }
+              }
+            } else {
+              if (inputElem.getAttribute('type') == 'checkbox') {
+                if (settings[key]) {
+                  inputElem.checked = true
+                }
+              } else {
+                document.getElementById(key).value = settings[key]
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  async function loadSettings() {
+    settingStore = localStorage.getItem('settings')
+    settings = undefined
+    if (settingStore) {
+      try {
+        settings = await JSON.parse(settingStore)
+      } catch {
+        console.log("Settings corrupted, saving defaults instead")
+      }
+    }
+    if (typeof settings == 'undefined') {
+      console.log("loading defaults")
+      localStorage.setItem('settings', await JSON.stringify(defaults))
+      settings = defaults
+    }
+    handleSettings()
+  }
+
+  async function saveSettings() {
+    console.log("saving")
+    handleSettings(true)
+    localStorage.setItem('settings', await JSON.stringify(settings))
+  }
+  document.getElementById('saveSettings').onclick = saveSettings
+  document.body.onload = loadSettings
+</script>
--- a/views/partials/head.ejs
+++ b/views/partials/head.ejs
@@ -60,4 +60,11 @@
    .nav-link, .navbar-brand{
        transition: all 0.5s;
    }
+    .card {
+        background-color: #262c2c;
+        border: 1px solid rgba(255,255,255,.325)
+    }
+    .form-control:focus {
+        border-color: rgb(255, 189, 51)!important;
+    }
    </style>