add non-game content filter

lib/nonGameTerms.json

@@ -18,6 +18,7 @@
     "editor",
     "emulator",
     "expansion",
+    "figurine",
     "firmware",
     "guide",
     "hack",
@@ -26,6 +27,7 @@
     "installer",
     "intro",
     "json",
+    "jpg",
     "manual",
     "mod",
     "movie",
@@ -44,6 +46,7 @@
     "sdk",
     "setup",
     "soundtrack",
+    "sqlite",
     "terms",
     "tool",
     "trainer",
@@ -51,7 +54,8 @@
     "update",
     "utility",
     "video",
-    "Virtual Console",
-    "wallpaper"
+    "virtual console",
+    "wallpaper",
+    "xml"
   ]
 }

lib/services/elasticsearch.js

@@ -1,6 +1,9 @@
 import { Client } from '@elastic/elasticsearch';
 import debugPrint from '../debugprint.js';
 import { File } from '../models/index.js';
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, resolve } from 'path';
 
 const client = new Client({
   node: process.env.ELASTICSEARCH_URL || 'http://localhost:9200'
@@ -8,6 +11,23 @@ const client = new Client({
 
 const INDEX_NAME = 'myrient_files';
 
+// Cache for nonGameTerms
+let nonGameTermsCache = null;
+
+function getNonGameTerms() {
+  if (nonGameTermsCache) {
+    return nonGameTermsCache;
+  }
+
+  const __filename = fileURLToPath(import.meta.url);
+  const __dirname = dirname(__filename);
+
+  const nonGameTermsPath = resolve(__dirname, '../../lib/nonGameTerms.json');
+  nonGameTermsCache = JSON.parse(readFileSync(nonGameTermsPath, 'utf8'));
+
+  return nonGameTermsCache;
+}
+
 export async function initElasticsearch() {
   try {
     const indexExists = await client.indices.exists({ index: INDEX_NAME });
@@ -111,7 +131,7 @@ export async function search(query, options) {
   const searchQuery = {
     index: INDEX_NAME,
     body: {
-      size: 1000,
+      size: 1500,
       query: {
         bool: {
           must: buildMustClauses(query, options),
@@ -132,7 +152,6 @@ export async function search(query, options) {
   try {
     const startTime = process.hrtime();
     const response = await client.search(searchQuery);
-    const elapsed = parseHrtimeToSeconds(process.hrtime(startTime));
 
     // Fetch full records from PostgreSQL for the search results
     const ids = response.hits.hits.map(hit => hit._id);
@@ -146,13 +165,28 @@ export async function search(query, options) {
       return map;
     }, {});
 
-    // Combine Elasticsearch results with full PostgreSQL records
+    // Build results with full PostgreSQL records
+    let results = response.hits.hits.map(hit => ({
+      ...recordMap[hit._id].dataValues,
+      score: hit._score,
+      highlights: hit.highlight
+    }));
+
+    // Apply non-game content filtering in JavaScript if the option is enabled
+    if (options.hideNonGame) {
+      const nonGameTerms = getNonGameTerms();
+      const termPatterns = nonGameTerms.terms.map(term => new RegExp(term, 'i'));
+
+      // Filter results in JavaScript (much faster than complex Elasticsearch queries)
+      results = results.filter(item => {
+        // Check if filename contains any of the non-game terms
+        return !termPatterns.some(pattern => pattern.test(item.filename));
+      });
+    }
+
+    const elapsed = parseHrtimeToSeconds(process.hrtime(startTime));
     return {
-      items: response.hits.hits.map(hit => ({
-        ...recordMap[hit._id].dataValues,
-        score: hit._score,
-        highlights: hit.highlight
-      })),
+      items: results,
       elapsed
     };
   } catch (error) {

server.js

@@ -44,6 +44,7 @@ let defaultSettings = {
   fields: searchFields,
   fuzzy: 0,
   prefix: true,
+  hideNonGame: true,
 };
 
 //programmatically set the default boosts while reducing overhead when adding another search field
@@ -79,7 +80,9 @@ let defaultOptions = {
   fileCount: fileCount,
   termCount: 0,
   generateAsciiArt: generateAsciiArt,
-  isEmulatorCompatible: isEmulatorCompatible
+  isEmulatorCompatible: isEmulatorCompatible,
+  isNonGameContent: isNonGameContent,
+  nonGameTerms: nonGameTerms
 };
 
 function updateDefaults(){
@@ -143,7 +146,8 @@ app.get("/search", async function (req, res) {
     results: results,
     pageNum: pageNum,
     indexing: search.indexing,
-    urlPrefix: urlPrefix
+    urlPrefix: urlPrefix,
+    settings: settings
   };
   let page = "results";
   options = buildOptions(page, options);

views/pages/results.ejs

@@ -30,7 +30,14 @@
             </div>
             <ul class="SuggestionList col-sm-12" id="suggestionList" style="width: 50%;left: 195px;"></ul>
         </div>
-        <p class="m-2">Found <%= results.items.length %> result<%= results.items.length != 1 ? 's': '' %> in <%= results.elapsed %> seconds. <%= indexing ? "Indexing in progress, if the list is missing something please try reloading in a few minutes" : "" %></p>
+        <p class="m-2">Found <%= results.items.length %> result<%= results.items.length != 1 ? 's': '' %> in <%= results.elapsed %> seconds. <%= indexing ? "Indexing in progress, if the list is missing something please try reloading in a few minutes" : "" %>
+            <% if (settings.hideNonGame) { %>
+            <span class="badge badge-info" data-toggle="tooltip" data-placement="top" title="Hiding ROM hacks, patches, and other non-game content. Disable this in Settings.">
+                Non-game content filter is active
+                <a href="/settings" class="text-white ml-1"><i class="bi bi-gear-fill"></i></a>
+            </span>
+            <% } %>
+        </p>
     </form>
 
     <div class="col-sm-12 w-100 mt-3">
@@ -165,4 +172,9 @@
             "bottomStart": ''
         }
     });
+
+    // Initialize tooltips
+    $(function () {
+        $('[data-toggle="tooltip"]').tooltip()
+    });
 </script>

views/pages/settings.ejs

@@ -49,6 +49,10 @@
                 <input type="checkbox" id="combineWith" value="AND">
                 Match All Words <i class="bi bi-question-circle" data-toggle="tooltip" data-placement="top" title="Requires all words in the search query to match."></i>
               </label>
+              <label class="checkbox-inline p-1">
+                <input type="checkbox" id="hideNonGame" value="true">
+                Hide Non-Game Content <i class="bi bi-question-circle" data-toggle="tooltip" data-placement="top" title="Filters out ROM hacks, patches, artwork, and other non-game content from search results."></i>
+              </label>
             </div>
           </div>
         </div>
@@ -96,9 +100,11 @@
         if(typeof settings.combineWith == 'undefined') {settings.combineWith = defaults.combineWith}
         if(typeof settings.fuzzy == 'undefined') {settings.fuzzy = defaults.fuzzy}
         if(typeof settings.prefix == 'undefined') {settings.prefix = defaults.prefix}
+        if(typeof settings.hideNonGame == 'undefined') {settings.hideNonGame = defaults.hideNonGame}
         document.getElementById('combineWith').checked = settings.combineWith ? true : false
         document.getElementById('fuzzy').value = settings.fuzzy
         document.getElementById('prefix').checked = settings.prefix
+        document.getElementById('hideNonGame').checked = settings.hideNonGame
       }
 
       function saveSettings(){
@@ -112,6 +118,7 @@
         settings.combineWith = document.getElementById('combineWith').checked ? 'AND' : ''
         settings.fuzzy = parseFloat (document.getElementById('fuzzy').value)
         settings.prefix = document.getElementById('prefix').checked
+        settings.hideNonGame = document.getElementById('hideNonGame').checked
         localStorage.setItem('settings', JSON.stringify(settings))
         window.location.href = '/'
       }