implement task queue for metadata retrieval to respect igdb limits

lib/metadatasearch.js

@@ -17,8 +17,8 @@ import {
   offset,
 } from "@phalcode/ts-igdb-client";
 import { File, Metadata } from "./database.js";
-import { Sequelize } from "sequelize";
 import debugPrint from "./debugprint.js";
+import TaskQueue from "./taskqueue.js";
 
 export default class MetadataSearch {
   constructor() {
@@ -27,6 +27,7 @@ export default class MetadataSearch {
       client_secret: process.env.TWITCH_CLIENT_SECRET,
     };
     this.setupClient();
+    this.queue = new TaskQueue();
   }
   gameFields = [
     "name",
@@ -139,7 +140,8 @@ export default class MetadataSearch {
     //special garbage because SOMEONE doesn't value consistency
     string = string.replace("Nintendo Wii", "Wii");
     string = string.replace("Nintendo Game Boy", "Game Boy");
-    string = string.replace("Sony PlayStation", "Playstation");
+    string = string.replace("Nintendo Satellaview", "Satellaview");
+    string = string.replace("Sony PlayStation", "PlayStation");
     string = string.replace("Microsoft Xbox", "Xbox");
     return [where("platforms.name", op, string, WhereFlags.CONTAINS)];
   }
@@ -157,7 +159,7 @@ export default class MetadataSearch {
 
   async getGamesMetadata(games) {
     try {
-      if (!this.authorized || !games.length) return [];
+      if (!this.authorized || !games.length) return;
       let gameQuery = [];
       for (let x in games) {
         if (!(await games[x].getDetails()))
@@ -171,10 +173,10 @@ export default class MetadataSearch {
             }
           }
       }
-      if (!gameQuery.length) return [];
-      let gameMetas = await this.getMetadata(gameQuery);
+      if (!gameQuery.length) return;
+      let gameMetas = await this.queue.enqueue(this.getMetadata, this, gameQuery)
       debugPrint(JSON.stringify(gameMetas, null, 2));
-      if (!gameMetas.length) return [];
+      if (!gameMetas.length) return;
       for (let x in gameMetas) {
         if (gameMetas[x].result.length) {
           await this.addMetadataToDb(
@@ -186,6 +188,14 @@ export default class MetadataSearch {
           games[x].save();
         }
       }
+    } catch (error) {
+      console.error("Error getting metadata:", error);
+    }
+  }
+
+  async queueGetGamesMetadata(games) {
+    try {
+      await this.getGamesMetadata(games); //we don't actually care as long as it finishes
       let details = await Promise.all(games.map((game) => game.getDetails()));
       let combined = [];
       //make sure the metadata gets included with the gamedata

lib/taskqueue.js

@@ -0,0 +1,81 @@
+export default class TaskQueue {
+  constructor(maxTasksPerSecond = 4, maxSimultaneousTasks = 8) {
+    this.maxTasksPerSecond = maxTasksPerSecond;
+    this.maxSimultaneousTasks = maxSimultaneousTasks;
+    this.queue = [];
+    this.processing = false;
+    this.lastProcessTime = 0;
+    this.taskCount = 0;
+    this.tasksWaiting = 0;
+  }
+
+  async enqueue(taskFunction, that=this, ...args) {
+    return new Promise((resolve, reject) => {
+      this.queue.push({
+        taskFunction,
+        that,
+        args,
+        resolve,
+        reject,
+      });
+
+      if (!this.processing) {
+        this.processQueue();
+      }
+    });
+  }
+
+  async processQueue() {
+    if (this.processing || this.queue.length === 0) {
+      return;
+    }
+
+    this.processing = true;
+
+    while (this.queue.length > 0) {
+      const now = Date.now();
+
+      if (now - this.lastProcessTime >= 1000) {
+        this.taskCount = 0;
+        this.lastProcessTime = now;
+      }
+
+      if (
+        this.taskCount >= this.maxTasksPerSecond ||
+        this.tasksWaiting >= this.maxSimultaneousTasks
+      ) {
+        const waitTime = 1000 - (now - this.lastProcessTime);
+        await this.sleep(waitTime);
+        continue;
+      }
+
+      const task = this.queue.shift();
+      this.taskCount++;
+      this.tasksWaiting++;
+
+      try {
+        const result = await task.taskFunction.apply(task.that, task.args);
+        this.tasksWaiting--;
+        task.resolve(result);
+      } catch (error) {
+        task.reject(error);
+      }
+    }
+
+    this.processing = false;
+  }
+
+  sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  getStatus() {
+    return {
+      queueLength: this.queue.length,
+      maxQueueSize: this.maxQueueSize,
+      tasksPerSecond: this.maxTasksPerSecond,
+      currentTaskCount: this.taskCount,
+      isProcessing: this.processing,
+    };
+  }
+}

server.js

@@ -200,7 +200,7 @@ app.get("/search", async function (req, res) {
   settings.page = pageNum - 1;
   let results = await search.findAllMatches(query, settings);
   debugPrint(results);
-  let metas = await metadataSearch.getGamesMetadata(results.db);
+  let metas = await metadataSearch.queueGetGamesMetadata(results.db);
   if (results.count && pageNum == 1) {
     queryCount += 1;
     await QueryCount.update({ count: queryCount }, { where: { id: 1 } });