Add filesystem cache

w3c · Oct 20, 2023 · a69a98e · a69a98e
1 parent 2f297f5
commit a69a98e
Show file tree

Hide file tree

Showing 5 changed files with 1,017 additions and 5,360 deletions.
diff --git a/lib/caching-queued-fetch.js b/lib/caching-queued-fetch.js
@@ -1,5 +1,4 @@
-const {END_EVENT, ITEM_EVENT, default:RequestQueue} = require('limited-request-queue');
-
+const fsCacheFetch = require('fetch-filecache-for-crawling');
 
 let ghToken = "";
 module.exports.DEFAULT_INTERVAL = 50;
@@ -13,7 +12,7 @@ class Queue {
   maxRetry = 2;
   cache = {};
   originQueue = {};
-  async request(url, options, {verbose}, attempts = 0) {
+  async request(url, options, {verbose, fsCachePath}, attempts = 0) {
     if (!this.cache[url]) {
       if (attempts > this.maxRetry) {
         const error = `HTTP error ${this.cache[url].status} ${this.cache[url].statusText} while fetching ${url} (tried ${attempts} times)`;
@@ -25,7 +24,12 @@ class Queue {
       if (verbose) {
         console.log("fetching " + url);
       }
-      const r = await fetch(url, options);
+      const _f = fsCachePath ? fsCacheFetch : fetch;
+      if (fsCachePath) {
+        options.logToConsole = verbose;
+        options.cacheFolder = fsCachePath;
+      }
+      const r = await _f(url, options);
       this.cache[url] = { status: r.status, statusText: r.statusText, headers: r.headers, body: await r.text()};
     }
     if (this.cache[url].status >= 400) {

diff --git a/monitor.js b/monitor.js
@@ -18,7 +18,7 @@ const authedFetch = (url) => {
     // Roughly matching github API rate limit of 5000 requests per hour
     interval = 750;
   }
-  return queuedFetch(url, { headers }, { interval, verbose: true });
+  return queuedFetch(url, { headers }, { interval, verbose: true, fsCachePath: ".cache" });
 };