Fix setting request headers in monitor script

lib/caching-queued-fetch.js

@@ -53,7 +53,7 @@ class Queue {
         }
         delete this.cache[url];
         await wait(retryAfter);
-        await this.request(url, options, { verbose }, attempts++);
+        await this.request(url, options, { verbose, fsCachePath }, attempts++);
       } else {
         const error = `HTTP error ${this.cache[url].status} ${this.cache[url].statusText} while fetching ${url}`;
         if (verbose) {
@@ -65,20 +65,22 @@ class Queue {
   }
 
   enqueue(url, options, queueOptions = { verbose: false, interval: module.exports.DEFAULT_INTERVAL}) {
+    if (this.cache[url]) return this.cache[url];
     const { origin } = new URL(url);
     if (!this.originQueue[origin]) {
       this.originQueue[origin] = Promise.resolve(true);
     }
     return new Promise((res, rej) => {
       this.originQueue[origin] = this.originQueue[origin]
         .then(async () => {
+          const inMemory = this.cache[url] !== undefined;
           await this.request(url, options, queueOptions);
-          return this.cache[url];
+          return [this.cache[url], inMemory];
         })
-        .then(async (ret) => {
+        .then(async ([ret, inMemory]) => {
           res(ret);
           // no need to wait if we're not hitting the network
-          if (ret.headers?.get("cache-status")?.match(/fetch-filecache-for-crawling; hit/)) {
+          if (inMemory || ret.headers?.get("cache-status")?.match(/fetch-filecache-for-crawling; hit/)) {
             return;
           }
           return wait(queueOptions.interval);

monitor.js

@@ -10,11 +10,11 @@ const authedFetch = (url) => {
   // this is the value used for the discourse API, and feels like a safe default in general
   let interval = 200;
   const u = new URL(url);
-  const headers =  [
-    ['User-Agent', 'W3C Group dashboard https://github.com/w3c/cg-monitor']
-  ];
+  const headers =  {
+    'User-Agent': 'W3C Group dashboard https://github.com/w3c/cg-monitor'
+  };
   if (u.href.startsWith("https://api.github.com/")) {
-    headers.push(['Authorization', 'token ' + config.ghapitoken]);
+    headers['Authorization'] = 'token ' + config.ghapitoken];
     // Roughly matching github API rate limit of 5000 requests per hour
     interval = 750;
   }
@@ -107,6 +107,7 @@ async function fetchWiki(url) {
     // based on https://stackoverflow.com/a/8573941
     return fetchRSS(url + ".atom");
   }
+  // TODO: handle case of a single wiki page
   return fetchRSS(url + '/api.php?action=feedrecentchanges&days=1000&limit=1000');
 }
 
@@ -176,6 +177,7 @@ async function fetchGithub(url) {
     if (r.status === 404) ownerType = 'users';
     const repos = await recursiveGhFetch(`https://api.github.com/${ownerType}/${owner}/repos?per_page=100`);
     const items = await Promise.all(repos.filter(r => !r.fork).map(r => r.owner ? fetchGithubRepo(r.owner.login, r.name) : []));
+    // TODO: this should instead be sent as a collection of services (1 per repo)
     return { items: items.flat() };
   } else {
     return {items: await fetchGithubRepo(owner, repo)};

test/caching-queued-fetch.js

@@ -173,15 +173,15 @@ describe('The HTTP request manager', function () {
     assert.equal(response.status, 304);
   });
 
-  it('loads a response from the FS cache', async () => {
+  it('loads a response from the FS cache with the proper request headers', async () => {
     const ims = "Fri, 20 Oct 2023 16:51:59 GMT";
     const fsUrl = testBaseUrl + "fs";
     const u = new URL(fsUrl);
     const interceptor = agent
           .get(u.origin)
-          .intercept({path: u.pathname, method: "GET", headers: {"if-modified-since": ims}})
+          .intercept({path: u.pathname, method: "GET", headers: {"if-modified-since": ims, authorization: "test"}})
         .reply(304);
-    const response = await queuedFetch(fsUrl, {}, {fsCachePath: "test/fs-cache"});
+    const response = await queuedFetch(fsUrl, {headers: {authorization: "test"}}, {fsCachePath: "test/fs-cache"});
     assert.match(response.headers.get("cache-status"), /hit/)
   });