Fix after GitHub's interface has changed // ghuser-io/ghuser.io#172

Author: lourot

README.md

@@ -67,47 +67,11 @@ Normally in order to retrieve all repositories a user has interacted with, one s
 [GitHub Events API](https://stackoverflow.com/a/37554614/1855917). Unfortunately it returns
 [only the last 90 days](https://stackoverflow.com/a/38274468/1855917), so we don't use it.
 
-Instead we noticed that the "Contribution Activity" on the
-[profile pages](https://github.com/AurelienLourot) queries such URLs in the background:
-
-* https://github.com/users/AurelienLourot/created_repositories?from=2018-05-17&to=2018-05-17
-* https://github.com/users/AurelienLourot/created_pull_request_reviews?from=2018-05-17&to=2018-05-17
-* https://github.com/users/AurelienLourot/created_issues?from=2018-07-10&to=2018-07-10
-
-So we're doing the same :)
-
-> **NOTES**:
->
-> * It seems like `created_issues` URLs don't deliver "hot issues" (issues which received
->   more comments than others):
->
->   ```bash
->   $ curl -s "https://github.com/users/AurelienLourot/created_issues?from=2015-09-23&to=2015-09-23"
->   <div class="profile-rollup-content">
->   </div>
->   ```
->
->   To get these, we also query the profile itself:
->
->   ```bash
->   $ curl -s "https://github.com/AurelienLourot?from=2015-09-23" | grep issues/
->           <a class="text-gray-dark" href="/jfrog/build-info/issues/60">Publish properties aren&#39;t used by build-info-extractor-gradle?</a>
->   ```
->
-> * In the past we used to get the pull requests from a `created_pull_requests` URL but this got
->   removed. We now get the pull requests from the profile itself as well:
->
->   ```bash
->   $ curl -s "https://github.com/AurelienLourot?from=2017-08-28" | grep pull/
->               <a href="/tt-gf/ant-ivy/pull/2" class="content-title no-underline">
->   ```
-
-> * Same now with `created_commits`:
->
->   ```bash
->   $ curl -s "https://github.com/AurelienLourot?from=2017-08-27" | grep commits?
->         <a href="/AurelienLourot/mybeir.ut/commits?author=AurelienLourot&amp;since=2017-08-27&amp;until=2017-08-28" class="f6 muted-link ml-1">
->   ```
+Instead we noticed that the "Contribution Activity" section's content on the
+[profile pages](https://github.com/AurelienLourot) comes from URLs like
+https://github.com/AurelienLourot?from=2018-10-09 .
+
+So we're fetching these URLs too and parsing their output.
 
 ### Why is it so slow?
 
@@ -135,6 +99,10 @@ doesn't discover commits in forks.
 
 ## Changelog
 
+**2.2.4** (2018-11-11):
+  * [ghuser-io/ghuser.io#172](https://github.com/ghuser-io/ghuser.io/issues/172) Fix after GitHub's
+    interface has changed. The `created_issues` "endpoint" is gone.
+
 **2.2.3** (2018-10-20):
   * [ghuser-io/ghuser.io#172](https://github.com/ghuser-io/ghuser.io/issues/172) Fix after GitHub's
     interface has changed. The `created_commits` "endpoint" is gone.

index.js

@@ -83,39 +83,6 @@
   };
 
   const getContribs = async (user, joinDate, since, until, ora, console, alsoIssues) => {
-    const issuesHtmlToRepos = html => {
-      const repos = new Set();
-
-      const handler = new htmlparser.DefaultHandler((error, dom) => {});
-      const parser = new htmlparser.Parser(handler);
-      parser.parseComplete(html);
-      for (let i = 0; i < handler.dom.length; ++i) {
-        if (handler.dom[i].type === 'tag' && handler.dom[i].name === 'div') {
-          const div1 = handler.dom[i].children;
-          for (let j = 0; j < div1.length; ++j) {
-            if (div1[j].type === 'tag' && div1[j].name === 'div') {
-              const div2 = div1[j].children;
-              for (let k = 0; k < div2.length; ++k) {
-                if (div2[k].type === 'tag' && div2[k].name === 'button') {
-                  const button = div2[k].children;
-                  for (let l = 0; l < button.length; ++l) {
-                    if (button[l].type === 'tag' && button[l].name === 'span') {
-                      const span = button[l].children[0].data.trim();
-                      if (span) {
-                        repos.add(span);
-                      }
-                    }
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-
-      return repos;
-    };
-
     const bigHtmlToRepos = (html, type) => { // type: 'issues', 'pull' or 'commits'
       const repos = new Set();
 
@@ -177,18 +144,7 @@
           )).text();
           const commitsRepos = bigHtmlToRepos(bigHtml, 'commits');
           const prsRepos = bigHtmlToRepos(bigHtml, 'pull');
-
-          let issuesRepos = [];
-          let hotIssuesRepos = [];
-          if (alsoIssues) {
-            const userIssues = await fetchRetry(
-              `https://github.com/users/${user}/created_issues?from=${currDateStr}&to=${currDateStr}`,
-            );
-            const userIssuesHtml = await userIssues.text();
-            issuesRepos = issuesHtmlToRepos(userIssuesHtml);
-
-            hotIssuesRepos = bigHtmlToRepos(bigHtml, 'issues');
-          }
+          const issuesRepos = alsoIssues ? bigHtmlToRepos(bigHtml, 'issues') : [];
 
           progressSpinner.stop(); // temporary stop for logging
           for (const repo of commitsRepos) {
@@ -203,10 +159,6 @@
             console.log(`${currDateStr}: (issues)     ${repo}`);
             result.add(repo);
           }
-          for (const repo of hotIssuesRepos) {
-            console.log(`${currDateStr}: (hot issues) ${repo}`);
-            result.add(repo);
-          }
           progressSpinner.start(
             progressMsg(false, alsoIssues, ++numOfQueriedDays, numOfDaysToQuery)
           );

package.json

@@ -73,6 +73,6 @@
       ".js"
     ],
     "check-coverage": true,
-    "lines": 70
+    "lines": 75
   }
 }

test.js

@@ -23,10 +23,10 @@ const ora = string => {
 };
 
 test('fetches commits and PRs', async t => {
-  /* AurelienLourot had the following activity:
-     * 2017-08-26: nothing
-     * 2017-08-27: 14 commits in AurelienLourot/mybeir.ut
-     * 2017-08-28: 1 PR in tt-gf/ant-ivy */
+  // AurelienLourot had the following activity:
+  // * 2017-08-26: nothing
+  // * 2017-08-27: 14 commits in AurelienLourot/mybeir.ut
+  // * 2017-08-28: 1 PR in tt-gf/ant-ivy
 
   const result = await m.fetch('AurelienLourot', '2017-08-26', '2017-08-28', ora, console);
 
@@ -42,19 +42,23 @@ test('fetches commits and PRs', async t => {
   t.true(result.has('AurelienLourot/myberl.in'));
 });
 
-test('fetches issues', async t => {
-  /* RichardLitt had the following activity:
-     * 2018-08-07: 1 issue in orbitdb/welcome */
+test('fetches issues and reviewed pull requests', async t => {
+  // RichardLitt had the following activity:
+  // * 2018-08-07: 1 issue in orbitdb/welcome
+  //               4 reviewed pull requests in rtfd/readthedocs.org
 
   const result = await m.fetch('RichardLitt', '2018-08-07', '2018-08-07', ora, console, true);
-  t.is(result.size, 1);
+  t.is(result.size, 2);
   t.true(result.has('orbitdb/welcome'));
+  t.true(result.has('rtfd/readthedocs.org'));
 });
 
 test('fetches hot issues', async t => {
-  /* AurelienLourot had the following activity:
-     * 2015-09-23: 1 commit in AurelienLourot/AurelienLourot.github.io
-                   1 hot issue in jfrog/build-info */
+  // Hot issues are issues which received more comments than others.
+  //
+  // AurelienLourot had the following activity:
+  // * 2015-09-23: 1 commit in AurelienLourot/AurelienLourot.github.io
+  //               1 hot issue in jfrog/build-info
 
   const result = await m.fetch('AurelienLourot', '2015-09-23', '2015-09-23', ora, console, true);
   t.is(result.size, 2);