Fix after GitHub's interface has changed // ghuser-io/ghuser.io#172

Author: lourot

README.md

@@ -70,7 +70,6 @@ Normally in order to retrieve all repositories a user has interacted with, one s
 Instead we noticed that the "Contribution Activity" on the
 [profile pages](https://github.com/AurelienLourot) queries such URLs in the background:
 
-* https://github.com/users/AurelienLourot/created_commits?from=2018-05-17&to=2018-05-17
 * https://github.com/users/AurelienLourot/created_repositories?from=2018-05-17&to=2018-05-17
 * https://github.com/users/AurelienLourot/created_pull_request_reviews?from=2018-05-17&to=2018-05-17
 * https://github.com/users/AurelienLourot/created_issues?from=2018-07-10&to=2018-07-10
@@ -103,6 +102,13 @@ So we're doing the same :)
 >               <a href="/tt-gf/ant-ivy/pull/2" class="content-title no-underline">
 >   ```
 
+> * Same now with `created_commits`:
+>
+>   ```bash
+>   $ curl -s "https://github.com/AurelienLourot?from=2017-08-27" | grep commits?
+>         <a href="/AurelienLourot/mybeir.ut/commits?author=AurelienLourot&amp;since=2017-08-27&amp;until=2017-08-28" class="f6 muted-link ml-1">
+>   ```
+
 ### Why is it so slow?
 
 We hit a [rate limit](https://en.wikipedia.org/wiki/Rate_limiting). And since it's not an official
@@ -129,6 +135,10 @@ doesn't discover commits in forks.
 
 ## Changelog
 
+**2.2.3** (2018-10-20):
+  * [ghuser-io/ghuser.io#172](https://github.com/ghuser-io/ghuser.io/issues/172) Fix after GitHub's
+    interface has changed. The `created_commits` "endpoint" is gone.
+
 **2.2.2** (2018-10-13):
   * [ghuser-io/ghuser.io#172](https://github.com/ghuser-io/ghuser.io/issues/172) Fix after GitHub's
     interface has changed. The `created_pull_requests` "endpoint" is gone.

index.js

@@ -83,39 +83,6 @@
   };
 
   const getContribs = async (user, joinDate, since, until, ora, console, alsoIssues) => {
-    const commitsHtmlToRepos = html => {
-      const repos = new Set();
-
-      const handler = new htmlparser.DefaultHandler((error, dom) => {});
-      const parser = new htmlparser.Parser(handler);
-      parser.parseComplete(html);
-      for (let i = 0; i < handler.dom.length; ++i) {
-        if (handler.dom[i].type === 'tag' && handler.dom[i].name === 'ul') {
-          const ul = handler.dom[i].children;
-          for (let j = 0; j < ul.length; ++j) {
-            if (ul[j].type === 'tag' && ul[j].name === 'li') {
-              const li = ul[j].children;
-              for (let k = 0; k < li.length; ++k) {
-                if (li[k].type === 'tag' && li[k].name === 'div') {
-                  const div = li[k].children;
-                  for (let l = 0; l < div.length; ++l) {
-                    if (div[l].type === 'tag' && div[l].name === 'a') {
-                      const a = div[l].children[0].data;
-                      if (!a.includes(' ')) {
-                        repos.add(a);
-                      }
-                    }
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-
-      return repos;
-    };
-
     const issuesHtmlToRepos = html => {
       const repos = new Set();
 
@@ -149,10 +116,11 @@
       return repos;
     };
 
-    const bigHtmlToRepos = (html, type) => { // type: 'issues' or 'pulls'
+    const bigHtmlToRepos = (html, type) => { // type: 'issues', 'pull' or 'commits'
       const repos = new Set();
 
-      const regex = new RegExp(`<a.*href="/(.*)/(.*)/${type}/`, 'g');
+      const charAfterType = type === 'commits' ? '?' : '/';
+      const regex = new RegExp(`<a.*href="/(.*)/(.*)/${type}${charAfterType}`, 'g');
       let linkToIssue;
       while ((linkToIssue = regex.exec(html))) {
         const owner = linkToIssue[1];
@@ -204,17 +172,11 @@
         currDate = prevDay(currDate);
 
         return (async () => {
-          const userCommits = await fetchRetry(
-            `https://github.com/users/${user}/created_commits?from=${currDateStr}&to=${currDateStr}`
-          );
-          const userCommitsHtml = await userCommits.text();
-          const commitsRepos = commitsHtmlToRepos(userCommitsHtml);
-
-          const userPRsAndHotIssues = await fetchRetry(
+          const bigHtml = await (await fetchRetry(
               `https://github.com/${user}?from=${currDateStr}`,
-          );
-          const userPRsAndHotIssuesHtml = await userPRsAndHotIssues.text();
-          const prsRepos = bigHtmlToRepos(userPRsAndHotIssuesHtml, 'pull');
+          )).text();
+          const commitsRepos = bigHtmlToRepos(bigHtml, 'commits');
+          const prsRepos = bigHtmlToRepos(bigHtml, 'pull');
 
           let issuesRepos = [];
           let hotIssuesRepos = [];
@@ -225,7 +187,7 @@
             const userIssuesHtml = await userIssues.text();
             issuesRepos = issuesHtmlToRepos(userIssuesHtml);
 
-            hotIssuesRepos = bigHtmlToRepos(userPRsAndHotIssuesHtml, 'issues');
+            hotIssuesRepos = bigHtmlToRepos(bigHtml, 'issues');
           }
 
           progressSpinner.stop(); // temporary stop for logging

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ghuser/github-contribs",
-  "version": "2.2.2",
+  "version": "2.2.3",
   "description": "List all GitHub repos a user has contributed to since the beginning of time.",
   "license": "Unlicense",
   "repository": {
@@ -73,6 +73,6 @@
       ".js"
     ],
     "check-coverage": true,
-    "lines": 75
+    "lines": 70
   }
 }

test.js

@@ -31,13 +31,15 @@ test('fetches commits and PRs', async t => {
   const result = await m.fetch('AurelienLourot', '2017-08-26', '2017-08-28', ora, console);
 
   // Because AurelienLourot had no activity on 2017-08-26, GitHub chooses to display older activity
-  // on his profile, namely a PR for `brandon-rhodes/uncommitted` on 2017-07-08. This is a known
-  // limitation and we think it's better to discover too much than not enough.
+  // on his profile, namely a PR for `brandon-rhodes/uncommitted` on 2017-07-08 and some commits to
+  // `AurelienLourot/myberl.in`. This is a known limitation and we think it's better to discover too
+  // much than not enough.
 
-  t.is(result.size, 3);
+  t.is(result.size, 4);
   t.true(result.has('AurelienLourot/mybeir.ut'));
   t.true(result.has('tt-gf/ant-ivy'));
   t.true(result.has('brandon-rhodes/uncommitted'));
+  t.true(result.has('AurelienLourot/myberl.in'));
 });
 
 test('fetches issues', async t => {