Makefile package.json utils.js

.gitignore

@@ -0,0 +1,27 @@
+# Numerous always-ignore extensions
+*.diff
+*.err
+*.orig
+*.log
+*~
+
+# OS or Editor folders
+.DS_Store
+.cache
+Icon?
+
+# Folders to ignore
+.hg
+.svn
+
+# Node.js package manager
+/node_modules
+/npm-debug.log
+
+# Other stuff
+*.pyc
+/tmp
+
+# Project stuff
+/temp-logins.json
+/old-logins.json

Makefile

@@ -0,0 +1,9 @@
+all: get format
+
+get:
+	coffee get-repositories.coffee
+	coffee get-details.coffee
+
+format:
+	coffee format-languages.coffee
+	coffee format-repositories.coffee

package.json

@@ -0,0 +1,22 @@
+{
+  "name": "cs-github-wikiteams",
+  "version": "0.0.0",
+  "description": "cs-github-wikiteams",
+  "main": "convert-markdown.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/wikiteams/cs-github-wikiteams"
+  },
+  "author": "Oskar Jarczyk",
+  "license": "MIT",
+  "gitHead": "e6a55ed9abd436d3c60323e6cc1d2b3ad6784abb",
+  "readmeFilename": "README.md",
+  "dependencies": {
+    "cheerio": "~0.10.5",
+    "superagent": "~0.14.0",
+    "batch": "~0.3.2"
+  }
+}

utils.js

@@ -0,0 +1,93 @@
+var fs = require('fs');
+var Batch = require('batch');
+var request = require('superagent');
+
+var batchGet = exports.batchGet = function(urls, progressback, callback) {
+  var batch = new Batch;
+  batch.concurrency(5);
+  urls.forEach(function(url) {
+    batch.push(function(done) {
+      request
+        .get(url)
+        .set('User-Agent', 'curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8r zlib/1.2.5')
+        .end(function(error, response) {
+          console.log(url);
+          if (error) throw new Error(error);
+          if (response.error) {
+            if (response.status === 404) {
+              done();
+            } else {
+              throw response.error;
+            }
+          }
+          var result;
+          try {
+            result = progressback(response.text);
+          } catch (err) {
+            error = err;
+          }
+          done(error, result);
+        });
+    });
+  });
+
+  batch.end(function(error, all) {
+    if (error) throw new Error(error);
+    callback(all);
+  });
+};
+
+exports.range = function(start, end, step) {
+  start = +start || 0;
+  step = +step || 1;
+
+  if (end == null) {
+    end = start;
+    start = 0;
+  }
+  // use `Array(length)` so V8 will avoid the slower "dictionary" mode
+  // http://youtu.be/XAqIpGU8ZZk#t=17m25s
+  var index = -1,
+      length = Math.max(0, Math.ceil((end - start) / step)),
+      result = Array(length);
+
+  while (++index < length) {
+    result[index] = start;
+    start += step;
+  }
+  return result;
+};
+
+exports.writeStats = function(filename, stats) {
+  fs.writeFileSync(filename, JSON.stringify(stats, null, 2) + '\n');
+  console.log('  Saved to ' + filename);
+};
+
+// For debugging GitHub search.
+var prop = function(name) {
+  return function(item) {return item[name];};
+};
+
+var isNotIn = function(list) {
+  return function(item) {return list.indexOf(item) === -1;};
+};
+
+var diff = function(prev, curr) {
+  return prev.map(prop('login')).filter(isNotIn(curr.map(prop('login'))));
+};
+
+var reverseFind = function(list) {
+  return function(login) {
+    return list.filter(function(item) {
+      return item.login === login;
+    })[0];
+  };
+};
+
+exports.prop = prop;
+exports.isNotIn = isNotIn;
+exports.diff = diff;
+exports.reverseFind = reverseFind;
+
+// diff(prev, curr).map(reverseFind(prev));
+// prev.map(prop('login')).filter(isNotIn(logins))