Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Restore hash #174

Merged
merged 11 commits into from
Nov 11, 2014
Merged

Restore hash #174

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
449a29b
prepare to re-use hash builds
SergioCrisostomo Nov 6, 2014
f9ebb64
Untested builderHash load and save functionality.
timwienk Nov 7, 2014
a37b5d4
add testing samples
SergioCrisostomo Nov 9, 2014
5812937
make builder save and load hash
SergioCrisostomo Nov 9, 2014
0381191
improvements after review
SergioCrisostomo Nov 11, 2014
f944570
Refactored the builder hasher
arian Nov 11, 2014
be18961
Make core/index not executable.
arian Nov 11, 2014
a7ee207
Move the BuilderHash from middelware to lib.
arian Nov 11, 2014
cecc92c
Rename middleware/buildHash - fix when the hash was not in the database
arian Nov 11, 2014
18c03f5
Merge pull request #1 from arian/restore-hash
SergioCrisostomo Nov 11, 2014
1ccdd1e
remove unused placeholder span
SergioCrisostomo Nov 11, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 24 additions & 3 deletions builder/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,18 @@ var packager = require('mootools-packager');
var getFiles = require('../lib/getFiles');
var projectPath = require('../lib/projectPath');
var bodyParser = require('body-parser');
var pkgProjects = require('../package.json')._projects;

var hashPath = (function(){
var pathObject = {};
var projects = Object.keys(pkgProjects).forEach(function(project){
pathObject[project] = pkgProjects[project].hashStorage;
})
return pathObject;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can do nicer with reduce instead.

return Object.keys(pkgProjects).reduce(function(pathObject, project){
    pathObject[project] = pkgProjects[project].hashStorage;
}, {});

And don't forget the ;.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And when you use the reduce version, you don't need the extra iife/function/closure.

})();
var builderHash = require('../middleware/builderHash')(hashPath);
var copyright = '/* MooTools: the javascript framework. license: MIT-style license. copyright: Copyright (c) 2006-' + new Date().getFullYear() + ' [Valerio Proietti](http://mad4milk.net/).*/ ';
var allVersions = require('../package.json');


function uglify(source){
var uglifyed = UglifyJS.minify(source, {
Expand All @@ -24,7 +33,7 @@ function processPost(req, res){
var postData = req.body;
var minified = postData.minified;
var project = postData.project;
var version = allVersions._projects[project.toLowerCase()].versions[0];
var version = pkgProjects[project.toLowerCase()].versions[0];

var corePath = projectPath('core', version);
var morePath = projectPath('more', version);
Expand Down Expand Up @@ -62,10 +71,22 @@ function processPost(req, res){
res.end();
}
}

function hash(req, res, next){
if (!req.query.packages) return res.end();
builderHash.save(req.query.project, req.query.packages, function(data) {
res.locals.hash = data.hash;
next();
});
}
module.exports = function(app){
app.use(bodyParser.urlencoded({
extended: true
}));
app.post('/builder', processPost);
app.get('/builder', hash, function(req, res){
res.send({
hash: res.locals.hash,
project: req.query.project
});
});
};
25 changes: 22 additions & 3 deletions core/index.js
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ var guides = require('../middleware/guides')('core', {
});

var project = 'core';
var versions = require('../package.json')._projects[project].versions;
var pkgProject = require('../package.json')._projects[project];
var versions = pkgProject.versions;
var links = versions.slice(1).map(function(version){
return {
version: version,
Expand All @@ -22,6 +23,19 @@ var links = versions.slice(1).map(function(version){
};
});

var builderHash = require('../middleware/builderHash')({
core: pkgProject.hashStorage
});

function hash(req, res, next){
var hash = req.params.hash;
if (!hash) return next();
builderHash.load(project, hash, function(data) {
res.locals.hash = data.packages;
next();
});
}

module.exports = function(app){

var core = function(req, res, next){
Expand All @@ -30,7 +44,6 @@ module.exports = function(app){
};

app.get('/core', core, function(req, res){

res.render('core/index', {
page: "/core",
title: "MooTools Core",
Expand All @@ -41,13 +54,14 @@ module.exports = function(app){
});
});

app.get('/core/builder', function(req, res){
app.get('/core/builder/:hash?', hash, function(req, res){
res.render('builder/index', {
title: 'MooTools Core Builder',
navigation: 'core',
page: 'builder',
project: 'Core',
site: 'core',
hashDependencies: res.locals.hash || [],
version: versions[0],
versions: links,
dependencies: require('../builder/dependencies.js')(project, versions[0])
Expand All @@ -61,4 +75,9 @@ module.exports = function(app){
app.get('/core/guides', core, guides.index);
app.get('/core/guides/:guide', core, guides.article);

// hash build redirect
app.get('/core/:hash', function(req, res){
res.redirect('/core/builder/' + req.params.hash);
});

};
61 changes: 61 additions & 0 deletions middleware/builderHash.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
'use strict';

var sqlite3 = require('sqlite3'),
md5 = require('md5');

var databasePaths = {},
databases = {};

function getDatabase(project){
if (!databases[project]) {
var path = databasePaths[project];
if (path){
databases[project] = new sqlite3.Database(path, sqlite3.OPEN_READWRITE, function(error){
if (error) throw error;
});
} else {
throw Error('No database found for "' + project + '".');
}
}
return databases[project];
}

function loadHash(project, hash, callback){
getDatabase(project).get('SELECT * FROM hashes WHERE md5 = ?', {1: hash}, function(error, row){
var data = null;
if (row) data = {hash: row.md5, packages: row.packages.split(';')};
if (callback) callback(data);
});
}

function saveHash(project, packages, callback){
if (packages && packages.length){
var db = getDatabase(project),
packageString = typeof packages == 'string' ? packages : packages.join(';'),
hash = md5.digest_s(packageString);
db.get('SELECT COUNT(*) AS count FROM hashes WHERE md5 = ?', {1: hash}, function(error, row){
if (error) throw error;
if (row.count){
if (callback) callback({hash: hash, packages: packages});
} else {
var values = {1: hash, 2: packageString, 3: Math.round(Date.now() / 1000)};
db.run('INSERT INTO hashes (md5, packages, date) VALUES (?, ?, ?)', values, function(error){
if (error) throw error;
if (callback) callback({hash: hash, packages: packages});
});
}
});
} else {
if (callback) callback(null);
}
}

module.exports = function(paths){
for (var key in paths){
databasePaths[key] = paths[key];
}
return {
load: loadHash,
save: saveHash
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wouldn't it be better if you create the functions where the paths thing is enclosed? Now it's in some global variable that gets mutated along the way... Probably the cleanest way is to create a JS Class / constructor thingy, such that this exported function is:

module.exports = function(paths){
  var db = new BuilderDatabase(paths);
  return {load: db.load.bind(db), save: db.save.bind(db)};
};

As prime is still a dependency, you could use that, or just go the vanilla JS way, as you don't need anything special like inheritance anyway.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Quite possibly. I figured this way the things could be set up in a configuration part of the app, while in a next part you only need to require the builderHash. I haven't looked much into the application structure and where configuration is accessible.

}
27 changes: 25 additions & 2 deletions more/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,21 @@ var guides = require('../middleware/guides')('more', {
});

var project = 'more';
var lastVersion = require('../package.json')._projects[project].versions[0];
var pkgProject = require('../package.json')._projects[project];
var lastVersion = pkgProject.versions[0];

var builderHash = require('../middleware/builderHash')({
more: pkgProject.hashStorage
});

function hash(req, res, next){
var hash = req.params.hash;
if (!hash) return next();
builderHash.load(project, hash, function(data) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With the thing I wrote above, you could pass the project into the var builderHash = require(...){{ ... }) code.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And as the builderHash is middelware, and the project is configurable there, can't you export this hash function as well, so you don't have to do it here?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The idea was that you'd only need to call load and save, and don't need to "manually" hash anything anywhere. Just call load to get a list of packages if the hash exists, and just call save to get a hash returned if it's saved (or already exists).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the hash is required anyway, yes, export it from the thing I threw together.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, only thing i noticed was code duplication of this 'hash' function in core and more...

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

res.locals.hash = data.packages;
next();
});
}

module.exports = function(app){

Expand All @@ -28,13 +42,15 @@ module.exports = function(app){
});
});

app.get('/more/builder', function(req, res){
app.get('/more/builder/:hash?', hash, function(req, res){
var hash = req.params.hash;
res.render('builder/index', {
title: 'MooTools More Builder',
navigation: 'more',
page: 'builder',
project: 'More',
site: 'more',
hashDependencies: res.locals.hash || [],
version: lastVersion,
dependencies: require('../builder/dependencies.js')(project, lastVersion)
});
Expand All @@ -47,4 +63,11 @@ module.exports = function(app){
app.get('/more/guides', more, guides.index);
app.get('/more/guides/:guide', more, guides.article);

// hash build redirect
var regex = /more\/([a-z]+[0-9]+[a-z0-9]*|[0-9]+[a-z]+[a-z0-9]*)$/;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as for core, we can probably just redirect /more/:hash, rather than a regex.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also makes it possible to have "custom" hashes, in case we'd want to do that for some reason. Old website functionality did have that as well, I don't know if any exist.

app.get(regex, function(req, res){
var hash = req.url.match(regex)[1];
res.redirect('/more/builder#' + hash);
});

};
4 changes: 4 additions & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
"highlight.js": "7.3",
"jade": "0.31",
"js-yaml": "^3.2.2",
"md5": "^1.0.0",
"mootools-packager": "~0.0.2",
"morgan": "~1.3.2",
"needle": "~0.6.3",
Expand All @@ -43,6 +44,7 @@
"semver": "2.0",
"serve-favicon": "^2.1.5",
"slugify": "0.1",
"sqlite3": "^3.0.2",
"stylus": "0.32",
"sync-prompt": "^0.4.1",
"twitter": "^0.2.13",
Expand All @@ -60,6 +62,7 @@
"core": {
"repository": "https://github.com/mootools/mootools-core",
"docsIntro": "Docs/Intro.md",
"hashStorage": "/srv/mootools/core/database/packager.sqlite",
"versions": [
"1.5.1",
"1.4.5",
Expand All @@ -70,6 +73,7 @@
"more": {
"repository": "https://github.com/mootools/mootools-more",
"docsIntro": "Docs/More/More.md",
"hashStorage": "/srv/mootools/more/database/packager.sqlite",
"versions": [
"1.5.1",
"1.5.0",
Expand Down
Binary file added tests/database/core.db
View file
Open in desktop
Binary file not shown.
Binary file added tests/database/more.db
View file
Open in desktop
Binary file not shown.
33 changes: 33 additions & 0 deletions tests/database/newDatabase.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
var fs = require('fs');
var file = __dirname + '/../tests/database/test.db';
var exists = fs.existsSync(file);
var sqlite3 = require('sqlite3').verbose();
var sampleData = require('../tests/database/sampleData.json');

if (!exists){
console.log('Creating DB file.');
fs.openSync(file, 'w');
}

var db = new sqlite3.Database(file);
db.serialize(function(){

if (!exists) db.run("CREATE TABLE hashes (md5, packages, date)");

if (!exists) sampleData.forEach(function (sample) {
var values = sample;
db.run('INSERT INTO hashes (md5, packages, date) VALUES (?, ?, ?)', values, function(error){
if (error) throw error;
});

});

db.each("SELECT md5, packages, date FROM hashes", function(err, row){
console.log(row.md5);
console.log(row.packages);
console.log(row.date);
console.log('.....................');
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indentation?

});
});

db.close();
27 changes: 27 additions & 0 deletions tests/database/sampleData.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
[
{
"1": "e8f3003df2d0919c1091b11854a53e9b",
"2": "Core/Core;Core/String;Core/Event;Core/Browser;Core/Class;Core/Element.Style;Core/Element.Event;Core/Element.Delegation;Core/Element.Dimensions;Core/Fx;Core/Fx.CSS;Core/Fx.Tween;Core/Fx.Morph;Core/Fx.Transitions;Core/Request.HTML;Core/Request.JSON;Core/Cookie;Core/DOMReady",
"3": "1415017941"
},
{
"1": "2aa60ee2887f132a617b2a49754ef7fd",
"2": "Core/Core;Core/Array;Core/String;Core/Number;Core/Function;Core/Object;Core/Event;Core/Browser;Core/Request;Core/Request.HTML;Core/Request.JSON;Core/Cookie;Core/JSON",
"3": "1415017221"
},
{
"1": "d1d2b98d3cae0d77aa29422996e67802",
"2": "Core/Core;Core/Event;Core/Element;Core/Element.Style;Core/Element.Delegation;Core/Element.Dimensions;Core/Fx;Core/Fx.Tween;Core/Fx.Morph;Core/JSON;Core/DOMReady",
"3": "1414767773"
},
{
"1": "73a246bd9f16ce89e3ab685ff1e08599",
"2": "Core/Array;Core/Function;Core/Event;Core/Class;Core/Element;Core/Element.Event;Core/Fx",
"3": "1414689276"
},
{
"1": "22af847e17dad82b2f32682e0f617e3e",
"2": "Core/Core;Core/Array;Core/String;Core/Number;Core/Function;Core/Object;Core/Event;Core/Browser;Core/Class;Core/Class.Extras;Core/Slick.Parser;Core/Slick.Finder;Core/Element;Core/Element.Style;Core/Element.Event;Core/Element.Dimensions;Core/Fx;Core/Fx.CSS;Core/Fx.Tween;Core/Fx.Morph;Core/Fx.Transitions;Core/Request;Core/Request.HTML;Core/Request.JSON;Core/Cookie;Core/JSON;Core/DOMReady",
"3": "1414636765"
}
]
17 changes: 12 additions & 5 deletions views/builder/index.jade
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ block main

div.header.clearfix
h1 Download complete source
form(method="post", action="../builder")
form(method="post", action="/builder")
input(hidden, name="project", value="#{project}")
.save

Expand Down Expand Up @@ -40,27 +40,34 @@ block main

h3 Choose which modules you want to use

form(method="post", action="../builder")
form#builderOptions(method="post", action="/builder")
input(hidden, name="project", value="#{project}")
table#builderOptions
table
tr
td
td File
td Provides
td Description

each yaml, module in dependencies
- var file = project + "/" + module;
- var hashRequested = hashDependencies.indexOf(file) != -1;
tr
td
input(type="checkbox", value="#{project}/#{module}", name="modules[]", data-provides="#{yaml.prov}", data-requires="#{yaml.req}")
input(type="checkbox", value=file, name="modules[]", data-provides="#{yaml.prov}", data-requires="#{yaml.req}", checked=hashRequested, class=hashRequested ? "activeChoice" : "")
td #{module}
td #{yaml.prov}
td #{yaml.desc}

h2.step Options:


.save
span(id="hashLink")

label
input(type="checkbox", id="hashOption")
| Save this build and give me a url to it


label
input(type="checkbox", name="minified", value="1")
Expand Down
5 changes: 5 additions & 0 deletions views/css/global.styl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,11 @@ input[type="checkbox"]
height: 1em;
position: relative;
width: 1em;
#builderOptions table input[type="checkbox"]
&:checked
background #f4f1ed
&.activeChoice:checked
background #7d8aa5

input[type="submit"]
background: #F4F1ED;
Expand Down
Loading