From 59e595ba2346a43e4c29f63647691e1fe0c78586 Mon Sep 17 00:00:00 2001 From: bettori Date: Sun, 27 Mar 2016 16:33:51 -0400 Subject: [PATCH 1/3] Add functionality to copy feed and append to it. Change module export name. Squash commits for appending feed --- index.js | 6 +++ lib/feedOperations.js | 48 +++++++++++++++++++ test/feedOperations.js | 103 +++++++++++++++++++++++++++++++++++++++++ test/import.js | 2 +- 4 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 lib/feedOperations.js create mode 100644 test/feedOperations.js diff --git a/index.js b/index.js index 0132dec..44aefde 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,7 @@ var level = require('level') var hypercore = require('hypercore') var createImportPipeline = require('./lib/import.js') +var feedOperations = require('./lib/feedOperations.js') module.exports = Jawn @@ -14,3 +15,8 @@ function Jawn (opts) { Jawn.prototype.createImportPipeline = function (opts) { return createImportPipeline(this, opts) } + +Jawn.prototype.createAppendableFeed = function (feedId, data) { + var feed = feedOperations.appendTo(this.core, feedId) + return feed +} diff --git a/lib/feedOperations.js b/lib/feedOperations.js new file mode 100644 index 0000000..c9765f0 --- /dev/null +++ b/lib/feedOperations.js @@ -0,0 +1,48 @@ + +module.exports.appendTo = appendableFeed + +function appendableFeed (core, feedId) { + var refFeed = core.get(feedId) + var newFeed = core.add() + var blockNumber = 0 + + var copyOriginal = function (resolve, reject) { + for (var i = 0; i < refFeed.blocks; i++) { + refFeed.get(i, function (err, block) { + if (err) { + console.log(err) + reject() + } + + newFeed.append(block, function () { + blockNumber++ + console.log('Copying block ' + blockNumber + '/' + newFeed.blocks + ' from original feed') + + if (blockNumber === refFeed.blocks) { + resolve() + } + }) + }) + } + } + + newFeed.append_p = function (data) { + return new Promise(function (resolve, reject) { + newFeed.append(data, resolve) + }) + } + + newFeed.finalize_p = function () { + return new Promise(function (resolve, reject) { + newFeed.finalize(resolve) + }) + } + + newFeed.initialize = function () { + return new Promise(function (resolve, reject) { + copyOriginal(resolve, reject) + }) + } + + return newFeed +} diff --git a/test/feedOperations.js b/test/feedOperations.js new file mode 100644 index 0000000..55125ae --- /dev/null +++ b/test/feedOperations.js @@ -0,0 +1,103 @@ +var test = require('tape') +var Jawn = require('../') +var memdb = require('memdb') +var feedOps = require('../lib/feedOperations.js') + +test('appendable feed', function (t) { + var jawn = freshJawn() + var feed = jawn.core.add() + + feed.pappend = function (data) { + return new Promise(function (resolve, reject) { + feed.append(data, resolve) + }) + } + + feed.pfinalize = function () { + return new Promise(function (resolve, reject) { + feed.finalize(resolve) + }) + } + + feed.pappend('hello').then(function () { + console.log('Appended') + return feed + }) + .then(function (feed) { + return feed.pfinalize() + }) + .then(function () { + console.log('Finalized with id ' + feed.id.toString('hex')) + var appfeed = feedOps.appendTo(jawn.core, feed.id) + appfeed.initialize().then(function () { + return appfeed.finalize_p() + }) + .then(function () { + t.same(appfeed.blocks, feed.blocks, 'testing') + t.end() + }) + }) +}) + +test('append to feed', function (t) { + var jawn = freshJawn() + var feed = jawn.core.add() + var expected = ['hello', 'there', 'goodbye'] + + storeDataInFeed(feed, ['hello', 'there']) + .then(function () { + console.log('Finalized with id ' + feed.id.toString('hex')) + + var appfeed = feedOps.appendTo(jawn.core, feed.id) + + appfeed.initialize().then(function () { + return appfeed.append_p('goodbye') + }) + .then(function () { + return appfeed.finalize_p() + }) + .then(function () { + t.same(appfeed.blocks, expected.length, 'Correct number of blocks') + for (var i = 0; i < appfeed.blocks; i++) { + appfeed.get(i, function (err, block) { + if (err) { + console.log(err) + } + t.same(block.toString(), expected.shift(), 'Feed block match') + if (expected.length === 0) { + t.end() + } + }) + } + }) + }) +}) + +function storeDataInFeed (feed, data) { + feed.pappend = function (data) { + return new Promise(function (resolve, reject) { + feed.append(data, resolve) + }) + } + + feed.pfinalize = function () { + return new Promise(function (resolve, reject) { + feed.finalize(resolve) + }) + } + + return new Promise(function (resolve, reject) { + feed.pappend(data).then(function () { + console.log('Appended') + return feed + }) + .then(function (feed) { + return feed.pfinalize() + }) + .then(resolve) + }) +} + +function freshJawn () { + return new Jawn({db: memdb()}) +} diff --git a/test/import.js b/test/import.js index 09a3c1b..4b16485 100644 --- a/test/import.js +++ b/test/import.js @@ -30,7 +30,7 @@ test('import json to jawn', function (t) { test('import csv to jawn', function (t) { var jawn = freshJawn() importFromFile(jawn, 'sample.csv', {'format': 'csv'}) - var importStream = importFromFile(jawn, 'sample.csv', {'format': 'csv'}, verify) + var importStream = importFromFile(jawn, 'sample.csv', {'format': 'csv'}) var expected = [ '{"Type of Experience":"Writing software in any programming language","Little/No Experience":"1","Some Experience":"5","Very Familiar":"4"}', '{"Type of Experience":"Frontend Web Development","Little/No Experience":"4","Some Experience":"3","Very Familiar":"3"}', From ac75f16ab7cbb1d6938f873577f68c4e8645fd91 Mon Sep 17 00:00:00 2001 From: bettori Date: Wed, 30 Mar 2016 20:36:28 -0400 Subject: [PATCH 2/3] Make tests more readable --- .travis.yml | 1 - lib/feedOperations.js | 2 +- test/feedOperations.js | 30 +++++++++--------------------- 3 files changed, 10 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6160d82..f3042df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,5 +4,4 @@ node_js: - "4.0" - "0.12" - "0.11" - - "0.10" - "iojs" \ No newline at end of file diff --git a/lib/feedOperations.js b/lib/feedOperations.js index c9765f0..85bac5d 100644 --- a/lib/feedOperations.js +++ b/lib/feedOperations.js @@ -16,7 +16,7 @@ function appendableFeed (core, feedId) { newFeed.append(block, function () { blockNumber++ - console.log('Copying block ' + blockNumber + '/' + newFeed.blocks + ' from original feed') + console.log('Copying block ' + blockNumber + '/' + refFeed.blocks + ' from original feed') if (blockNumber === refFeed.blocks) { resolve() diff --git a/test/feedOperations.js b/test/feedOperations.js index 55125ae..447fcca 100644 --- a/test/feedOperations.js +++ b/test/feedOperations.js @@ -3,32 +3,17 @@ var Jawn = require('../') var memdb = require('memdb') var feedOps = require('../lib/feedOperations.js') -test('appendable feed', function (t) { +test('Create appendable feed and copy data from reference feed', function (t) { var jawn = freshJawn() var feed = jawn.core.add() - feed.pappend = function (data) { - return new Promise(function (resolve, reject) { - feed.append(data, resolve) - }) - } - - feed.pfinalize = function () { - return new Promise(function (resolve, reject) { - feed.finalize(resolve) - }) - } - - feed.pappend('hello').then(function () { - console.log('Appended') - return feed - }) - .then(function (feed) { - return feed.pfinalize() - }) + storeDataInFeed(feed, ['hello']) .then(function () { + // Verify feed was finalized and has an id console.log('Finalized with id ' + feed.id.toString('hex')) + // Create appendable feed var appfeed = feedOps.appendTo(jawn.core, feed.id) + // Copy data from reference feed, finalize it, then verify the feed has the same number of blocks as the reference appfeed.initialize().then(function () { return appfeed.finalize_p() }) @@ -39,7 +24,7 @@ test('appendable feed', function (t) { }) }) -test('append to feed', function (t) { +test('Create appendable feeed, copy data from reference feed and append to feed', function (t) { var jawn = freshJawn() var feed = jawn.core.add() var expected = ['hello', 'there', 'goodbye'] @@ -50,6 +35,7 @@ test('append to feed', function (t) { var appfeed = feedOps.appendTo(jawn.core, feed.id) + // Copy data from original feed with initialize(), then append an additional block and finalize appfeed.initialize().then(function () { return appfeed.append_p('goodbye') }) @@ -57,7 +43,9 @@ test('append to feed', function (t) { return appfeed.finalize_p() }) .then(function () { + // Verify the feed has the correct number of blocks, and the blocks match what was expected t.same(appfeed.blocks, expected.length, 'Correct number of blocks') + for (var i = 0; i < appfeed.blocks; i++) { appfeed.get(i, function (err, block) { if (err) { From 3f712ec8b191c43d03036464bc622f2f857426de Mon Sep 17 00:00:00 2001 From: bettori Date: Sun, 3 Apr 2016 16:26:37 -0400 Subject: [PATCH 3/3] Add methods to replicate and append to feeds without using promises. Add tape tests for these methods. --- index.js | 4 +-- lib/feedOperations.js | 66 +++++++++++++++++++++++++++++++----------- test/feedOperations.js | 62 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 110 insertions(+), 22 deletions(-) diff --git a/index.js b/index.js index 44aefde..80225de 100644 --- a/index.js +++ b/index.js @@ -1,7 +1,7 @@ var level = require('level') var hypercore = require('hypercore') var createImportPipeline = require('./lib/import.js') -var feedOperations = require('./lib/feedOperations.js') +var FeedOperations = require('./lib/feedOperations.js') module.exports = Jawn @@ -17,6 +17,6 @@ Jawn.prototype.createImportPipeline = function (opts) { } Jawn.prototype.createAppendableFeed = function (feedId, data) { - var feed = feedOperations.appendTo(this.core, feedId) + var feed = new FeedOperations(this.core).appendableFeed(feedId) return feed } diff --git a/lib/feedOperations.js b/lib/feedOperations.js index 85bac5d..15f2631 100644 --- a/lib/feedOperations.js +++ b/lib/feedOperations.js @@ -1,30 +1,38 @@ +module.exports = FeedOperations -module.exports.appendTo = appendableFeed +function FeedOperations (core) { + this.core = core + this.replicate = copyOriginal -function appendableFeed (core, feedId) { - var refFeed = core.get(feedId) - var newFeed = core.add() - var blockNumber = 0 + function copyOriginal (refFeed, newFeed, resolve, reject) { + loop(null) - var copyOriginal = function (resolve, reject) { - for (var i = 0; i < refFeed.blocks; i++) { - refFeed.get(i, function (err, block) { + function loop (err) { + if (err) { + console.log(err) + if (reject) { + return reject() + } + } + + refFeed.get(newFeed.blocks, function (err, block) { if (err) { console.log(err) - reject() } - newFeed.append(block, function () { - blockNumber++ - console.log('Copying block ' + blockNumber + '/' + refFeed.blocks + ' from original feed') + if (!block) { + return resolve() + } - if (blockNumber === refFeed.blocks) { - resolve() - } - }) + newFeed.append(block, loop) }) } } +} + +FeedOperations.prototype.appendableFeed = function (feedId) { + var refFeed = this.core.get(feedId) + var newFeed = this.core.add() newFeed.append_p = function (data) { return new Promise(function (resolve, reject) { @@ -38,11 +46,35 @@ function appendableFeed (core, feedId) { }) } + var copyOriginal = this.replicate + newFeed.initialize = function () { return new Promise(function (resolve, reject) { - copyOriginal(resolve, reject) + copyOriginal(refFeed, newFeed, resolve, reject) }) } return newFeed } + +FeedOperations.prototype.replicateFeed = function (feedId, callback) { + var refFeed = this.core.get(feedId) + var newFeed = this.core.add() + + this.replicate(refFeed, newFeed, callback) + return newFeed +} + +// Copies data from reference feed, appends data to new feed and finalizes it +FeedOperations.prototype.append = function (feedId, data, callback) { + var refFeed = this.core.get(feedId) + var newFeed = this.core.add() + + this.replicate(refFeed, newFeed, function () { + newFeed.append(data, function () { + newFeed.finalize(callback) + }) + }) + + return newFeed +} diff --git a/test/feedOperations.js b/test/feedOperations.js index 447fcca..7b94783 100644 --- a/test/feedOperations.js +++ b/test/feedOperations.js @@ -1,7 +1,7 @@ var test = require('tape') var Jawn = require('../') var memdb = require('memdb') -var feedOps = require('../lib/feedOperations.js') +var FeedOps = require('../lib/feedOperations.js') test('Create appendable feed and copy data from reference feed', function (t) { var jawn = freshJawn() @@ -12,7 +12,8 @@ test('Create appendable feed and copy data from reference feed', function (t) { // Verify feed was finalized and has an id console.log('Finalized with id ' + feed.id.toString('hex')) // Create appendable feed - var appfeed = feedOps.appendTo(jawn.core, feed.id) + var feedOperations = new FeedOps(jawn.core) + var appfeed = feedOperations.appendableFeed(feed.id) // Copy data from reference feed, finalize it, then verify the feed has the same number of blocks as the reference appfeed.initialize().then(function () { return appfeed.finalize_p() @@ -33,7 +34,7 @@ test('Create appendable feeed, copy data from reference feed and append to feed' .then(function () { console.log('Finalized with id ' + feed.id.toString('hex')) - var appfeed = feedOps.appendTo(jawn.core, feed.id) + var appfeed = new FeedOps(jawn.core).appendableFeed(feed.id) // Copy data from original feed with initialize(), then append an additional block and finalize appfeed.initialize().then(function () { @@ -61,6 +62,61 @@ test('Create appendable feeed, copy data from reference feed and append to feed' }) }) +test('Create appendable feed without promises, copy data from reference feed', function (t) { + var jawn = freshJawn() + var feed = jawn.core.add() + + storeDataInFeed(feed, ['hello', 'there']) + .then(function () { + var appFeed = new FeedOps(jawn.core).replicateFeed(feed.id, finalizeCallback) + function finalizeCallback () { + appFeed.finalize(function () { + t.same(appFeed.blocks, feed.blocks, 'Correct number of blocks') + t.end() + }) + } + }) +}) + +test('Append data to feed without promises', function (t) { + var jawn = freshJawn() + var feed = jawn.core.add() + var expected = ['hello', 'there', 'goodbye'] + + storeDataInFeed(feed, ['hello', 'there']) + .then(function () { + var appFeed = new FeedOps(jawn.core).append(feed.id, ['goodbye'], testCallback) + function testCallback () { + t.same(appFeed.blocks, expected.length, 'Correct number of blocks in appended feed') + testFeedContents(appFeed, expected, t) + } + }) +}) + +function testFeedContents (feed, expected, t) { + var blockNumber = 0 + feed.get(0, loop) + + function loop (err, block) { + if (err) { + console.log(err) + } + + blockNumber += 1 + + if (block) { + t.same(block.toString(), expected.shift(), 'Feed block matches expected value') + } + + if (expected.length === 0) { + return t.end() + } + + feed.get(blockNumber, loop) + } +} + +// Create a feed, store data in it and finalize it. function storeDataInFeed (feed, data) { feed.pappend = function (data) { return new Promise(function (resolve, reject) {