Split lib into two objects: Waterdata for creating waterdata and Meta…

…data for handling the metadata.
dogodigi · Apr 5, 2013 · ebb4752 · ebb4752
1 parent 1052291
commit ebb4752
Show file tree

Hide file tree

Showing 5 changed files with 83 additions and 95 deletions.
diff --git a/cli.coffee b/cli.coffee
@@ -1,7 +1,6 @@
 flatiron = require 'flatiron'
-processWaterdata = require('./lib/lib').processWaterdata
-dbToJson = require('./lib/lib').dbToJson
-XML = require('./lib/lib').XML
+Waterdata = require('./lib/lib').Waterdata
+Metadata = require('./lib/lib').Metadata
 Location = require('./lib/models').Location
 
 app = flatiron.app
@@ -16,16 +15,16 @@ app.config.defaults({
 Location.connect(app.config.get('MONGOHQ_URL'))
 
 read = ->
-  dbToJson (error, res) -> 
+  Waterdata.dbToJson (error, res) -> 
     console.log res
     Location.db.close()
 
 process = ->
-  processWaterdata ->
+  Waterdata.processWaterdata ->
     Location.db.close()
 
 metadata = ->
-  XML.process ->
+  Metadata.process ->
     Location.db.close()
 
 app.cmd 'process', process

diff --git a/lib/lib.coffee b/lib/lib.coffee
@@ -5,75 +5,76 @@ zip = require 'node-zip'
 xml2js = require 'xml2js'
 Location = require('./models').Location
 
-# RWS open data url
-url = 'http://www.rws.nl/rws/opendata/meetdata/meetdata.zip'
+Waterdata = 
+  # RWS open data url
+  url: 'http://www.rws.nl/rws/opendata/meetdata/meetdata.zip'
 
-# Download a zipfile
-getZip = (url, callback) ->
-  http.get url, (res) ->
-    zipdata = new Buffer(0)
-    res.on('data', (data) ->
-      zipdata = Buffer.concat([zipdata, data])
-    ).on('end', ->
-      callback zipdata.toString('binary')
-    )
+  # Download a zipfile
+  getZip: (url, callback) ->
+    http.get url, (res) ->
+      zipdata = new Buffer(0)
+      res.on('data', (data) ->
+        zipdata = Buffer.concat([zipdata, data])
+      ).on('end', ->
+        callback zipdata.toString('binary')
+      )
 
-# Handle zip file. Extracts the two files we need.
-handleZip = (zipStream) ->
-  metadataFileName = 'update.adm'
-  dataFileName = 'update.dat'
-  files = zip zipStream
-  metadataFile = files.files[metadataFileName]
-  dataFile = files.files[dataFileName]
-  return [metadataFile.data, dataFile.data]
+  # Handle zip file. Extracts the two files we need.
+  handleZip: (zipStream) ->
+    metadataFileName = 'update.adm'
+    dataFileName = 'update.dat'
+    files = zip zipStream
+    metadataFile = files.files[metadataFileName]
+    dataFile = files.files[dataFileName]
+    return [metadataFile.data, dataFile.data]
 
-# Convert a csv file to an array of arrays
-csvToArray = (csvString, callback) ->
-  csv().from.string(csvString)
-    .to.array (data, count) ->
-      callback null, data
+  # Convert a csv file to an array of arrays
+  csvToArray: (csvString, callback) ->
+    csv().from.string(csvString)
+      .to.array (data, count) ->
+        callback null, data
 
-# Process the data in the arrays
-readData = (arrays) ->
-  res = []
-  metadataArray = arrays[0]
-  dataArray = arrays[1]
-  parameterType = 3
-  name = 1
-  lastUpdate = 8
-  lastValue = 5
-  for data, index in dataArray
-    metadata = metadataArray[index]
-    if metadata[parameterType].trim() == 'H10'
-      #callback(metadata[name], parseInt data[lastValue])
-      res[res.length] = {lokatie: metadata[name].trim(), waarde:parseInt(data[lastValue]), laatstBijgewerkt: metadata[lastUpdate].trim()}
-  return res
+  # Process the data in the arrays
+  readData: (arrays) ->
+    res = []
+    metadataArray = arrays[0]
+    dataArray = arrays[1]
+    parameterType = 3
+    name = 1
+    lastUpdate = 8
+    lastValue = 5
+    for data, index in dataArray
+      metadata = metadataArray[index]
+      if metadata[parameterType].trim() == 'H10'
+        #callback(metadata[name], parseInt data[lastValue])
+        res[res.length] = {lokatie: metadata[name].trim(), waarde:parseInt(data[lastValue]), laatstBijgewerkt: metadata[lastUpdate].trim()}
+    return res
 
-# Write the new values to the database for storage
-writeToDb = (object, callback) ->
-  Location.update {lokatie:object.lokatie}, object, {upsert: true}, callback
+  # Write the new values to the database for storage
+  writeToDb: (object, callback) ->
+    Location.update {lokatie:object.lokatie}, object, {upsert: true}, callback
 
-# Turn the database into json. Dynamic for now.
-dbToJson = (callback) ->
-  Location.find {}, '-_id', (error, res) ->
-    callback(error, res)
+  # Turn the database into json. Dynamic for now.
+  dbToJson: (callback) ->
+    Location.find {}, '-_id', (error, res) ->
+      callback(error, res)
 
-processWaterdata = (callback) ->
-  # get the zipfile
-  console.log 'Starting'
-  getZip url, (zipstream) ->
-    console.log 'File received'
-    # get the two files we need
-    files = handleZip zipstream
-    # get the arrays
-    console.log 'Arrays created'
-    async.map files, csvToArray, (error, arrays) ->
-      # read the relevant files
-      values = readData arrays
-      # store the data in the database
-      async.each values, writeToDb, callback
+  processWaterdata: (callback) ->
+    # get the zipfile
+    console.log 'Starting'
+    @getZip @url, (zipstream) =>
+      console.log 'File received'
+      # get the two files we need
+      files = @handleZip zipstream
+      # get the arrays
+      console.log 'Arrays created'
+      async.map files, @csvToArray, (error, arrays) =>
+        # read the relevant files
+        values = @readData arrays
+        # store the data in the database
+        async.each values, @writeToDb, callback
 
-XML =
+Metadata =
   getXml: (url, callback) ->
     http.get url, (res) ->
       xmldata = new Buffer(0)
@@ -103,12 +104,5 @@ XML =
 
   url: 'http://rws.nl/system/externen/meetnet-repository.aspx'
 
-exports.getZip = getZip
-exports.handleZip = handleZip
-exports.csvToArray = csvToArray
-exports.url = url
-exports.processWaterdata = processWaterdata
-exports.dbToJson = dbToJson
-
-exports.XML = XML
-
+exports.Waterdata = Waterdata
+exports.Metadata = Metadata
diff --git a/test/xmlTest.coffee b/test/xmlTest.coffee
@@ -1,11 +1,11 @@
-XML = require('../lib/lib').XML
+Metadata = require('../lib/lib').Metadata
 Location = require('../lib/models').Location
 assert = require 'assert'
 
 describe 'get xml file from rws', ->
 
   it 'should get the xml file', (done) ->
     this.timeout 15000
-    XML.getXml XML.url, ->
+    Metadata.getXml Metadata.url, ->
       done()
 
diff --git a/test/zipTest.coffee b/test/zipTest.coffee
@@ -1,10 +1,4 @@
-getZip = require('../lib/lib').getZip
-rwsUrl = require('../lib/lib').url
-handleZip = require('../lib/lib').handleZip
-processWaterdata = require('../lib/lib').processWaterdata
-
-csvToArray = require('../lib/lib').csvToArray
-
+Waterdata = require('../lib/lib').Waterdata
 Location = require('../lib/models').Location
 
 fs = require 'fs'
@@ -15,12 +9,12 @@ describe 'get zip file from rws', ->
 
   it 'should get the zip file', (done) ->
     # this.timeout(10000)
-    getZip rwsUrl, ->
+    Waterdata.getZip Waterdata.url, ->
       done()
 
   it 'should get two files out of the zipfile', (done) ->
-    getZip rwsUrl, (zipstream) ->
-      files = handleZip zipstream
+    Waterdata.getZip Waterdata.url, (zipstream) ->
+      files = Waterdata.handleZip zipstream
       assert.equal files.length, 2
       assert.equal typeof(files[0]), 'string'
       done()
@@ -29,24 +23,24 @@ describe 'convert csv to array', ->
 
   it 'should convert one csv to an array', (done) ->
     csv = "1,2,3\n4,5,6"
-    csvToArray csv, (error, res) ->
+    Waterdata.csvToArray csv, (error, res) ->
       assert.equal res.length, 2
       assert.equal res[0].length, 3
       assert.equal res[1][1], '5'
       done()
 
   it 'should be able to convert arrays out of the rws data', (done) ->
-    getZip rwsUrl, (zipstream) ->
-      files = handleZip zipstream
+    Waterdata.getZip Waterdata.url, (zipstream) ->
+      files = Waterdata.handleZip zipstream
       dataFile = files[1]
-      csvToArray dataFile, (error, res) ->
+      Waterdata.csvToArray dataFile, (error, res) ->
         assert.equal typeof(res), 'object'
         done()
 
 describe 'the whole process', ->
   it 'should not time out', (done) ->
     Location.connect('mongodb://localhost:27017/waterstanden-test')
     this.timeout 10000
-    processWaterdata ->
+    Waterdata.processWaterdata ->
       Location.db.close()
       done()
diff --git a/web.coffee b/web.coffee
@@ -1,5 +1,5 @@
 flatiron = require 'flatiron'
-dbToJson = require('./lib/lib').dbToJson
+Waterdata = require('./lib/lib').Waterdata
 Location = require('./lib/models').Location
 
 app = flatiron.app
@@ -14,11 +14,12 @@ app.config.defaults({
 Location.connect(app.config.get('MONGOHQ_URL'))
 
 read = ->
-  dbToJson (error, json) =>
+  Waterdata.dbToJson (error, json) =>
     @res.writeHead(200, {'Content-Type': 'application/json'})
     @res.write JSON.stringify(json)
     @res.end()
 
 port = process.env.PORT || 5000
 app.router.get '/', read
 app.listen(port)
+console.log "Waterdata draait op #{port}"