Merge pull request #64 from codingtools/cdt-63

Add to_csv support for Avro command

Author: pateash

.gitignore

@@ -9,6 +9,12 @@ node_modules
 /.idea
 cdt.iml
 **/.DS_Store
-output/*
-!output/.gitkeep
+
+output
+output/**.json
+output/**.avro
+output/**.avsc
+
 coverage/*.json
+
+

README.md

@@ -161,11 +161,11 @@ USAGE
   $ cdt hash [STRING]
 
 OPTIONS
-  -f, --file=file              file to be hashed
-  -h, --help                   show CLI help
-  -o, --outputFile=outputFile  output file path
-  -s, --string=string          string to be hashed
-  -t, --type=type              type of hash [SHA1(default), MD5, SHA256, SHA512, RMD160 or RIPEMD160]
+  -f, --file=file      file to be hashed
+  -h, --help           show CLI help
+  -o, --output=output  output file path
+  -s, --string=string  string to be hashed
+  -t, --type=type      type of hash [SHA1(default), MD5, SHA256, SHA512, RMD160 or RIPEMD160]
 ```
 
 _See code: [src/commands/hash.ts](https://github.com/codingtools/cdt/blob/v0.1.6/src/commands/hash.ts)_
@@ -196,12 +196,10 @@ USAGE
   $ cdt minify [FILE]
 
 OPTIONS
-  -f, --file=file              file to be minified
-  -h, --help                   show CLI help
-  -o, --outputFile=outputFile  output file path
-
-  -t, --type=type              type of file to be minified, it will try to find type with extension supported: JS,
-                               HTML/HTM, CSS
+  -f, --file=file      file to be minified
+  -h, --help           show CLI help
+  -o, --output=output  output file path
+  -t, --type=type      type of file to be minified, it will try to find type with extension supported: JS, HTML/HTM, CSS
 ```
 
 _See code: [src/commands/minify.ts](https://github.com/codingtools/cdt/blob/v0.1.6/src/commands/minify.ts)_

oclif.manifest.json

@@ -0,0 +1 @@
+{"version":"0.1.6","commands":{"avro":{"id":"avro","description":"Avro Utility command","pluginName":"@codingtools/cdt","pluginType":"core","aliases":[],"flags":{"help":{"name":"help","type":"boolean","char":"h","description":"show CLI help","allowNo":false},"file":{"name":"file","type":"option","char":"f","description":"input file path"},"output":{"name":"output","type":"option","char":"o","description":"output file path"},"schemaType":{"name":"schemaType","type":"option","char":"t","description":"schema type file path"}},"args":[{"name":"command"}]},"bundlephobia":{"id":"bundlephobia","description":"Find cost of adding a npm/yarn packages or all dependencies in package.json file","pluginName":"@codingtools/cdt","pluginType":"core","aliases":[],"flags":{"help":{"name":"help","type":"boolean","char":"h","description":"show CLI help","allowNo":false},"packages":{"name":"packages","type":"option","char":"p","description":"packages for which cost is required, can pass more than one separated by space"},"file":{"name":"file","type":"option","char":"f","description":"path for package.json file"}},"args":[{"name":"package"}]},"crypto":{"id":"crypto","description":"Encryption and Decryption functionality for File/String","pluginName":"@codingtools/cdt","pluginType":"core","aliases":[],"flags":{"help":{"name":"help","type":"boolean","char":"h","description":"show CLI help","allowNo":false},"encryption":{"name":"encryption","type":"option","char":"e","description":"encryption type, Supported [AES, DES, 3DES, Rabbit, RC4, RC4Drop]"},"decryption":{"name":"decryption","type":"option","char":"d","description":"decryption type, Supported [AES, DES, 3DES, Rabbit, RC4, RC4Drop]"},"string":{"name":"string","type":"option","char":"s","description":"string to be encrypted/decrypted"},"file":{"name":"file","type":"option","char":"f","description":"file to be encrypted/decrypted"},"key":{"name":"key","type":"option","char":"k","description":"key for encryption/decryption"},"mode":{"name":"mode","type":"option","char":"m","description":"Block Mode, Supported [CBC, CFB, CTR, OFB, ECB]"}},"args":[{"name":"string"}]},"datetime":{"id":"datetime","description":"Date and Time utility","pluginName":"@codingtools/cdt","pluginType":"core","aliases":[],"flags":{"help":{"name":"help","type":"boolean","char":"h","description":"show CLI help","allowNo":false},"date":{"name":"date","type":"option","char":"d","description":"Datetime input string, default: Current Datetime, could also be passed through argument"},"format":{"name":"format","type":"option","char":"f","description":"Datetime format, default: Do MMMM YYYY, h:m:s A, Z UTC"},"timezone":{"name":"timezone","type":"option","char":"z","description":"Timezone for Datetime parsing, default: Your timezone"},"locale":{"name":"locale","type":"option","char":"l","description":"Locale, default: en"}},"args":[{"name":"date"}]},"hash":{"id":"hash","description":"Hashing functionality for a string/file","pluginName":"@codingtools/cdt","pluginType":"core","aliases":[],"flags":{"help":{"name":"help","type":"boolean","char":"h","description":"show CLI help","allowNo":false},"type":{"name":"type","type":"option","char":"t","description":"type of hash [SHA1(default), MD5, SHA256, SHA512, RMD160 or RIPEMD160]"},"string":{"name":"string","type":"option","char":"s","description":"string to be hashed"},"file":{"name":"file","type":"option","char":"f","description":"file to be hashed"},"output":{"name":"output","type":"option","char":"o","description":"output file path"}},"args":[{"name":"string"}]},"minify":{"id":"minify","description":"File Minifier","pluginName":"@codingtools/cdt","pluginType":"core","aliases":[],"flags":{"help":{"name":"help","type":"boolean","char":"h","description":"show CLI help","allowNo":false},"type":{"name":"type","type":"option","char":"t","description":"type of file to be minified, it will try to find type with extension supported: JS, HTML/HTM, CSS"},"file":{"name":"file","type":"option","char":"f","description":"file to be minified"},"output":{"name":"output","type":"option","char":"o","description":"output file path"}},"args":[{"name":"file"}]}}}

package-lock.json

@@ -24,6 +24,7 @@
     "detect-character-encoding": "^0.8.0",
     "encoding-japanese": "^1.0.30",
     "jshashes": "^1.0.7",
+    "json-2-csv": "^3.5.9",
     "minify": "^5.0.0",
     "moment": "^2.24.0",
     "moment-timezone": "^0.5.27",

package.json

@@ -2,6 +2,7 @@ import {Command, flags} from '@oclif/command'
 import * as avro from 'avsc'
 import * as chalk from 'chalk'
 import * as fs from 'fs' // includes all from avro-js and some more
+import * as Json2Csv from 'json-2-csv'
 
 import Logger from '../utilities/logger'
 import Utilities from '../utilities/utilities'
@@ -12,9 +13,10 @@ export default class Avro extends Command {
   static GET_SCHEMA = 'get_schema'
   static TO_JSON = 'to_json'
   static TO_AVRO = 'to_avro'
+  static TO_CSV = 'to_csv'
 
   // do not change order otherwise we need to change order in getCommand() also
-  static SupportedCommands = [Avro.GET_SCHEMA, Avro.TO_JSON, Avro.TO_AVRO]
+  static SupportedCommands = [Avro.GET_SCHEMA, Avro.TO_JSON, Avro.TO_AVRO, Avro.TO_CSV]
   static flags = {
     help: flags.help({char: 'h'}),
     file: flags.string({char: 'f' , description: 'input file path'}),
@@ -52,12 +54,14 @@ export default class Avro extends Command {
 
   private executeCommand(flags: any, args: any) {
     switch (args.command) {
-    case Avro.SupportedCommands[0]:
+    case Avro.GET_SCHEMA:
       return this.getSchema(flags, args)
-    case Avro.SupportedCommands[1]:
+    case Avro.TO_JSON:
       return this.toJson(flags, args)
-    case Avro.SupportedCommands[2]:
+    case Avro.TO_AVRO:
       return this.toAvro(flags, args)
+    case Avro.TO_CSV:
+      return this.toCsv(flags, args)
     default:
       Logger.error(this, 'Unsupported Command, supported: ' + Avro.SupportedCommands)
     }
@@ -83,19 +87,55 @@ export default class Avro extends Command {
 
   // tslint:disable-next-line:no-unused
   private toJson(flags: any, args: any) {
+    Logger.progressStart(this, 'Converting Avro To Json')
+    // setTimeout(() => {
+    Logger.progressStop(this, ' Converting Avro To Json')
     Utilities.truncateFile(this, flags.output)
     avro.createFileDecoder(flags.file)
       .on('data', function (recordStr) {
         // @ts-ignore
         Utilities.appendStringToFile(this, flags.output, JSON.stringify(recordStr))
       })
     Logger.success(this, `${chalk.blue('Json')} written to file: ${chalk.green(flags.output)}`) // this will output error and exit command
+    // }, 1000)
+  }
+
+// tslint:disable-next-line:no-unused
+  private toCsv(flags: any, args: any) {
+    Logger.progressStart(this, 'Converting Avro To Csv')
+
+    // setTimeout(() => {
+    Logger.progressStop(this, ' Converting Avro To Csv')
+    Utilities.truncateFile(this, flags.output)
+    let prependHeader = true // only write on the first line
+    avro.createFileDecoder(flags.file)
+      .on('data', function (recordStr) {
+          // @ts-ignore
+        let json = JSON.parse(JSON.stringify(recordStr))
+        Json2Csv.json2csv(json, (err?: Error, csv?: string) => {
+          if (csv) {
+              // @ts-ignore
+            Utilities.appendStringToFile(this, flags.output, csv + '\n')
+          }
+          if (err) {
+              // @ts-ignore
+            Logger.error(this, err.toString())
+          }
+        }, {prependHeader})
+        prependHeader = false
+      })
+    Logger.success(this, `${chalk.blue('Csv')} written to file: ${chalk.green(flags.output)}`) // this will output error and exit command
+    // }, 300)
   }
 
   private toAvro(flags: any, args: any) {
     if (!flags.schemaType)
       Logger.error(this, 'Schema file is not provided')
 
+    Logger.progressStart(this, 'Generating Avro')
+    // setTimeout(() => {
+    Logger.progressStop(this, ' Generating Avro')
+
     let schema = avro.parse(flags.schemaType)
     let avroEncoder = new avro.streams.BlockEncoder(schema)
 
@@ -104,9 +144,9 @@ export default class Avro extends Command {
 // We write the records to the block encoder, which will take care of serializing them
 // into an object container file.
 
-    let jsonStr = '[' + Utilities.getInputString(this, flags, args) + ']'
-    jsonStr = jsonStr.replace(/[\s\n]+/mg, '')
-    jsonStr = jsonStr.replace(/\}\{/mg, '},{')
+    let inputString = Utilities.getInputString(this, flags, args)
+    let jsonStr = this.convertAvroJsonToValidJson(inputString)
+
     let jsonObjects = JSON.parse(jsonStr)
 
     jsonObjects.forEach(function (data: any) {
@@ -119,5 +159,14 @@ export default class Avro extends Command {
     })
     Logger.success(this, `${chalk.blue('Avro')} written to file: ${chalk.green(flags.output)}`) // this will output error and exit command
     avroEncoder.end()
+    // }, 300)
+
+  }
+
+  private convertAvroJsonToValidJson(json: string) {
+    let jsonStr = '[' + json + ']'
+    jsonStr = jsonStr.replace(/[\s\n]+/mg, '')
+    jsonStr = jsonStr.replace(/\}\{/mg, '},{')
+    return jsonStr
   }
 }

src/commands/avro.ts

@@ -22,19 +22,6 @@ export default class Bundlephobia extends Command {
 
   static args = [{name: 'package'}] // only one can be passed club which one passed through flag and arg
 
-  private static getErrorMessage(pkg: string, message: string) {
-    // replacing will be useful when we do not have specific version
-    // output will be like below
-/*
-    ⚠ @codingtools/[email protected] This package has not been published with this particular version.
-                     Valid versions - `<code>latest</code>`, `<code>0.1.1</code>` and `<code>0.1.2</code>`
-*/
-    if (message.includes('This package has not been published with this particular version.'))
-      message = message.replace(/`<code>|<\/code>`/g, '')
-
-    return `${chalk.red(pkg)} ${message}`
-  }
-
   private static getSize(byteSize: number) {
     if (byteSize >= 1024 * 1024)
       return `${chalk.red((byteSize / (1024 * 1024)).toFixed(1) + 'MB')}`
@@ -55,6 +42,19 @@ export default class Bundlephobia extends Command {
     this.bundlePhobia(flags, args)
   }
 
+  private getErrorMessage(pkg: string, message: string) {
+    // replacing will be useful when we do not have specific version
+    // output will be like below
+/*
+    ⚠ @codingtools/[email protected] This package has not been published with this particular version.
+                     Valid versions - `<code>latest</code>`, `<code>0.1.1</code>` and `<code>0.1.2</code>`
+*/
+    if (message.includes('This package has not been published with this particular version.'))
+      message = message.replace(/`<code>|<\/code>`/g, '')
+
+    return `${chalk.red(pkg)} ${message}`
+  }
+
   private getPackages(flags: any, args: any) {
     let packages = []
 
@@ -104,7 +104,7 @@ export default class Bundlephobia extends Command {
         dependencyCount += successResponse.data.dependencyCount
         Logger.progressStop(this, this.getSuccessMessage(successResponse.data))
       }).catch(errorResponse => {
-        Logger.progressStopError(this, Bundlephobia.getErrorMessage(packageInfo.pkg, errorResponse.response.data.error.message))
+        Logger.progressStopError(this, this.getErrorMessage(packageInfo.pkg, errorResponse.response.data.error.message))
       })
       // tslint:disable-next-line:no-unused
     }))

src/commands/bundlephobia.ts

@@ -61,8 +61,8 @@ export default class Utilities {
   }
 
   public static truncateFile(thisRef: Command, filePath: string) {
-    if (fs.existsSync(filePath))
-      Logger.info(thisRef, `file found: ${chalk.yellow(filePath + ', truncating')}`) // this will output error and exit command
+    // if (fs.existsSync(filePath))
+    //   Logger.info(thisRef, `file found: ${chalk.yellow(filePath + ', truncating')}`) // this will output error and exit command
     Utilities.writeStringToFile(thisRef, filePath, '') // write nothing
   }
 }

src/utilities/utilities.ts

@@ -1,4 +1,5 @@
 import {expect, test} from '@oclif/test'
+import {readFileSync} from 'fs'
 
 describe('avro', () => {
   //todo if file is invalid
@@ -49,6 +50,8 @@ describe('avro', () => {
       expect(ctx.stdout).to.contain('Schema file is not provided')
     })
 
+
+  // TODO: BUG this is just skipping test , find a way to implement wait() etc.
   test
     .stdout()
     .command(['avro', '-f', 'test/resources/avro/person.avro', 'get_schema'])
@@ -58,19 +61,24 @@ describe('avro', () => {
         , 5000) //  wait for it to write stuff on console
     })
 
-
   test
     .stdout()
-    .command(['avro', '-f', 'test/resources/avro/person.avro', '-o', 'test/resources/avro/output/person.json','to_json'])
-    .it('if to_json commands run with success', ctx => {
+    .command(['avro', '-f', 'test/resources/avro/twitter.json', '-o', 'test/resources/avro/output/twitter.avro', '-t', 'test/resources/avro/twitter.avsc', 'to_avro'])
+    .it('if to_avro commands run with success', ctx => {
       expect(ctx.stdout).to.contain('success')
     })
 
   test
-    .timeout(20000) // added timeout to resolve timeout problem
     .stdout()
-    .command(['avro', '-f', 'test/resources/avro/person.json', '-o', 'test/resources/avro/output/person.avro', '-t', 'test/resources/avro/person.avsc', 'to_avro'])
-    .it('if to_avro commands run with success', ctx => {
+    .command(['avro', '-f', 'test/resources/avro/person.avro', '-o', 'test/resources/avro/output/person.json','to_json'])
+    .it('if to_json commands run with success', ctx => {
       expect(ctx.stdout).to.contain('success')
     })
 })
+
+test
+  .stdout()
+  .command(['avro', '-f', 'test/resources/avro/person.avro', '-o', 'test/resources/avro/output/person.csv','to_csv'])
+  .it('if to_csv commands run with success', ctx => {
+    expect(ctx.stdout).to.contain('success')
+  })

test/commands/avro.test.ts

@@ -0,0 +1,19 @@
+{
+  "type" : "record",
+  "name" : "twitter_schema",
+  "namespace" : "com.miguno.avro",
+  "fields" : [ {
+    "name" : "username",
+    "type" : "string",
+    "doc"  : "Name of the user account on Twitter.com"
+  }, {
+    "name" : "tweet",
+    "type" : "string",
+    "doc"  : "The content of the user's Twitter message"
+  }, {
+    "name" : "timestamp",
+    "type" : "long",
+    "doc"  : "Unix epoch time in seconds"
+  } ],
+  "doc:" : "A basic schema for storing Twitter messages"
+}

test/resources/avro/complex-record.avro renamed to test/resources/avro/complex.avro

@@ -0,0 +1,2 @@
+{"username":"miguno","tweet":"Rock: Nerf paper, scissors is fine.","timestamp": 1366150681 }
+{"username":"BlizzardCS","tweet":"Works as intended.  Terran is IMBA.","timestamp": 1366154481 }