Skip to content

Commit 5351ee6

Browse files
pateashpateash
committed
[cdt-63] : to_csv added, progress bar added,
closes #63 Signed-off-by: ashish <[email protected]>
1 parent a3efc62 commit 5351ee6

File tree

3 files changed

+68
-47
lines changed

3 files changed

+68
-47
lines changed

‎src/commands/avro.ts

Lines changed: 56 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -87,65 +87,80 @@ export default class Avro extends Command {
8787

8888
// tslint:disable-next-line:no-unused
8989
private toJson(flags: any, args: any) {
90-
Utilities.truncateFile(this, flags.output)
91-
avro.createFileDecoder(flags.file)
92-
.on('data', function (recordStr) {
90+
Logger.progressStart(this, 'Converting Avro To Json')
91+
setTimeout(() => {
92+
Logger.progressStop(this, ' Done')
93+
Utilities.truncateFile(this, flags.output)
94+
avro.createFileDecoder(flags.file)
95+
.on('data', function (recordStr) {
9396
// @ts-ignore
94-
Utilities.appendStringToFile(this, flags.output, JSON.stringify(recordStr))
95-
})
96-
Logger.success(this, `${chalk.blue('Json')} written to file: ${chalk.green(flags.output)}`) // this will output error and exit command
97+
Utilities.appendStringToFile(this, flags.output, JSON.stringify(recordStr))
98+
})
99+
Logger.success(this, `${chalk.blue('Json')} written to file: ${chalk.green(flags.output)}`) // this will output error and exit command
100+
}, 1000)
97101
}
98102

99-
// // tslint:disable-next-line:no-unused
103+
// tslint:disable-next-line:no-unused
100104
private toCsv(flags: any, args: any) {
101-
Logger.progressStart(this, '')
102-
Utilities.truncateFile(this, flags.output)
103-
104-
let prependHeader = true // only write on the first line
105-
106-
avro.createFileDecoder(flags.file)
107-
.on('data', function (recordStr) {
108-
// @ts-ignore
109-
let json = JSON.parse(JSON.stringify(recordStr))
110-
Json2Csv.json2csv(json, (err?: Error, csv?: string) => {
111-
if (csv)
112-
Utilities.appendStringToFile(this, flags.output, csv + '\n')
113-
if (err)
114-
Logger.error(this, err)
115-
}, {prependHeader})
116-
prependHeader = false
117-
})
118-
Logger.progressStop(this, "done")
119-
Logger.success(this, `${chalk.blue('Csv')} written to file: ${chalk.green(flags.output)}`) // this will output error and exit command
105+
Logger.progressStart(this, 'Converting Avro To Csv')
106+
107+
setTimeout(() => {
108+
Logger.progressStop(this, ' Done')
109+
Utilities.truncateFile(this, flags.output)
110+
let prependHeader = true // only write on the first line
111+
avro.createFileDecoder(flags.file)
112+
.on('data', function (recordStr) {
113+
// @ts-ignore
114+
let json = JSON.parse(JSON.stringify(recordStr))
115+
Json2Csv.json2csv(json, (err?: Error, csv?: string) => {
116+
if (csv) {
117+
// @ts-ignore
118+
Utilities.appendStringToFile(this, flags.output, csv + '\n')
119+
}
120+
if (err) {
121+
// @ts-ignore
122+
Logger.error(this, err.toString())
123+
}
124+
}, {prependHeader})
125+
prependHeader = false
126+
})
127+
Logger.success(this, `${chalk.blue('Csv')} written to file: ${chalk.green(flags.output)}`) // this will output error and exit command
128+
}, 1000)
120129
}
121130

122131
private toAvro(flags: any, args: any) {
123132
if (!flags.schemaType)
124133
Logger.error(this, 'Schema file is not provided')
125134

126-
let schema = avro.parse(flags.schemaType)
127-
let avroEncoder = new avro.streams.BlockEncoder(schema)
135+
Logger.progressStart(this, 'Generating Avro')
136+
setTimeout(() => {
137+
Logger.progressStop(this, ' Done')
138+
139+
let schema = avro.parse(flags.schemaType)
140+
let avroEncoder = new avro.streams.BlockEncoder(schema)
128141

129-
avroEncoder.pipe(fs.createWriteStream(flags.output))
142+
avroEncoder.pipe(fs.createWriteStream(flags.output))
130143

131144
// We write the records to the block encoder, which will take care of serializing them
132145
// into an object container file.
133146

134-
let inputString = Utilities.getInputString(this, flags, args)
135-
let jsonStr = this.convertAvroJsonToValidJson(inputString)
147+
let inputString = Utilities.getInputString(this, flags, args)
148+
let jsonStr = this.convertAvroJsonToValidJson(inputString)
136149

137-
let jsonObjects = JSON.parse(jsonStr)
150+
let jsonObjects = JSON.parse(jsonStr)
138151

139-
jsonObjects.forEach(function (data: any) {
140-
if (schema.isValid(data)) {
141-
avroEncoder.write(data)
142-
} else {
152+
jsonObjects.forEach(function (data: any) {
153+
if (schema.isValid(data)) {
154+
avroEncoder.write(data)
155+
} else {
143156
// @ts-ignore
144-
Logger.warn(this, `${chalk.yellow('[SKIPPING RECORD]')} schema is invalid: ${chalk.yellowBright(JSON.stringify(data))}`)
145-
}
146-
})
147-
Logger.success(this, `${chalk.blue('Avro')} written to file: ${chalk.green(flags.output)}`) // this will output error and exit command
148-
avroEncoder.end()
157+
Logger.warn(this, `${chalk.yellow('[SKIPPING RECORD]')} schema is invalid: ${chalk.yellowBright(JSON.stringify(data))}`)
158+
}
159+
})
160+
Logger.success(this, `${chalk.blue('Avro')} written to file: ${chalk.green(flags.output)}`) // this will output error and exit command
161+
avroEncoder.end()
162+
}, 1000)
163+
149164
}
150165

151166
private convertAvroJsonToValidJson(json: string) {

‎src/utilities/utilities.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ export default class Utilities {
6161
}
6262

6363
public static truncateFile(thisRef: Command, filePath: string) {
64-
if (fs.existsSync(filePath))
65-
Logger.info(thisRef, `file found: ${chalk.yellow(filePath + ', truncating')}`) // this will output error and exit command
64+
// if (fs.existsSync(filePath))
65+
// Logger.info(thisRef, `file found: ${chalk.yellow(filePath + ', truncating')}`) // this will output error and exit command
6666
Utilities.writeStringToFile(thisRef, filePath, '') // write nothing
6767
}
6868
}

‎test/commands/avro.test.ts

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,21 +64,27 @@ describe('avro', () => {
6464
.stdout()
6565
.command(['avro', '-f', 'test/resources/avro/person.avro', '-o', 'test/resources/avro/output/person.json','to_json'])
6666
.it('if to_json commands run with success', ctx => {
67-
expect(ctx.stdout).to.contain('success')
67+
setTimeout(() =>
68+
expect(ctx.stdout).to.contain('success')
69+
, 5000) // wait for it to write stuff on console, we had to add this as we have setTimeout in our code
6870
})
6971

7072
test
7173
.stdout()
7274
.command(['avro', '-f', 'test/resources/avro/person.avro', '-o', 'test/resources/avro/output/person.csv','to_csv'])
7375
.it('if to_csv commands run with success', ctx => {
74-
expect(ctx.stdout).to.contain('success')
76+
setTimeout(() =>
77+
expect(ctx.stdout).to.contain('success')
78+
, 5000) // wait for it to write stuff on console
7579
})
7680

7781
test
7882
.stdout()
79-
.command(['avro', '-f', 'test/resources/avro/person.json', '-o', 'test/resources/avro/output/person.avro', '-t', 'test/resources/avro/person.avsc', 'to_avro'])
83+
.command(['avro', '-f', 'test/resources/avro/twitter.json', '-o', 'test/resources/avro/output/twitter.avro', '-t', 'test/resources/avro/twitter.avsc', 'to_avro'])
8084
.it('if to_avro commands run with success', ctx => {
81-
expect(ctx.stdout).to.contain('success')
85+
setTimeout(() =>
86+
expect(ctx.stdout).to.contain('success')
87+
, 5000) // wait for it to write stuff on console
8288
})
8389

8490
})

0 commit comments

Comments
 (0)