✨ feat(bigquery): inserindo dados no BQ na requisição post

Author: 1cadumagalhaes

bigquery.js

@@ -18,23 +18,23 @@ let schemaExample = [
  */
 async function getOrCreateDataset(datasetId) {
   const bq = new BigQuery();
-  const dataset = await bq.dataset(datasetId).get({ autoCreate: true });
-  return dataset;
+  await bq.dataset(datasetId).get({ autoCreate: true });
+  return bq.dataset(datasetId);
 }
 
 async function getOrCreateTable({ datasetId, tableName, schema }) {
   try {
     const dataset = await getOrCreateDataset(datasetId);
-
     const options = {
       schema,
       location: 'US',
       type: 'TABLE',
     };
-    return dataset.table(tableName).get({
+    const table = await dataset.table(tableName).get({
       autoCreate: true,
       ...options,
     });
+    return dataset.table(tableName);
   } catch (error) {
     console.error(`Error getting table ${tableName}:`, error);
     throw new Error(error);
@@ -65,23 +65,29 @@ async function insertRowsAsStream(rows, datasetId, tableName, schema) {
  * @param {*} tableName
  * @param {*} schema - Schema of the table, in case you need to create it.
  */
-async function loadData(rows, datasetId, tableName, schema) {
+async function loadData(rows, datasetId, tableName) {
   try {
     const metadata = {
       sourceFormat: 'NEWLINE_DELIMITED_JSON',
       location: 'US',
       writeDisposition: 'WRITE_APPEND',
     };
-    const tempFilePath = path.join(os.tmpdir(), 'data.json');
     let data = rows instanceof Array ? rows : [rows];
+    const schema = generateSchema(data[0]);
     data = data.map((r) => JSON.stringify(r)).join('\n');
+
+    const tempFilePath = path.join(os.tmpdir(), 'data.json');
     fs.writeFileSync(tempFilePath, data);
     const table = await getOrCreateTable({ datasetId, tableName, schema });
+    console.info(`Trying to create load job with ${data.length} rows.`);
     await table.load(tempFilePath, metadata, (err, apiResponse) => {
-      if (err) throw new Error('Error loading job');
+      if (err) {
+        throw new Error(err);
+      }
     });
+    fs.unlinkSync(tempFilePath);
   } catch (error) {
-    console.error(`Error inserting ${rows.lenght} in ${tableId}. \n`, error);
+    console.error(`Error inserting ${rows.lenght || 1} row in ${tableName}. \n`, error);
   }
 }
 

index.js

@@ -3,11 +3,12 @@ const express = require('express');
 const cors = require('cors');
 const { insertRowAsStream, loadData } = require('./bigquery');
 const { hubMessage } = require('./message');
+const schema = require('./log_schema.json');
 
 const app = express();
 const port = 8000;
 const DATASET_ID = 'raft_suite',
-  TABLE_ID = 'record';
+  TABLE_ID = 'hub';
 
 app.use((req, res, next) => {
   res.header('Access-Control-Allow-Origin', '*');
@@ -21,10 +22,10 @@ app.use(cors());
 app
   .route('/')
   .get(function (req, res) {
-    const message = JSON.stringify(hubMessage(), null, 2);
+    let message = hubMessage();
+    message = JSON.stringify(message, null, 2);
     const response = `This is a example message:\n${message}`;
     res.status(200).send(response);
-    loadData(message, DATASET_ID, TABLE_ID);
     console.log(message);
   })
   .post(function (req, res) {
@@ -41,6 +42,7 @@ app
       const data = hubMessage(body);
       res.status(200).send(data.jobId);
       console.log(JSON.stringify(data));
+      loadData(data, DATASET_ID, TABLE_ID);
     } catch (error) {
       res.status(400).send(error.toString());
       console.error(error);

log.json

@@ -12,5 +12,5 @@
     "description": "",
     "details": ""
   },
-  "payload": "{\"teste\":: \"Cadu\"}"
+  "payload": "{\"teste\": \"Cadu\"}"
 }

log_schema.json

@@ -15,10 +15,6 @@
     "name": "module",
     "type": "STRING"
   },
-  {
-    "name": "project",
-    "type": "STRING"
-  },
   {
     "name": "spec",
     "type": "STRING"

message.js

@@ -1,5 +1,6 @@
 const cuid = require('cuid');
 const status = require('./status.json');
+const { version } = require('./package.json');
 
 function getStatusMessage(statusCode) {
   let [dimension, code] = statusCode.split('-');
@@ -35,16 +36,16 @@ function hubMessage(body) {
   return {
     timestamp: new Date().toISOString(),
     jobId: cuid(),
-    project: project,
-    module: module,
-    spec: spec,
-    deploy: deploy,
-    version: '0.0.1',
+    project,
+    module,
+    spec,
+    deploy,
+    version,
     status: {
-      code: code,
+      code,
       message: getStatusMessage(code),
-      description: description,
-      details: details,
+      description,
+      details,
     },
     payload: JSON.stringify(payload),
   };

package.json

@@ -1,7 +1,7 @@
 {
   "name": "raft-suite-hub",
   "private": "true",
-  "version": "0.1.5",
+  "version": "1.0.0",
   "description": "Google Cloud Function",
   "repository": {
     "type": "git",
@@ -80,4 +80,4 @@
     "sinon": "^11.1.1",
     "uuid": "^8.3.2"
   }
-}
+}

test/unit/bigquery.test.js

@@ -1,6 +1,7 @@
 const { assert, expect } = require('chai');
 
 const { formatEntrie, generateSchema } = require('../../bigquery');
+const { hubMessage } = require('../../message');
 
 const COLUMN_NAME = 'Teste';
 const TYPES = [
@@ -33,4 +34,8 @@ describe.only('BigQuery helper', () => {
     expect(name).to.equal('a');
     expect(type).to.equal('INT64');
   });
+  it('formatEntrie - Converte uma mensagem em schema', () => {
+    const message = hubMessage();
+    const schema = generateSchema(message);
+  });
 });