Merge pull request #224 from topcoder-platform/Issue_223

Issue 223

Author: sachin-maheshwari

package.json

@@ -15,6 +15,7 @@
     "init-es": "node scripts/loadES.js",
     "db-to-es": "node scripts/migrateFromDBToES.js",
     "update-to-v5-challengeId": "node scripts/updateToV5ChallengeId.js",
+    "update-to-v5-challengeId-v2": "node scripts/updateToV5ChallengeIdV2.js",
     "test": "mocha test/unit/*.test.js --require test/unit/prepare.js --exit",
     "e2e": "mocha test/e2e/*.test.js --require test/e2e/prepare.js --exit",
     "cov": "nyc --reporter=html --reporter=text mocha test/unit/*.test.js --require test/unit/prepare.js --exit",

scripts/updateToV5ChallengeIdV2.js

@@ -0,0 +1,87 @@
+/**
+ * Store v5 challenge id for current records
+ */
+
+const _ = require('lodash')
+const co = require('co')
+const config = require('config')
+const logger = require('../src/common/logger')
+const dbhelper = require('../src/common/dbhelper')
+const helper = require('../src/common/helper')
+
+/**
+ * Update Submission's challenge id to v5
+ * @param {Object} submission The submission record
+ * @param {Array} failedContainer The failed records container
+ * @param {String} v5challengeId The v5 challenge id
+ * @returns {Promise}
+ */
+function * updateRecord (submission, failedContainer, v5challengeId) {
+  const record = {
+    TableName: 'Submission',
+    Key: {
+      id: submission.id
+    },
+    UpdateExpression: `set challengeId = :c, legacyChallengeId = :l`,
+    ExpressionAttributeValues: {
+      ':c': v5challengeId,
+      ':l': submission.challengeId
+    }
+  }
+  try {
+    yield dbhelper.updateRecord(record)
+  } catch (err) {
+    logger.error(`update submission record error: ${err.message}`)
+    failedContainer.push(submission)
+  }
+}
+
+/*
+ * Update all submission's challenge id to v5
+ * @returns {Promise}
+ */
+function * updateRecords () {
+  const tableName = config.SUBMISSION_TABLE_NAME
+  const promises = []
+  const failedRecords = []
+  // Process until all the records from DB is fetched
+  const challengeIds = yield helper.getLatestChallenges()
+  logger.debug(`Number of challenges fetched from api - ${challengeIds.length}.`)
+  const params = {
+    TableName: tableName
+  }
+  while (true) {
+    const records = yield dbhelper.scanRecords(params)
+    const totalRecords = records.Items.length
+    logger.debug(`Number of ${tableName}s fetched from DB - ${totalRecords}. More fetch iterations may follow (pagination in progress)`)
+    for (let i = 0; i < totalRecords; i++) {
+      const record = records.Items[i]
+      const v5ChallengeId = _.get(_.find(challengeIds, ['legacyId', record.challengeId]), 'id')
+      if (v5ChallengeId) {
+        promises.push(updateRecord(record, failedRecords, v5ChallengeId))
+      }
+    }
+    // Continue fetching the remaining records from Database
+    if (typeof records.LastEvaluatedKey !== 'undefined') {
+      params.ExclusiveStartKey = records.LastEvaluatedKey
+    } else {
+      break // If there are no more records to process, exit the loop
+    }
+  }
+  logger.debug(`All records fetched. Proceeding to update them in batches of ${config.UPDATE_V5_CHALLENGE_BATCH_SIZE}`)
+  const paraRecords = _.chunk(promises, config.UPDATE_V5_CHALLENGE_BATCH_SIZE)
+  for (const rs of paraRecords) {
+    yield rs
+  }
+  logger.info(`Processed ${promises.length - failedRecords.length} records successfully`)
+  if (failedRecords.length > 0) {
+    logger.warn(`Processing of ${failedRecords.length} records failed`)
+    logger.info(`Failed records: ${_.join(_.map(failedRecords, f => JSON.stringify(_.pick(f, ['id', 'challengeId'])), ','))}`)
+  }
+}
+
+co(function * () {
+  yield updateRecords()
+}).catch((err) => {
+  logger.logFullError(err)
+})