Merge pull request #80 from SparkPost/feat/sqs-extended-send-uuid-prefix

feat: sqs extended send UUID prefix

Author: viniciusgiles

lib/sqs/index.js

@@ -231,12 +231,12 @@ module.exports = ({
      * too large to fit in sqs, it will write a sqs with a pointer to an s3
      * bucket
      *
-     * @param queueName  Name of queue
-     * @param payload    body of message
-     * @param attrs      attributes attached to message
-     * @param s3Bucket   bucket name for message storage, if it exceeds limit
-     * @param prefix     bucket path prefix (no slashes)
-     * @param shards     number of bucket shard paths
+     * @param {string}        queueName  Name of queue
+     * @param {string|object} payload    body of message
+     * @param {object}        attrs      attributes attached to message
+     * @param {string}        s3Bucket   bucket name for message storage, if it exceeds limit
+     * @param {number}        shards     (deprecated) number of bucket shard paths
+     * @param {boolean}       uuidPrefix use uuid() as S3 path prefix to prevent S3 throttling during high throughput scenarios. When true, ignores 'shards' parameter.
      */
     extendedSend: ({
       queueName,
@@ -245,7 +245,8 @@ module.exports = ({
       s3Bucket,
       messageGroupId,
       messageDeduplicationId,
-      shards = 50
+      shards = 50,
+      uuidPrefix = false
     }) => {
       if (!s3Bucket) {
         return Promise.reject(new Error('S3 Bucket name required'));
@@ -269,7 +270,13 @@ module.exports = ({
       return compress(payload, { level })
         .then((payload) => {
           if (computeMessageSize(payload, attrs) > MAX_MSG_SIZE) {
-            key = `/${Math.floor(Math.random() * shards)}/${uuid()}.json.gz`;
+            // use a random shard as path prefix by default for backwards compatibility,
+            // however, S3 throttles during high throughput events are more likely.
+            // `uuidPrefix` allows ultra high throughput by avoiding a fixed number of shards.
+            const prefix = uuidPrefix
+              ? uuid()
+              : Math.floor(Math.random() * shards);
+            key = `/${prefix}/${uuid()}.json.gz`;
 
             return S3.uploadAsync({
               Body: payload,