Skip to content

Commit

Permalink
clearer logic
Browse files Browse the repository at this point in the history
  • Loading branch information
JeanRemiDelteil committed Mar 30, 2018
1 parent cb4a995 commit 6d02626
Showing 1 changed file with 88 additions and 62 deletions.
150 changes: 88 additions & 62 deletions src/ErrorHandler.gs.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
* _convertErrorStack()
*****************************************************************/


/**
* Invokes a function, performing up to 5 retries with exponential backoff.
* Retries with delays of approximately 1, 2, 4, 8 then 16 seconds for a total of
Expand All @@ -31,77 +32,102 @@
* @return {*} - The value returned by the called function.
*/
function expBackoff(func) {

// execute func() then retry 5 times at most if errors
for (var n = 0; n < 6; n++) {
try {
var response = func();
// actual exponential backoff
n && Utilities.sleep((Math.pow(2, n-1) * 1000) + (Math.round(Math.random() * 1000)));

var response = undefined;
var error = undefined;

var noError = true;
var isUrlFetchResponse = false;

// Try / catch func()
try { response = func() }
catch(err) {
error = err;
noError = false;
}


// Handle retries on UrlFetch calls with muteHttpExceptions
if (noError && typeof response.getResponseCode === "function") {
isUrlFetchResponse = true;

var responseCode = response.getResponseCode();

// Handle retries on UrlFetch calls with muteHttpExceptions
if (response && typeof response.getResponseCode === "function") {
var responseCode = response.getResponseCode();

// Only perform retries on error 500 for now
if (responseCode == 500) {
if (n === 5) {
ErrorHandler.logError(new Error(response.getContentText()), {
shouldInvestigate: true,
failedAfter5Retries: true,
urlFetchWithMuteHttpExceptions: true,
context: "Exponential Backoff"
});
return response;
}
Utilities.sleep((Math.pow(2, n) * 1000) + (Math.round(Math.random() * 1000)));
continue;
}
// Only perform retries on error 500 for now
if (responseCode === 500) {
error = response;
noError = false;
}
return response;
}
catch(e) {
if (e.message) {
// Check for errors thrown by Google APIs on which there's no need to retry
// eg: "Access denied by a security policy established by the administrator of your organization.
// Please contact your administrator for further assistance."
if (e.message.indexOf('Invalid requests') !== -1
|| e.message.indexOf('Access denied') !== -1
|| e.message.indexOf('Mail service not enabled') !== -1) {
throw e;
}

// TODO: YAMM specific ?: move to YAMM code
else if (e.message.indexOf('response too large') !== -1) {
// Thrown after calling Gmail.Users.Threads.get()
// maybe because a specific thread contains too many messages
// best to skip the thread
return null;
}

// Return result that is not an error
if (noError) return response;


// Process error retry
if (!isUrlFetchResponse && error.message) {
// Check for errors thrown by Google APIs on which there's no need to retry
// eg: "Access denied by a security policy established by the administrator of your organization.
// Please contact your administrator for further assistance."
if (error.message.indexOf('Invalid requests') !== -1
|| error.message.indexOf('Access denied') !== -1
|| error.message.indexOf('Mail service not enabled') !== -1) {
throw error;
}

if (n === 5) {
// 'User-rate limit exceeded' is always followed by 'Retry after' + timestamp
// Maybe we should parse the timestamp to check how long we need to wait
// and if we should abort or not
// 'User Rate Limit Exceeded' (without '-') isn't followed by 'Retry after' and it makes sense to retry
if (e.message && e.message.indexOf('User-rate limit exceeded') !== -1) {
ErrorHandler.logError(e, {
shouldInvestigate: true,
failedAfter5Retries: true,
context: "Exponential Backoff"
});
return null;
}

// Investigate on errors that are still happening after 5 retries
// Especially error "Not Found" - does it make sense to retry on it?
ErrorHandler.logError(e, {
failedAfter5Retries: true,
context: "Exponential Backoff"
});

throw e;
// TODO: YAMM specific ?: move to YAMM code
else if (error.message.indexOf('response too large') !== -1) {
// Thrown after calling Gmail.Users.Threads.get()
// maybe because a specific thread contains too many messages
// best to skip the thread
return null;
}
}

}


// Action after last re-try
if (isUrlFetchResponse) {

ErrorHandler.logError(new Error(response.getContentText()), {
shouldInvestigate: true,
failedAfter5Retries: true,
urlFetchWithMuteHttpExceptions: true,
context: "Exponential Backoff"
});

return response;
}

else {
// 'User-rate limit exceeded' is always followed by 'Retry after' + timestamp
// Maybe we should parse the timestamp to check how long we need to wait
// and if we should abort or not
// 'User Rate Limit Exceeded' (without '-') isn't followed by 'Retry after' and it makes sense to retry
if (error.message && error.message.indexOf('User-rate limit exceeded') !== -1) {
ErrorHandler.logError(error, {
shouldInvestigate: true,
failedAfter5Retries: true,
context: "Exponential Backoff"
});

Utilities.sleep((Math.pow(2, n) * 1000) + (Math.round(Math.random() * 1000)));
return null;
}

// Investigate on errors that are still happening after 5 retries
// Especially error "Not Found" - does it make sense to retry on it?
ErrorHandler.logError(error, {
failedAfter5Retries: true,
context: "Exponential Backoff"
});

throw error;
}
}

Expand Down

0 comments on commit 6d02626

Please sign in to comment.