Skip to content

Commit f59cc7d

Browse files
dgillingdgilling
committed
Update: AWS Lambda to be feature parity with moesif-express
1. Add support for sampling and user supression 2. Add company support 3. Use AWS Request time 4. Add metatdata support and store extra metadata 5. General clean up
1 parent f229bdf commit f59cc7d

File tree

5 files changed

+360
-187
lines changed

5 files changed

+360
-187
lines changed

‎index.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/**
2-
* Created by derric on 8/23/17.
32
* This file is an example AWS Lambda function.
3+
* The Moesif AWS Lambda SDK is in the ./lib directory
44
*/
55

66
const moesif = require('./lib');

‎lib/index.js

+126-50
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,13 @@
1-
/**
2-
* Created by derric on 8/22/17.
3-
*/
41
'use strict'
5-
var _ = require('lodash');
6-
var url = require('url');
7-
var moesifapi = require('moesifapi');
8-
var EventModel = moesifapi.EventModel;
9-
var requestIp = require('request-ip');
10-
var logData = {};
11-
logData.request = {};
12-
logData.response = {};
13-
logData.request.time = Date.now();
2+
const _ = require('lodash');
3+
const url = require('url');
4+
const moesifapi = require('moesifapi');
5+
const requestIp = require('request-ip');
6+
const moesifConfigManager = require('./moesifConfigManager');
7+
const EventModel = moesifapi.EventModel;
8+
const UserModel = moesifapi.UserModel;
9+
const CompanyModel = moesifapi.CompanyModel;
10+
var startTime = Date.now();
1411

1512
//
1613
// ### function moesifExpress(options)
@@ -43,8 +40,15 @@ module.exports = function (options, handler) {
4340
options.identifyCompany = options.identifyCompany || function() {};
4441

4542
options.getSessionToken = options.getSessionToken || function (event, context) {
46-
return (event.requestContext && event.requestContext.identity && event.requestContext.identity.apiKey) ;
43+
return (event.requestContext && event.requestContext.identity && event.requestContext.identity.apiKey);
4744
};
45+
options.getMetadata = options.getMetadata || function (event, context) {
46+
const metadata = {};
47+
metadata.trace_id = context.awsRequestId;
48+
metadata.function_name = context.functionName;
49+
metadata.request_context = event && event.requestContext;
50+
return metadata;
51+
};
4852
options.getTags = options.getTags || function () {
4953
return undefined;
5054
};
@@ -61,15 +65,23 @@ module.exports = function (options, handler) {
6165
return false;
6266
};
6367

68+
var logBody = true;
69+
if (typeof options.logBody !== 'undefined' && options.logBody !== null) {
70+
logBody = Boolean(options.logBody);
71+
}
72+
options.logBody = logBody;
73+
6474
ensureValidOptions(options);
6575

6676
// config moesifapi
6777
var config = moesifapi.configuration;
68-
config.ApplicationId = options.applicationId || process.env.MOESIF_APPLICATION_ID;
78+
config.ApplicationId = options.applicationId || options.ApplicationId || process.env.MOESIF_APPLICATION_ID;
79+
config.BaseUri = options.baseUri || options.BaseUri || config.BaseUri;
6980
var moesifController = moesifapi.ApiController;
7081

7182
var moesifMiddleware = function (event, context, callback) {
7283
logMessage(options.debug, 'moesifMiddleware', 'start');
84+
moesifConfigManager.tryGetConfig();
7385

7486
var next = function (err, result) {
7587
logEvent(event, context, err, result, options, moesifController);
@@ -79,18 +91,42 @@ module.exports = function (options, handler) {
7991
handler(event, context, next);
8092
};
8193

82-
moesifMiddleware.updateUser = function (userModel, cb) {
94+
moesifMiddleware.updateUser = function(userModel, cb) {
95+
const user = new UserModel(userModel);
8396
logMessage(options.debug, 'updateUser', 'userModel=' + JSON.stringify(userModel));
84-
ensureValidUserModel(userModel);
97+
ensureValidUserModel(user);
8598
logMessage(options.debug, 'updateUser', 'userModel valid');
86-
moesifController.updateUser(userModel, cb);
99+
moesifController.updateUser(user, cb);
100+
};
101+
102+
moesifMiddleware.updateUsersBatch = function(usersBatchModel, cb) {
103+
usersBatch = [];
104+
for (let userModel of usersBatchModel) {
105+
usersBatch.push(new UserModel(userModel));
106+
}
107+
logMessage(options.debug, 'updateUsersBatch', 'usersBatchModel=' + JSON.stringify(usersBatchModel));
108+
ensureValidUsersBatchModel(usersBatch);
109+
logMessage(options.debug, 'updateUsersBatch', 'usersBatchModel valid');
110+
moesifController.updateUsersBatch(usersBatch, cb);
87111
};
88112

89-
moesifMiddleware.updateCompany = function (companyModel, cb) {
113+
moesifMiddleware.updateCompany = function(companyModel, cb) {
114+
const company = new CompanyModel(companyModel);
90115
logMessage(options.debug, 'updateCompany', 'companyModel=' + JSON.stringify(companyModel));
91-
ensureValidCompanyModel(companyModel);
116+
ensureValidCompanyModel(company);
92117
logMessage(options.debug, 'updateCompany', 'companyModel valid');
93-
moesifController.updateCompany(companyModel, cb);
118+
moesifController.updateCompany(company, cb);
119+
}
120+
121+
moesifMiddleware.updateCompaniesBatch = function(companiesBatchModel, cb) {
122+
companiesBatch = [];
123+
for (let companyModel of companiesBatchModel) {
124+
companiesBatch.push(new CompanyModel(companyModel));
125+
}
126+
logMessage(options.debug, 'updateCompaniesBatch', 'companiesBatchModel=' + JSON.stringify(companiesBatchModel));
127+
ensureValidCompaniesBatchModel(companiesBatch);
128+
logMessage(options.debug, 'updateCompaniesBatch', 'companiesBatchModel valid');
129+
moesifController.updateCompaniesBatch(companiesBatch, cb);
94130
};
95131

96132
logMessage(options.debug, 'moesifInitiator', 'returning moesifMiddleware Function');
@@ -99,10 +135,6 @@ module.exports = function (options, handler) {
99135

100136
function mapResponseHeaders(event, context, result) {
101137
const headers = result.headers || {}; // NOTE: Mutating event.headers; prefer deep clone of event.headers
102-
103-
headers['x-amzn-trace-id'] = context.awsRequestId;
104-
headers['x-amzn-function-name'] = context.functionName;
105-
headers['x-apigateway-trace-id'] = (event && event.requestContext && event.requestContext.requestId) || (context && context.requestContext && context.requestContext.requestId);
106138
return headers;
107139
}
108140

@@ -114,44 +146,45 @@ function logEvent(event, context, err, result, options, moesifController) {
114146
return;
115147
}
116148

149+
var logData = {};
150+
logData.request = {};
151+
logData.response = {};
152+
logData.request.time = event && event.requestContext && event.requestContext.requestTimeEpoch ?
153+
new Date(event && event.requestContext && event.requestContext.requestTimeEpoch) :
154+
startTime;
155+
117156
logData.request.uri = getPathWithQueryStringParams(event);
118157
logData.request.verb = event.httpMethod;
119158
logData.request.apiVerion = options.getApiVersion(event, context);
120159
logData.request.ipAddress = requestIp.getClientIp(event) || (event.requestContext && event.requestContext.identity && event.requestContext.identity.sourceIp);
121160
logData.request.headers = event.headers || {};
161+
logData.metadata = options.getMetadata(event, context);
122162

123-
if (event.body) {
163+
if (options.logBody && event.body) {
124164
if (event.isBase64Encoded) {
125-
logData.request.transferEncoding = 'base64';
126-
logData.request.body = bodyToBase64(event.body);
165+
logData.request.body = event.body;
166+
logData.request.transferEncoding = 'base64';
127167
} else {
128-
try {
129-
logData.request.body = JSON.parse(event.body);
130-
} catch (err) {
131-
logData.request.body = event.body;
132-
}
168+
const bodyWrapper = safeJsonParse(event.body);
169+
logData.request.body = bodyWrapper.body
170+
logData.request.transferEncoding = bodyWrapper.transferEncoding
133171
}
134172
}
135173

136174
logMessage(options.debug, 'logEvent', 'created request: \n' + JSON.stringify(logData.request));
137175
var safeRes = result || {};
138-
logData.response.time = Date.now();
176+
logData.response.time = new Date(Math.max(logData.request.time.getTime(), Date.now()));
139177
logData.response.status = safeRes.statusCode ? parseInt(safeRes.statusCode) : 599;
140178
logData.response.headers = mapResponseHeaders(event, context, safeRes);
141179

142-
if (safeRes.body) {
180+
if (options.logBody && safeRes.body) {
143181
if (safeRes.isBase64Encoded) {
144-
// does this flag exists from AWS?
145-
logData.response.transferEncoding = 'base64';
146-
logData.response.body = bodyToBase64(safeRes.body);
182+
logData.response.body = safeRes.body;
183+
logData.response.transferEncoding = 'base64';
147184
} else {
148-
try {
149-
logData.response.body = JSON.parse(safeRes.body);
150-
} catch (err) {
151-
// if JSON decode fails, we'll try to base64 encode the body.
152-
logData.response.transferEncoding = 'base64';
153-
logData.response.body = bodyToBase64(safeRes.body);
154-
}
185+
const bodyWrapper = safeJsonParse(safeRes.body);
186+
logData.response.body = bodyWrapper.body
187+
logData.response.transferEncoding = bodyWrapper.transferEncoding
155188
}
156189
}
157190

@@ -169,7 +202,7 @@ function logEvent(event, context, err, result, options, moesifController) {
169202
ensureValidLogData(logData);
170203

171204
// This is fire and forget, we don't want logging to hold up the request so don't wait for the callback
172-
if (!options.skip(event, context)) {
205+
if (!options.skip(event, context) && moesifConfigManager.shouldSend(logData && logData.userId, logData && logData.companyId)) {
173206
logMessage(options.debug, 'logEvent', 'sending data invoking moesifAPI');
174207

175208
moesifController.createEvent(new EventModel(logData), function(err) {
@@ -203,8 +236,48 @@ function bodyToBase64(body) {
203236
}
204237
}
205238

239+
function safeJsonParse(body) {
240+
try {
241+
if (!Buffer.isBuffer(body) &&
242+
(typeof body === 'object' || Array.isArray(body))) {
243+
return {
244+
body: body,
245+
transferEncoding: undefined
246+
}
247+
}
248+
return {
249+
body: JSON.parse(body.toString()),
250+
transferEncoding: undefined
251+
}
252+
} catch (e) {
253+
return {
254+
body: bodyToBase64(body),
255+
transferEncoding: 'base64'
256+
}
257+
}
258+
}
259+
260+
function bodyToBase64(body) {
261+
if (!body) {
262+
return body;
263+
}
264+
if (Buffer.isBuffer(body)) {
265+
return body.toString('base64');
266+
} else if (typeof body === 'string') {
267+
return Buffer.from(body).toString('base64');
268+
} else if (typeof body.toString === 'function') {
269+
return Buffer.from(body.toString()).toString('base64');
270+
} else {
271+
return '';
272+
}
273+
}
274+
206275
function getPathWithQueryStringParams(event) {
207-
return url.format({ pathname: event.path, query: event.queryStringParameters })
276+
try {
277+
return url.format({ pathname: event.path, query: event.queryStringParameters });
278+
} catch (err) {
279+
return '/';
280+
}
208281
}
209282

210283
function ensureValidOptions(options) {
@@ -213,14 +286,20 @@ function ensureValidOptions(options) {
213286
if (options.identifyUser && !_.isFunction(options.identifyUser)) {
214287
throw new Error('identifyUser should be a function');
215288
}
289+
if (options.identifyCompany && !_.isFunction(options.identifyCompany)) {
290+
throw new Error('identifyCompany should be a function');
291+
}
216292
if (options.getSessionToken && !_.isFunction(options.getSessionToken)) {
217293
throw new Error('getSessionToken should be a function');
218294
}
295+
if (options.getMetadata && !_.isFunction(options.getMetadata)) {
296+
throw new Error('getMetadata should be a function');
297+
}
219298
if (options.getTags && !_.isFunction(options.getTags)) {
220299
throw new Error('getTags should be a function');
221300
}
222301
if (options.getApiVersion && !_.isFunction(options.getApiVersion)) {
223-
throw new Error('identifyUser should be a function');
302+
throw new Error('getApiVersion should be a function');
224303
}
225304
if (options.maskContent && !_.isFunction(options.maskContent)) {
226305
throw new Error('maskContent should be a function');
@@ -249,9 +328,6 @@ function ensureValidLogData(logData) {
249328
throw new Error('For Moesif events, request and response objects are required. Please check your maskContent function do not remove this');
250329
}
251330
else {
252-
// if (!logData.response.body) {
253-
// throw new Error('for log events, response body objects is required but can be empty object');
254-
// }
255331
if (!logData.request.time) {
256332
throw new Error('For Moesif events, response time is required. The middleware should populate it automatically. Please check your maskContent function do not remove this');
257333
}

‎lib/moesifConfigManager.js

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
* MoesifConfigManager is responsible for fetching and ensuring
3+
* the config for our api appId is up to date.
4+
*
5+
* This is done by ensuring the x-moesif-config-etag doesn't change.
6+
*/
7+
8+
var moesifController = require('moesifapi').ApiController;
9+
10+
const CONFIG_UPDATE_DELAY = 300000; // 5 minutes
11+
const HASH_HEADER = 'x-moesif-config-etag';
12+
13+
function now() {
14+
return new Date().getTime();
15+
}
16+
17+
function MoesifConfigManager() {
18+
this._lastConfigUpdate = 0;
19+
}
20+
21+
MoesifConfigManager.prototype.hasConfig = function () {
22+
return Boolean(this._config);
23+
};
24+
25+
MoesifConfigManager.prototype.shouldFetchConfig = function () {
26+
// wait to reload the config, since different collector instances
27+
// might have different versions of the config
28+
return !this._config || (
29+
this._lastSeenHash !== this._configHash &&
30+
now() - this._lastConfigUpdate > CONFIG_UPDATE_DELAY
31+
);
32+
};
33+
34+
MoesifConfigManager.prototype.tryGetConfig = function () {
35+
if (!this._loadingConfig && this.shouldFetchConfig()) {
36+
// only send one config request at a time
37+
this._loadingConfig = true;
38+
39+
var that = this;
40+
41+
moesifController.getAppConfig(function (_, __, event) {
42+
that._loadingConfig = false;
43+
44+
if (event && event.response.statusCode === 200) {
45+
that._configHash = event.response.headers[HASH_HEADER];
46+
try {
47+
that._config = JSON.parse(event.response.body);
48+
that._lastConfigUpdate = now();
49+
} catch (e) {
50+
console.warn('moesif: error parsing config');
51+
}
52+
}
53+
});
54+
}
55+
};
56+
57+
MoesifConfigManager.prototype._getSampleRate = function (userId, companyId) {
58+
if (!this._config) return 100;
59+
60+
if (userId && this._config.user_sample_rate && typeof this._config.user_sample_rate[userId] === 'number') {
61+
return this._config.user_sample_rate[userId];
62+
}
63+
64+
if (companyId && this._config.company_sample_rate && typeof this._config.company_sample_rate[companyId] === 'number') {
65+
return this._config.company_sample_rate[companyId];
66+
}
67+
68+
return (typeof this._config.sample_rate === 'number') ? this._config.sample_rate : 100;
69+
}
70+
71+
72+
MoesifConfigManager.prototype.shouldSend = function (userId, companyId) {
73+
const random = Math.random() * 100;
74+
return random <= this._getSampleRate(userId, companyId);
75+
};
76+
77+
MoesifConfigManager.prototype.tryUpdateHash = function (response) {
78+
if (response && response.headers && response.headers[HASH_HEADER]) {
79+
this._lastSeenHash = response.headers[HASH_HEADER];
80+
}
81+
};
82+
83+
module.exports = new MoesifConfigManager();

0 commit comments

Comments
 (0)