Skip to content

Commit f360e6f

Browse files
author
Shiva Kayathi
committed
WIP APPS-26 -- x-notify add bull queue and Jobs
1 parent 42d8eaa commit f360e6f

10 files changed

Lines changed: 438 additions & 13 deletions

File tree

‎controllers/mailing.js‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ const dbConn = module.parent.parent.exports.dbConn;
1111
const ObjectId = require('mongodb').ObjectId;
1212

1313
const { Worker } = require('worker_threads');
14+
const { createJob } = require("../jobs/bullConfig");
15+
16+
console.log("mailing " + typeof createJob)
17+
console.log(createJob)
1418

1519
const _mailingState = {
1620
cancelled: "cancelled",
@@ -395,8 +399,18 @@ async function mailingUpdate( mailingId, newHistoryState, options ) {
395399
async function sendMailingToSubs ( mailingId, topicId, mailingSubject, mailingBody ) {
396400

397401
// When completed, change state to "sent"
398-
402+
399403
// Start the worker.
404+
workerData = {
405+
topicId: topicId,
406+
mailingBody: mailingBody,
407+
mailingSubject: mailingSubject,
408+
typeMailing: "msgUpdates",
409+
sentTo: "allSubs",
410+
dbConn: true //dbConn
411+
};
412+
createJob("getSubscribers", workerData);
413+
/*
400414
const worker = new Worker( './controllers/workerSendEmail.js', {
401415
workerData: {
402416
topicId: topicId,
@@ -408,6 +422,11 @@ async function sendMailingToSubs ( mailingId, topicId, mailingSubject, mailingBo
408422
}
409423
});
410424
425+
426+
427+
428+
429+
411430
worker.on('message', function(msg){
412431
413432
if ( msg.completed ) {
@@ -419,12 +438,14 @@ async function sendMailingToSubs ( mailingId, topicId, mailingSubject, mailingBo
419438
}
420439
421440
console.log( msg.msg );
441+
422442
});
423443
424444
worker.on('error', function(msg){
425445
console.log( "Send to subs - Worker ERRROR: " + msg );
426446
});
427-
447+
*/
448+
mailingUpdate( mailingId, _mailingState.sent, { historyState: _mailingState.sending } );
428449
}
429450

430451
// Simple worker to send mailing

‎controllers/mailing_view.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
const mustache = require('mustache');
1111
const fsPromises = require('fs').promises;
1212
const mailing = require('./mailing');
13+
//const subscription = require('./subscriptions');
14+
1315
const _mailingState = mailing.mailingState;
1416
const _baseRedirFolder = ( process.env.baseFolder || "" ) + "/api/v1/mailing/";
1517

‎controllers/subscriptions.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1103,7 +1103,8 @@ exports.simulateAddPost = async ( req, res, next ) => {
11031103
/**
11041104
* This is the future REST endpoint handler function for queuing a mailing with Notify
11051105
*/
1106-
/* It is commented until the mailing.js module is updated to use Bull to make the API call. Related to APPS-53 work.
1106+
//It is commented until the mailing.js module is updated to use Bull to make the API call. Related to APPS-53 work.
1107+
/*
11071108
exports.sendMailing = async ( req, res, next ) => {
11081109
const email = req.body.email,
11091110
templateId = req.body.templateId,
@@ -1126,8 +1127,8 @@ exports.sendMailing = async ( req, res, next ) => {
11261127
11271128
11281129
res.json( _successJSO );
1129-
}*/
1130-
1130+
}
1131+
*/
11311132
/**
11321133
* This is the function for queuing a subscriber confirmation email
11331134
* send via notify.

‎controllers/workerSendEmail.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,8 @@ async function init() {
108108
* Send the mailing
109109
*
110110
*/
111+
console.log("worker _notifyEndPoint " + _notifyEndPoint)
112+
console.log("notifyKey " + notifyKey);
111113
let notifyClient = new NotifyClient( _notifyEndPoint, notifyKey );
112114

113115
//console.log( "_notifyEndPoint: " + _notifyEndPoint );

‎helpers/sendEmail.js‎

Lines changed: 279 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
const chalk = require('chalk'); // To color message in console log
2+
3+
const MongoClient = require('mongodb').MongoClient;
4+
const NotifyClient = require('notifications-node-client').NotifyClient;
5+
6+
const ObjectId = require('mongodb').ObjectId;
7+
//const dbConn = module.parent.parent.exports.dbConn;
8+
9+
const processEnv = process.env,
10+
_notifyEndPoint = processEnv.notifyEndPoint || "https://api.notification.alpha.canada.ca",
11+
_unsubBaseURL = process.env.removeURL || "https://apps.canada.ca/x-notify/subs/remove/",
12+
_subsLinkSuffix = process.env.subsLinkSuffix || "853e0212b92a127"
13+
14+
15+
let dbConn, notifyKey;
16+
17+
MongoClient.connect( processEnv.MONGODB_URI || '', {useUnifiedTopology: true} ).then( ( mongoInstance ) => {
18+
19+
dbConn = mongoInstance.db( processEnv.MONGODB_NAME || 'subs' );
20+
21+
}).catch( (e) => { console.log( "%s Worker MongoDB ERRROR: %s", chalk.red('✗'), e ) } );
22+
23+
24+
exports.getSubscribers = async (job, done, createJob) => {
25+
console.log("job data -----> ")
26+
console.log(job)
27+
28+
// Ensure we have received all the data
29+
if ( !job.mailingBody || !job.mailingSubject ) {
30+
throw new Error( "Send Email: No email body" );
31+
}
32+
33+
if ( !job.topicId ) {
34+
throw new Error( "Send Email: No topicId selected" );
35+
}
36+
37+
/*
38+
* Get mailing notify information
39+
*
40+
*/
41+
console.log("module.parent.parent.exports.dbConn " + module.parent.parent.exports )
42+
let topic = await dbConn.collection( "topics" ).findOne(
43+
{ _id: job.topicId },
44+
{ projection: {
45+
nTemplateMailingId: 1,
46+
templateId: 1,
47+
notifyKey: 1,
48+
}
49+
} ).catch( (e) => {
50+
console.log( "sendEmail-getTopic" );
51+
console.log( e );
52+
throw new Error( "sendEmail: Can't find the topic: " + job.topicId );
53+
});
54+
55+
let templateId;
56+
notifyKey = topic.notifyKey;
57+
58+
59+
if ( !topic.nTemplateMailingId ) {
60+
throw new Error( "Worker: There is no mailing template associated with : " + topicId );
61+
}
62+
63+
console.log("topic.nTemplateMailingId " + topic.nTemplateMailingId);
64+
65+
// Get the correct notify email template
66+
if ( job.typeMailing === "msgUpdates" ) {
67+
templateId = topic.nTemplateMailingId;
68+
} else if ( job.typeMailing === "confirmSubs" ) {
69+
templateId = topic.templateId;
70+
} else {
71+
throw new Error( "Worker: Invalid type mailing, was : " + job.typeMailing );
72+
}
73+
74+
/*
75+
* Get list of confirmed subscribers
76+
*
77+
*/
78+
let listEmail = [];
79+
80+
if ( Array.isArray( job.sentTo ) ) {
81+
listEmail = sentTo;
82+
} else if ( job.sentTo === "allSubs" ) {
83+
listEmail = await getConfirmedSubscriberAsArray( job.topicId );
84+
}
85+
86+
// No subscribers
87+
if ( !listEmail.length ){
88+
console.log( "Worker: No subscriber" );
89+
90+
//parentPort.postMessage( { msg: "No subscriber" } );
91+
92+
}
93+
let emailData = {
94+
listEmail : listEmail,
95+
notifyKey : notifyKey,
96+
emailData : job,
97+
98+
}
99+
console.log("sendM+EMAil")
100+
console.log(typeof createJob);
101+
createJob("sendEmail", emailData);
102+
}
103+
104+
/*
105+
* Utilities function
106+
*/
107+
getConfirmedSubscriberAsArray = async ( topicId ) => {
108+
109+
// Get all the emails for the given topic
110+
let docs = await dbConn.collection( "subsConfirmed" ).find(
111+
{
112+
topicId: topicId
113+
},
114+
{
115+
projection: {
116+
email: 1,
117+
subscode: 1
118+
}
119+
}
120+
);
121+
122+
let docsItems = await docs.toArray();
123+
124+
return docsItems;
125+
};
126+
127+
exports.sendEmail = async (job, done) => {
128+
//console.log("emailListv " + job)
129+
console.log("sendEmail _notifyEndPoint " + _notifyEndPoint)
130+
console.log("notifyKey " + job.notifyKey);
131+
132+
133+
notifyClient = new NotifyClient( _notifyEndPoint, job.notifyKey );
134+
let listEmail = job.listEmail;
135+
let i, i_len = listEmail.length, i_cache;
136+
for( i = 0; i !== i_len; i++) {
137+
i_cache = listEmail[ i ];
138+
139+
const { email, subscode } = i_cache;
140+
141+
const userCodeUrl = ( subscode.id ? subscode.toHexString() : subscode );
142+
143+
//console.log( "Worker: Send for : " + email );
144+
145+
if ( !email ) {
146+
continue;
147+
}
148+
149+
//parentPort.postMessage( { msg: "Send for : " + email } );
150+
151+
152+
//console.log( "templateId: " + templateId );
153+
//console.log( "email: " + email );
154+
//console.log( "subject: " + mailingSubject );
155+
//console.log( "body: " + mailingBody );
156+
//console.log( "unsub_link: " + _unsubBaseURL + userCodeUrl + "/" + _subsLinkSuffix );
157+
//console.log( "reference: " + "x-notify_" + typeMailing );
158+
/*
159+
notifyClient.sendEmail( templateId, email,
160+
{
161+
personalisation: {
162+
body: mailingBody,
163+
subject: mailingSubject,
164+
unsub_link: _unsubBaseURL + userCodeUrl + "/" + _subsLinkSuffix
165+
},
166+
reference: "x-notify_" + typeMailing
167+
}).catch( ( e ) => {
168+
// Log the Notify errors
169+
// console.log( "Error in Notify" );
170+
// console.log( e );
171+
172+
//parentPort.postMessage( { msg: "worker-Error in Notify" } );
173+
174+
const currDate = new Date(),
175+
currDateTime = currDate.getTime(),
176+
errDetails = e.error.errors[0],
177+
statusCode = e.error.status_code,
178+
msg = errDetails.message;
179+
180+
181+
182+
if ( statusCode === 400 && msg.indexOf( "email_address" ) !== -1 ) {
183+
184+
//
185+
// We need to remove that user and log it
186+
//
187+
// Removal of bad email should be done after 25 min, same delay used to the not-before
188+
// The following task need to be quoeud and delayed. It could be addressed at the same time of APPS-26
189+
//dbConn.collection( "subsUnconfirmed" ).findOneAndDelete(
190+
// {
191+
// email: email
192+
// }
193+
//)
194+
//dbConn.collection( "subsExist" ).findOneAndDelete(
195+
// {
196+
// e: email
197+
// }
198+
//)
199+
200+
201+
// Log
202+
dbConn.collection( "notify_badEmail_logs" ).insertOne(
203+
{
204+
createdAt: currDate,
205+
code: userCodeUrl,
206+
email: email
207+
}
208+
).catch( (e2) => {
209+
console.log( "worker-sendNotifyConfirmEmail: notify_badEmail_logs: " + userCodeUrl );
210+
console.log( e2 );
211+
console.log( e );
212+
});
213+
214+
} else if ( statusCode === 429 ) {
215+
216+
//
217+
// This is a rate limit error, the system should notify us
218+
//
219+
dbConn.collection( "notify_tooManyReq_logs" ).insertOne(
220+
{
221+
createdAt: currDate,
222+
email: email,
223+
code: userCodeUrl,
224+
templateId: templateId,
225+
details: msg
226+
}
227+
).catch( (e2) => {
228+
console.log( "worker-sendNotifyConfirmEmail: notify_tooManyReq_logs: " + userCodeUrl );
229+
console.log( e2 );
230+
console.log( e );
231+
});
232+
233+
//
234+
// Try to email us (only with the predefined interval)
235+
//
236+
if ( _notifyUsNotBeforeTimeLimit <= currDateTime ) {
237+
238+
letUsKnow( "429 Too Many Request error", {
239+
type: "ratelimit",
240+
currTime: currDateTime,
241+
lastTime: _notifyUsNotBeforeTimeLimit
242+
},
243+
true );
244+
245+
// Readjust the limit for the next period
246+
_notifyUsNotBeforeTimeLimit = currDateTime + _notifyUsTimeLimit;
247+
248+
}
249+
250+
} else {
251+
252+
//
253+
// Any other kind of error - https://docs.notifications.service.gov.uk/node.html#send-an-email-error-codes
254+
//
255+
// notify_logs entry - this can be async
256+
dbConn.collection( "notify_logs" ).insertOne(
257+
{
258+
createdAt: currDate,
259+
templateId: templateId,
260+
e: errDetails.error,
261+
msg: msg,
262+
statusCode: statusCode,
263+
err: e.toString(),
264+
code: userCodeUrl
265+
}
266+
).catch( (e2) => {
267+
console.log( "worker-sendNotifyConfirmEmail: notify_logs: " + userCodeUrl );
268+
console.log( e2 );
269+
console.log( e );
270+
});
271+
272+
}
273+
274+
console.log( "worker-sendNotifyConfirmEmail: sendEmail " + userCodeUrl );
275+
//mailingUpdate( mailingId, _mailingState.sent, { historyState: _mailingState.sending } );
276+
}); */
277+
}
278+
}
279+

0 commit comments

Comments
 (0)