Skip to content

Commit f4ac768

Browse files
author
vikasrohit
authored
Merge pull request #75 from topcoder-platform/feature/multi-service-deployment
Multi service deployment to avoid silent failures of kafka consumers
2 parents fff088c + 013abf1 commit f4ac768

File tree

7 files changed

+72
-40
lines changed

7 files changed

+72
-40
lines changed

‎.circleci/config.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ workflows:
8282
- "build-dev":
8383
filters:
8484
branches:
85-
only: [dev, architectt1-feature/emailTemplatesClean]
85+
only: [dev]
8686
- "build-prod":
8787
filters:
8888
branches:

‎connect/connectNotificationServer.js

+8-8
Original file line numberDiff line numberDiff line change
@@ -361,13 +361,13 @@ if (config.ENABLE_EMAILS) {
361361
}
362362

363363
// init database, it will clear and re-create all tables
364-
notificationServer
365-
.initDatabase()
366-
.then(() => notificationServer.start())
367-
.catch((e) => {
368-
console.log(e); // eslint-disable-line no-console
369-
notificationServer.logger.error('Notification server errored out');
370-
});
364+
// notificationServer
365+
// .initDatabase()
366+
// .then(() => notificationServer.startKafkaConsumers())
367+
// .catch((e) => {
368+
// console.log(e); // eslint-disable-line no-console
369+
// notificationServer.logger.error('Notification server errored out');
370+
// });
371371

372372
// if no need to init database, then directly start the server:
373-
// notificationServer.start();
373+
notificationServer.startKafkaConsumers();

‎deploy.sh

+20-12
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ AWS_SECRET_ACCESS_KEY=$(eval "echo \$${ENV}_AWS_SECRET_ACCESS_KEY")
2929
AWS_ACCOUNT_ID=$(eval "echo \$${ENV}_AWS_ACCOUNT_ID")
3030
AWS_REPOSITORY=$(eval "echo \$${ENV}_AWS_REPOSITORY")
3131
AWS_ECS_CLUSTER=$(eval "echo \$${ENV}_AWS_ECS_CLUSTER")
32-
AWS_ECS_SERVICE=$(eval "echo \$${ENV}_AWS_ECS_SERVICE")
32+
AWS_ECS_SERVICE_API=$(eval "echo \$${ENV}_AWS_ECS_SERVICE")
33+
AWS_ECS_SERVICE_CONSUMERS=$(eval "echo \$${ENV}_AWS_ECS_SERVICE_CONSUMERS")
3334

3435
KAFKA_CLIENT_CERT=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT")
3536
KAFKA_CLIENT_CERT_KEY=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT_KEY")
@@ -98,9 +99,9 @@ deploy_cluster() {
9899

99100
#family="nginx-api-dev-task"
100101

101-
make_task_def
102-
register_definition
103-
update_result=$(aws ecs update-service --cluster $AWS_ECS_CLUSTER --service $AWS_ECS_SERVICE --task-definition $revision )
102+
make_task_def $1 $2 $3 $4
103+
register_definition $1
104+
update_result=$(aws ecs update-service --cluster $AWS_ECS_CLUSTER --service $1 --task-definition $revision )
104105
#echo $update_result
105106
result=$(echo $update_result | $JQ '.service.taskDefinition' )
106107
echo $result
@@ -119,8 +120,9 @@ make_task_def(){
119120
"name": "%s",
120121
"image": "%s.dkr.ecr.%s.amazonaws.com/%s:%s",
121122
"essential": true,
122-
"memory": 1536,
123-
"cpu": 768,
123+
"memory": 768,
124+
"cpu": 512,
125+
"entryPoint": ["%s", "%s", "%s"],
124126
"environment": [
125127
{
126128
"name": "ENV",
@@ -261,11 +263,11 @@ make_task_def(){
261263
}
262264
]'
263265

264-
task_def=$(printf "$task_template" $AWS_ECS_CONTAINER_NAME $AWS_ACCOUNT_ID $AWS_REGION $AWS_REPOSITORY $TAG $ENV "$KAFKA_CLIENT_CERT" "$KAFKA_CLIENT_CERT_KEY" $KAFKA_GROUP_ID $KAFKA_URL $DATABASE_URL $AUTHSECRET $TC_API_BASE_URL $TC_API_V3_BASE_URL $TC_API_V4_BASE_URL $TC_API_V5_BASE_URL $MESSAGE_API_BASE_URL $CONNECT_URL $ENABLE_EMAILS $MENTION_EMAIL $REPLY_EMAIL_PREFIX $REPLY_EMAIL_DOMAIN $REPLY_EMAIL_FROM $DEFAULT_REPLY_EMAIL $ENABLE_DEV_MODE $DEV_MODE_EMAIL $LOG_LEVEL $VALID_ISSUERS $PORT "$API_CONTEXT_PATH" "$AUTH0_URL" "$AUTH0_AUDIENCE" $AUTH0_CLIENT_ID "$AUTH0_CLIENT_SECRET" $TOKEN_CACHE_TIME $AWS_ECS_CLUSTER $AWS_REGION $AWS_ECS_CLUSTER $ENV)
266+
task_def=$(printf "$task_template" $1 $AWS_ACCOUNT_ID $AWS_REGION $AWS_REPOSITORY $TAG $2 $3 $4 $ENV "$KAFKA_CLIENT_CERT" "$KAFKA_CLIENT_CERT_KEY" $KAFKA_GROUP_ID $KAFKA_URL $DATABASE_URL $AUTHSECRET $TC_API_BASE_URL $TC_API_V3_BASE_URL $TC_API_V4_BASE_URL $TC_API_V5_BASE_URL $MESSAGE_API_BASE_URL $CONNECT_URL $ENABLE_EMAILS $MENTION_EMAIL $REPLY_EMAIL_PREFIX $REPLY_EMAIL_DOMAIN $REPLY_EMAIL_FROM $DEFAULT_REPLY_EMAIL $ENABLE_DEV_MODE $DEV_MODE_EMAIL $LOG_LEVEL $VALID_ISSUERS $PORT "$API_CONTEXT_PATH" "$AUTH0_URL" "$AUTH0_AUDIENCE" $AUTH0_CLIENT_ID "$AUTH0_CLIENT_SECRET" $TOKEN_CACHE_TIME $AWS_ECS_CLUSTER $AWS_REGION $AWS_ECS_CLUSTER $ENV)
265267
}
266268

267269
register_definition() {
268-
if revision=$(aws ecs register-task-definition --container-definitions "$task_def" --family $family | $JQ '.taskDefinition.taskDefinitionArn'); then
270+
if revision=$(aws ecs register-task-definition --container-definitions "$task_def" --family $1 2> /dev/null | $JQ '.taskDefinition.taskDefinitionArn'); then
269271
echo "Revision: $revision"
270272
else
271273
echo "Failed to register task definition"
@@ -277,13 +279,13 @@ register_definition() {
277279
check_service_status() {
278280
counter=0
279281
sleep 60
280-
servicestatus=`aws ecs describe-services --service $AWS_ECS_SERVICE --cluster $AWS_ECS_CLUSTER | $JQ '.services[].events[0].message'`
282+
servicestatus=`aws ecs describe-services --service $1 --cluster $AWS_ECS_CLUSTER | $JQ '.services[].events[0].message'`
281283
while [[ $servicestatus != *"steady state"* ]]
282284
do
283285
echo "Current event message : $servicestatus"
284286
echo "Waiting for 30 seconds to check the service status...."
285287
sleep 30
286-
servicestatus=`aws ecs describe-services --service $AWS_ECS_SERVICE --cluster $AWS_ECS_CLUSTER | $JQ '.services[].events[0].message'`
288+
servicestatus=`aws ecs describe-services --service $1 --cluster $AWS_ECS_CLUSTER | $JQ '.services[].events[0].message'`
287289
counter=`expr $counter + 1`
288290
if [[ $counter -gt $COUNTER_LIMIT ]] ; then
289291
echo "Service does not reach steady state within 10 minutes. Please check"
@@ -295,5 +297,11 @@ check_service_status() {
295297

296298
configure_aws_cli
297299
push_ecr_image
298-
deploy_cluster
299-
check_service_status
300+
301+
deploy_cluster $AWS_ECS_SERVICE_API "npm" "run" "startAPI"
302+
303+
deploy_cluster $AWS_ECS_SERVICE_CONSUMERS "npm" "run" "start"
304+
305+
check_service_status $AWS_ECS_SERVICE_API
306+
307+
check_service_status $AWS_ECS_SERVICE_CONSUMERS

‎index-api.js

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
const notificationServer = require('./index');
2+
3+
// // init database, it will clear and re-create all tables
4+
// notificationServer
5+
// .initDatabase()
6+
// .then(() => notificationServer.startAPI())
7+
// .catch((e) => {
8+
// console.log(e); // eslint-disable-line no-console
9+
// notificationServer.logger.error('Notification API server errored out');
10+
// });
11+
12+
module.exports = notificationServer.startAPI();

‎index.js

+14-4
Original file line numberDiff line numberDiff line change
@@ -95,15 +95,24 @@ function getAllHandlers() {
9595
}
9696

9797
/**
98-
* Start the notification server.
98+
* Start the notification API server.
9999
*/
100-
function start() {
100+
function startAPI() {
101+
// load app only after config is set
102+
const app = require('./src/app');
103+
app.start();
104+
}
105+
106+
/**
107+
* Start the event bus consumer.
108+
*/
109+
function startKafkaConsumers() {
101110
if (_.isEmpty(handlers)) {
102111
throw new errors.ValidationError('Missing handler(s).');
103112
}
104113
// load app only after config is set
105114
const app = require('./src/app');
106-
app.start(handlers, notificationServiceHandlers);
115+
app.startKafkaConsumer(handlers, notificationServiceHandlers);
107116
}
108117

109118
/**
@@ -122,7 +131,8 @@ module.exports = {
122131
addTopicHandler,
123132
removeTopicHandler,
124133
getAllHandlers,
125-
start,
134+
startAPI,
135+
startKafkaConsumers,
126136
initDatabase,
127137
addNotificationServiceHandler,
128138

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
"main": "./index.js",
66
"scripts": {
77
"start": "node connect/connectNotificationServer",
8+
"startAPI": "node index-api",
89
"lint": "eslint *.js src config test connect || true",
910
"lint:fix": "eslint *.js --fix src config test connect || true",
1011
"postinstall": "npm run build",

‎src/app.js

+16-15
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ const models = require('./models');
1717
const Kafka = require('no-kafka');
1818

1919
/**
20-
* Start Kafka consumer.
20+
* Start Kafka consumer for event bus events.
2121
* @param {Object} handlers the handlers
2222
* @param {Array} notificationServiceHandlers list of notification service handlers
2323
*/
@@ -74,6 +74,7 @@ function startKafkaConsumer(handlers, notificationServiceHandlers) {
7474
});
7575
});
7676

77+
7778
consumer
7879
.init()
7980
.then(() => _.each(_.keys(handlers),
@@ -85,11 +86,9 @@ function startKafkaConsumer(handlers, notificationServiceHandlers) {
8586
}
8687

8788
/**
88-
* Start the notification server.
89-
* @param {Object} handlers the handlers
90-
* @param {Array} notificationServiceHandlers list of notification service handlers
89+
* Start the notifications API server.
9190
*/
92-
function start(handlers, notificationServiceHandlers) {
91+
function start() {
9392
const app = express();
9493
app.set('port', config.PORT);
9594

@@ -157,19 +156,21 @@ function start(handlers, notificationServiceHandlers) {
157156
}
158157
});
159158

160-
models
161-
.init()
162-
.then(() => {
163-
app.listen(app.get('port'), () => {
164-
logger.info(`Express server listening on port ${app.get('port')}`);
165-
});
166-
167-
startKafkaConsumer(handlers, notificationServiceHandlers);
168-
})
169-
.catch((err) => logger.error(err));
159+
// models
160+
// .init()
161+
// .then(() => {
162+
// app.listen(app.get('port'), () => {
163+
// logger.info(`Express server listening on port ${app.get('port')}`);
164+
// });
165+
// })
166+
// .catch((err) => logger.error(err));
167+
app.listen(app.get('port'), () => {
168+
logger.info(`Express server listening on port ${app.get('port')}`);
169+
});
170170
}
171171

172172
// Exports
173173
module.exports = {
174174
start,
175+
startKafkaConsumer,
175176
};

0 commit comments

Comments
 (0)