feat: remove RabbitMQ and use Kafka

- move actual handles from RabbitMQ to Kafka
- removed unused RabbitMQ handles code
- removed all RabbitMQ events firing
- added support for graceful shutdown of index-kafka.js
- removed RabbitMQ from production and local deployment
- fix unit tests to use a single ES service with index refresh fix
- removed redundant seedElasticsearchIndex.js script (we have a new one "npm run data:import")

Author: maxceem

.circleci/config.yml

@@ -48,7 +48,6 @@ jobs:
         - POSTGRES_USER: circle_test
         - POSTGRES_DB: circle_test
       - image: elasticsearch:2.3
-      - image: rabbitmq:3-management
     environment:
       DEPLOY_ENV: "DEV"
       LOGICAL_ENV: "dev"

.ebextensions/01-environment-variables.config

@@ -23,9 +23,6 @@ option_settings:
   - namespace: aws:elasticbeanstalk:application:environment
     option_name: DB_MASTER_URL
     value: TBD
-  - namespace: aws:elasticbeanstalk:application:environment
-    option_name: RABBITMQ_URL
-    value: TBD
   - namespace: aws:elasticbeanstalk:application:environment
     option_name: PROJECTS_ES_URL
     value: TBD

.prettierrc

@@ -0,0 +1,4 @@
+{
+  "singleQuote": true,
+  "trailingComma": "all"
+}

README.md

@@ -9,8 +9,8 @@ Microservice to manage CRUD operations for all things Projects.
   - [Steps to run locally](#steps-to-run-locally)
 - [Run Connect App with Project Service locally](#run-connect-app-with-project-service-locally)
 - [Import and Export data](#import-and-export-data)
-  - [📤 Export data](#%f0%9f%93%a4-export-data)
-  - [📥 Import data](#%f0%9f%93%a5-import-data)
+  - [📤 Export data](#-export-data)
+  - [📥 Import data](#-import-data)
 - [Import metadata from api.topcoder-dev.com (deprecated)](#import-metadata-from-apitopcoder-devcom-deprecated)
 - [Run via Docker](#run-via-docker)
 - [NPM Commands](#npm-commands)
@@ -54,7 +54,6 @@ Local setup should work good on **Linux**, **macOS** and **Windows**.
        # Locally deployed services (via docker-compose)
        PROJECTS_ES_URL=dockerhost:9200
        DB_MASTER_URL=postgres://coder:mysecretpassword@dockerhost:5432/projectsdb
-       RABBITMQ_URL=amqp://dockerhost:5672
        BUS_API_URL=http://dockerhost:8002/v5
 
        # Locally we usually run in Development mode
@@ -110,7 +109,6 @@ Local setup should work good on **Linux**, **macOS** and **Windows**.
       |----------|:-----:|:----:|
       | PostgreSQL | db | 5432 |
       | Elasticsearch | esearch | 9200 |
-      | RabbitMQ | queue | 5672, 15672  |
       | Mock Service (not in use) | jsonserver | 3001  |
       | Zookeeper | zookeeper | 2181  |
       | Kafka | kafka | 9092  |
@@ -148,7 +146,6 @@ Local setup should work good on **Linux**, **macOS** and **Windows**.
          |----------|:-----:|:----:|
          | PostgreSQL | db | 5432 |
          | Elasticsearch | esearch | 9200 |
-         | RabbitMQ | queue | 5672, 15672  |
          | Mock Service (not in use) | jsonserver | 3001  |
 
          </details>

config/custom-environment-variables.json

@@ -17,7 +17,6 @@
     "metadataDocType": "METADATA_ES_DOC_TYPE",
     "metadataDocDefaultId": "METADATA_ES_DOC_DEFAULT_ID"
   },
-  "rabbitmqURL": "RABBITMQ_URL",
   "pubsubQueueName": "PUBSUB_QUEUE_NAME",
   "pubsubExchangeName": "PUBSUB_EXCHANGE_NAME",
   "directProjectServiceEndpoint": "DIRECT_PROJECT_SERVICE_ENDPOINT",

config/default.json

@@ -6,7 +6,6 @@
   "captureLogs": "false",
   "enableFileUpload": "true",
   "logentriesToken": "",
-  "rabbitmqURL": "",
   "pubsubQueueName": "project.service",
   "pubsubExchangeName": "projects",
   "fileServiceEndpoint": "",

config/m2m.local.js

@@ -12,7 +12,6 @@ if (process.env.NODE_ENV === 'test') {
     logLevel: 'debug',
     captureLogs: 'false',
     logentriesToken: '',
-    rabbitmqURL: 'amqp://dockerhost:5672',
     fileServiceEndpoint: 'https://api.topcoder-dev.com/v3/files/',
     directProjectServiceEndpoint: 'https://api.topcoder-dev.com/v3/direct',
     connectProjectsUrl: 'https://connect.topcoder-dev.com/projects/',

config/mock.local.js

@@ -12,7 +12,6 @@ if (process.env.NODE_ENV === 'test') {
     logLevel: 'debug',
     captureLogs: 'false',
     logentriesToken: '',
-    rabbitmqURL: 'amqp://dockerhost:5672',
     fileServiceEndpoint: 'https://api.topcoder-dev.com/v3/files/',
     directProjectServiceEndpoint: 'https://api.topcoder-dev.com/v3/direct',
     connectProjectsUrl: 'https://connect.topcoder-dev.com/projects/',

config/test.json

@@ -15,7 +15,6 @@
       "metadataIndexName": "metadata_test",
       "metadataDocType": "doc"
     },
-    "rabbitmqUrl": "amqp://localhost:5672",
     "connectProjectsUrl": "https://local.topcoder-dev.com/projects/",
     "dbConfig": {
         "masterUrl": "postgres://coder:mysecretpassword@localhost:5432/projectsdb_test",

deploy.sh

@@ -136,10 +136,6 @@ make_task_def(){
           "name": "PROJECTS_ES_INDEX_NAME",
           "value": "%s"
         },
-        {
-          "name": "RABBITMQ_URL",
-          "value": "%s"
-        },
         {
           "name": "DIRECT_PROJECT_SERVICE_ENDPOINT",
           "value": "%s"
@@ -250,7 +246,6 @@ make_task_def(){
   LOG_LEVEL=$(eval "echo \$${ENV}_LOG_LEVEL")
   PROJECTS_ES_URL=$(eval "echo \$${ENV}_PROJECTS_ES_URL")
   PROJECTS_ES_INDEX_NAME=$(eval "echo \$${ENV}_PROJECTS_ES_INDEX_NAME")
-  RABBITMQ_URL=$(eval "echo \$${ENV}_RABBITMQ_URL")
   DIRECT_PROJECT_SERVICE_ENDPOINT=$(eval "echo \$${ENV}_DIRECT_PROJECT_SERVICE_ENDPOINT")
   FILE_SERVICE_ENDPOINT=$(eval "echo \$${ENV}_FILE_SERVICE_ENDPOINT")
   CONNECT_PROJECTS_URL=$(eval "echo \$${ENV}_CONNECT_PROJECTS_URL")
@@ -283,7 +278,7 @@ make_task_def(){
   INVITE_EMAIL_SUBJECT=$(eval "echo \$${ENV}_INVITE_EMAIL_SUBJECT")
   INVITE_EMAIL_SECTION_TITLE=$(eval "echo \$${ENV}_INVITE_EMAIL_SECTION_TITLE")
 
-  task_def=$(printf "$task_template" $1 $AWS_ACCOUNT_ID $AWS_ACCOUNT_ID $AWS_ECS_CONTAINER_NAME $AWS_ACCOUNT_ID $AWS_REGION $AWS_REPOSITORY $CIRCLE_SHA1 $2 $3 $4 $NODE_ENV $ENABLE_FILE_UPLOAD $LOG_LEVEL $CAPTURE_LOGS $LOGENTRIES_TOKEN $API_VERSION $AWS_REGION $AUTH_DOMAIN $AUTH_SECRET $VALID_ISSUERS $DB_MASTER_URL $MEMBER_SERVICE_ENDPOINT $IDENTITY_SERVICE_ENDPOINT $BUS_API_URL $MESSAGE_SERVICE_URL $SYSTEM_USER_CLIENT_ID $SYSTEM_USER_CLIENT_SECRET $PROJECTS_ES_URL $PROJECTS_ES_INDEX_NAME $RABBITMQ_URL $DIRECT_PROJECT_SERVICE_ENDPOINT $FILE_SERVICE_ENDPOINT $CONNECT_PROJECTS_URL $CONNECT_URL $ACCOUNTS_APP_URL $SEGMENT_ANALYTICS_KEY "$AUTH0_URL" "$AUTH0_AUDIENCE" $AUTH0_CLIENT_ID "$AUTH0_CLIENT_SECRET" $TOKEN_CACHE_TIME "$KAFKA_CLIENT_CERT" "$KAFKA_CLIENT_CERT_KEY" $KAFKA_GROUP_ID $KAFKA_URL "$AUTH0_PROXY_SERVER_URL" "$EMAIL_INVITE_FROM_NAME" "$EMAIL_INVITE_FROM_EMAIL" "$INVITE_EMAIL_SUBJECT" "$INVITE_EMAIL_SECTION_TITLE" $PORT $PORT $AWS_ECS_CLUSTER $AWS_REGION $NODE_ENV)
+  task_def=$(printf "$task_template" $1 $AWS_ACCOUNT_ID $AWS_ACCOUNT_ID $AWS_ECS_CONTAINER_NAME $AWS_ACCOUNT_ID $AWS_REGION $AWS_REPOSITORY $CIRCLE_SHA1 $2 $3 $4 $NODE_ENV $ENABLE_FILE_UPLOAD $LOG_LEVEL $CAPTURE_LOGS $LOGENTRIES_TOKEN $API_VERSION $AWS_REGION $AUTH_DOMAIN $AUTH_SECRET $VALID_ISSUERS $DB_MASTER_URL $MEMBER_SERVICE_ENDPOINT $IDENTITY_SERVICE_ENDPOINT $BUS_API_URL $MESSAGE_SERVICE_URL $SYSTEM_USER_CLIENT_ID $SYSTEM_USER_CLIENT_SECRET $PROJECTS_ES_URL $PROJECTS_ES_INDEX_NAME $DIRECT_PROJECT_SERVICE_ENDPOINT $FILE_SERVICE_ENDPOINT $CONNECT_PROJECTS_URL $CONNECT_URL $ACCOUNTS_APP_URL $SEGMENT_ANALYTICS_KEY "$AUTH0_URL" "$AUTH0_AUDIENCE" $AUTH0_CLIENT_ID "$AUTH0_CLIENT_SECRET" $TOKEN_CACHE_TIME "$KAFKA_CLIENT_CERT" "$KAFKA_CLIENT_CERT_KEY" $KAFKA_GROUP_ID $KAFKA_URL "$AUTH0_PROXY_SERVER_URL" "$EMAIL_INVITE_FROM_NAME" "$EMAIL_INVITE_FROM_EMAIL" "$INVITE_EMAIL_SUBJECT" "$INVITE_EMAIL_SECTION_TITLE" $PORT $PORT $AWS_ECS_CLUSTER $AWS_REGION $NODE_ENV)
 }
 
 push_ecr_image(){

local/full/docker-compose.yml

@@ -17,11 +17,6 @@ services:
     ports:
       - "9200:9200"
       - "9300:9300"
-  queue:
-    image: "rabbitmq:3-management"
-    ports:
-      - "5672:5672"
-      - "15672:15672"
   zookeeper:
     image: wurstmeister/zookeeper
     ports: