Re-Enable web server node cluster, and fix FD inheritance issue for listeners

Author: nimrod-becker

src/deploy/NVA_build/noobaa_supervisor.conf

@@ -17,6 +17,7 @@ autorestart=true
 stopsignal=KILL
 killasgroup=true
 stopasgroup=true
+autorestart=true
 directory=/root/node_modules/noobaa-core
 command=/usr/local/bin/node src/server/web_server.js
 

src/deploy/NVA_build/supervisord.orig

@@ -21,7 +21,13 @@ start() {
 
 stop() {
     echo "Stopping ..."
-    kill -9 `cat $PIDFILE`
+    local p=$(cat $PIDFILE)
+    kill -9 ${p}
+    #unlink /tmp/supervisor.sock
+    #local pid=$(lsof -i :8080|awk  '{ print $2 }' | sed '1 d')
+    #for p in ${pid} ; do
+      #kill -9 ${p}
+    #done
     [ $? -eq 0 ] && rm -f $PIDFILE
     return $retval
 }
@@ -35,6 +41,7 @@ case $1 in
     ;;
     restart)
         stop
+        sleep 1
         start
     ;;
     *)

src/deploy/NVA_build/upgrade.sh

@@ -11,6 +11,7 @@ VER_CHECK="/root/node_modules/noobaa-core/src/deploy/NVA_build/version_check.js"
 NEW_UPGRADE_SCRIPT="${EXTRACTION_PATH}noobaa-core/src/deploy/NVA_build/upgrade.sh"
 
 function disable_supervisord {
+  deploy_log "disable_supervisord"
   #services under supervisord
   local services=$($SUPERCTL status | grep pid | sed 's:.*pid \(.*\),.*:\1:')
   #disable the supervisord
@@ -22,9 +23,8 @@ function disable_supervisord {
 }
 
 function enable_supervisord {
+  deploy_log "enable_supervisord"
   ${SUPERD}
-
-
 }
 
 function restart_webserver {
@@ -114,18 +114,29 @@ function do_upgrade {
 
   deploy_log "Running post upgrade"
   ${WRAPPER_FILE_PATH}${WRAPPER_FILE_NAME} post
+  deploy_log "Finished post upgrade"
 
   enable_supervisord
+  deploy_log "Enabling supervisor"
   #workaround - from some reason, without sleep + restart, the server starts with odd behavior
   #TODO: understand why and fix.
   sleep 5;
   restart_s3rver
+  deploy_log "Restarted s3rver"
   restart_webserver
   deploy_log "Upgrade finished successfully!"
 }
 
 deploy_log "upgrade.sh called with $@"
 
+#Node.js Cluster chnages the .spawn behavour. On a normal spawn FDs are not inherited,
+#on a node cluster they are, which meand the listening ports of the webserver are inherited by this create_multipart_upload.
+#murder them
+fds=`lsof -p $$ | grep LISTEN | awk '{print $4}' | sed 's:\(.*\)u:\1:'`
+for f in ${fds}; do
+  exec ${f}<&-
+done
+
 if [ "$1" == "from_file" ]; then
   if [ "$2" != "" ]; then
     cp -f $2 ${TMP_PATH}${PACKAGE_FILE_NAME}

src/deploy/NVA_build/upgrade_wrapper.sh

@@ -156,7 +156,7 @@ function post_upgrade {
 
   #Installation ID generation if needed
   #TODO: Move this into the mongo_upgrade.js
-  local id=$(/mongodb/bin/mongo nbcore --eval "db.clusters.find().shellPrint()" | grep cluster_id | wc -l)
+  local id=$(/usr/bin/mongo nbcore --eval "db.clusters.find().shellPrint()" | grep cluster_id | wc -l)
   if [ ${id} -eq 0 ]; then
       id=$(uuidgen)
       /usr/bin/mongo nbcore --eval "db.clusters.insert({cluster_id: '${id}'})"
@@ -191,7 +191,6 @@ function post_upgrade {
   deploy_log "list core dir"
   deploy_log "$(ls -R ${CORE_DIR}/build/)"
 
-  /etc/rc.d/init.d/supervisord restart
   sudo grep noobaa /etc/sudoers
   if [ $? -ne 0 ]; then
       deploy_log "adding noobaa to sudoers"
@@ -203,8 +202,10 @@ function post_upgrade {
 
   fi
 
-
-  rm -f /tmp/*.tar.gz
+	rm -f /tmp/*.tar.gz
+	
+	/etc/rc.d/init.d/supervisord stop
+	/etc/rc.d/init.d/supervisord start
 }
 
 

src/server/web_server.js

@@ -43,19 +43,19 @@ dotenv.load();
 
 
 // Temporary removed - causes issues with upgrade.
-//var numCPUs = Math.ceil(require('os').cpus().length / 2);
-// if (cluster.isMaster) {
-//     // Fork MD Servers
-//     for (var i = 0; i < numCPUs; i++) {
-//         console.warn('Spawning MD Server', i + 1);
-//         cluster.fork();
-//     }
-//
-//     cluster.on('exit', function(worker, code, signal) {
-//         console.log('MD Server ' + worker.process.pid + ' died');
-//     });
-//     return;
-// }
+var numCPUs = Math.ceil(require('os').cpus().length / 2);
+if (cluster.isMaster) {
+    // Fork MD Servers
+    for (var i = 0; i < numCPUs; i++) {
+        console.warn('Spawning MD Server', i + 1);
+        cluster.fork();
+    }
+
+    cluster.on('exit', function(worker, code, signal) {
+        console.log('MD Server ' + worker.process.pid + ' died');
+    });
+    return;
+}
 
 dbg.set_process_name('WebServer');