Merge pull request #2380 from noobaa/evgb-0.5.4

Fixes #2368: make sense in rpc_address in nodes_monitor after node disconnect

Author: guymguym

src/agent/agent.js

@@ -208,6 +208,7 @@ class Agent {
         this.is_started = false;
         // mark the rpc state as disconnected to close and reject incoming connections
         this.rpc.set_disconnected_state(true);
+        this.rpc_address = '';
         this._start_stop_server();
         // reset the n2n config to close any open ports
         this.n2n_agent.update_n2n_config({

src/server/node_services/nodes_monitor.js

@@ -440,12 +440,10 @@ class NodesMonitor extends EventEmitter {
             item.node.is_internal_node = true;
         }
         dbg.log0('_add_new_node', item.node);
+        this._set_need_update.add(item);
         this._add_node_to_maps(item);
         this._set_node_defaults(item);
         this._set_connection(item, conn);
-        this._set_need_update.add(item);
-        // we hurry the next run to save the new node
-        this._schedule_next_run(3000);
     }
 
     _add_node_to_maps(item) {
@@ -510,6 +508,8 @@ class NodesMonitor extends EventEmitter {
         dbg.warn('_close_node_connection', item.node.name, item.connection.connid);
         item.connection.close();
         item.connection = null;
+        item.agent_info = null;
+        item.node.rpc_address = '';
     }
 
     _disconnect_node(item) {
@@ -528,6 +528,7 @@ class NodesMonitor extends EventEmitter {
         item.node.heartbeat = Date.now();
         this._set_need_update.add(item);
         this._update_status(item);
+        this._run_node_delayed(item);
     }
 
     _on_connection_close(item, conn) {
@@ -611,6 +612,11 @@ class NodesMonitor extends EventEmitter {
             }));
     }
 
+    _run_node_delayed(item) {
+        return P.delay(100)
+            .then(() => this._run_node(item));
+    }
+
     _get_agent_info(item) {
         if (!item.connection) return;
         dbg.log0('_get_agent_info:', item.node.name);
@@ -754,6 +760,7 @@ class NodesMonitor extends EventEmitter {
 
     _test_network_to_server(item) {
         if (!item.connection) return;
+        if (!item.node.rpc_address) return;
 
         const start = Date.now();
 
@@ -783,6 +790,7 @@ class NodesMonitor extends EventEmitter {
     // Test with few other nodes and detect if we have a NAT preventing TCP to this node
     _test_network_perf(item) {
         if (!item.connection) return;
+        if (!item.node.rpc_address) return;
 
         const items_without_issues = this._get_detention_test_nodes(item, config.NODE_IO_DETENTION_TEST_NODES);
         return P.each(items_without_issues, item_without_issues => {
@@ -1123,6 +1131,7 @@ class NodesMonitor extends EventEmitter {
             item.online &&
             item.trusted &&
             item.node_from_store &&
+            item.node.rpc_address &&
             !item.io_detention &&
             !item.node.migrating_to_pool &&
             !item.node.decommissioning &&
@@ -1134,6 +1143,7 @@ class NodesMonitor extends EventEmitter {
             item.online &&
             item.trusted &&
             item.node_from_store &&
+            item.node.rpc_address &&
             !item.io_detention &&
             !item.node.decommissioned && // but readable when decommissioning !
             !item.node.deleting &&
@@ -1143,6 +1153,7 @@ class NodesMonitor extends EventEmitter {
             item.online &&
             item.trusted &&
             item.node_from_store &&
+            item.node.rpc_address &&
             !item.io_detention &&
             !item.storage_full &&
             !item.node.migrating_to_pool &&