fix race condition in SupervisorProxyServer

CHANGES.md

@@ -1,5 +1,10 @@
 # Change Log
 
+## 0.18.7 (2025-01-07)
+
+* Fix a random Python crash due to race condition when restarting **Supvisors**.
+
+
 ## 0.18.6 (2024-08-20)
 
 * Completion of fix about process CPU statistics when using SOLARIS mode.

supvisors/internal_com/supervisorproxy.py

@@ -349,6 +349,8 @@ def __init__(self, supvisors):
         """ Initialization of the attributes. """
         self.supvisors = supvisors
         self.proxies: Dict[str, SupervisorProxyThread] = {}
+        # do not allow the creation of a new proxy when stop is requested
+        self.stop_event: threading.Event = threading.Event()
 
     @property
     def logger(self) -> Logger:
@@ -382,6 +384,9 @@ def get_proxy(self, identifier: str) -> Optional[SupervisorProxyThread]:
 
     def stop(self):
         """ Stop all the proxy threads. """
+        # prevent the creation of new threads, especially for INSTANCE_FAILURE notification
+        self.stop_event.set()
+        # stop all threads
         self.logger.debug(f'SupervisorProxyServer.stop: {list(self.proxies.keys())}')
         for proxy in self.proxies.values():
             proxy.stop()
@@ -395,28 +400,31 @@ def push_request(self, identifier: str, message):
         :param message: the message to send.
         :return: None.
         """
-        proxy = self.get_proxy(identifier)
-        if proxy:
-            proxy.push_message((InternalEventHeaders.REQUEST, (self.local_identifier, message)))
+        if not self.stop_event.is_set():
+            proxy = self.get_proxy(identifier)
+            if proxy:
+                proxy.push_message((InternalEventHeaders.REQUEST, (self.local_identifier, message)))
 
     def push_publication(self, message):
         """ Send a publication to all remote Supervisor proxies.
 
         :param message: the message to send.
         :return: None.
         """
-        for identifier in self.supvisors.mapper.instances:
-            # No publication to self instance because the event has already been processed.
-            if identifier != self.local_identifier:
-                proxy = self.get_proxy(identifier)
-                if proxy:
-                    proxy.push_message((InternalEventHeaders.PUBLICATION, (self.local_identifier, message)))
+        if not self.stop_event.is_set():
+            for identifier in self.supvisors.mapper.instances:
+                # No publication to self instance because the event has already been processed.
+                if identifier != self.local_identifier:
+                    proxy = self.get_proxy(identifier)
+                    if proxy:
+                        proxy.push_message((InternalEventHeaders.PUBLICATION, (self.local_identifier, message)))
 
     def push_notification(self, message):
         """ Send a discovery event to all remote Supervisor proxies.
 
         :param message: the message to send.
         :return: None.
         """
-        proxy = self.get_proxy(self.local_identifier)
-        proxy.push_message((InternalEventHeaders.NOTIFICATION, message))
+        if not self.stop_event.is_set():
+            proxy = self.get_proxy(self.local_identifier)
+            proxy.push_message((InternalEventHeaders.NOTIFICATION, message))

supvisors/tests/test_supervisorproxy.py

@@ -607,6 +607,7 @@ def test_proxy_server_creation(supvisors, proxy_server):
     """ Test the SupervisorProxyServer creation. """
     assert proxy_server.supvisors is supvisors
     assert proxy_server.proxies == {}
+    assert not proxy_server.stop_event.is_set()
     assert proxy_server.local_identifier == supvisors.mapper.local_identifier
 
 
@@ -646,6 +647,7 @@ def test_proxy_server_get_proxy(supvisors, mocked_rpc, proxy_server):
     assert proxy_server.proxies == {'10.0.0.2:25000': proxy_2bis}
     # test stop
     proxy_server.stop()
+    assert proxy_server.stop_event.is_set()
     assert proxy_server.proxies == {'10.0.0.2:25000': proxy_2bis}
     assert not proxy_2bis.is_alive()
 
@@ -677,6 +679,8 @@ def test_server_proxy_push_message(mocker, supvisors, mocked_rpc, proxy_server):
     # stop all and reset
     proxy_server.stop()
     proxy_server.proxies = {}
+    assert proxy_server.stop_event.is_set()
+    proxy_server.stop_event.clear()
     # test publish
     proxy_server.push_publication({'message': 'test publish'})
     assert len(mocked_push.call_args_list) == 4
@@ -687,6 +691,8 @@ def test_server_proxy_push_message(mocker, supvisors, mocked_rpc, proxy_server):
     # stop all and reset
     proxy_server.stop()
     proxy_server.proxies = {}
+    assert proxy_server.stop_event.is_set()
+    proxy_server.stop_event.clear()
     # test post_discovery
     proxy_server.push_notification({'message': 'test discovery'})
     assert mocked_push.call_args_list == [call((InternalEventHeaders.NOTIFICATION, {'message': 'test discovery'}))]

supvisors/ui/application.html

@@ -56,7 +56,7 @@ <h2 meld:id="appli_h_mid">Applications</h2>
 
             <footer>
                 <p>Contact: <a href="mailto:[email protected]">Julien Le Cl&amp;eacute;ach</a></p>
-                <p id="copyright">&amp;copy; Copyright 2016-2024 Julien LE CLEACH</p>
+                <p id="copyright">&amp;copy; Copyright 2016-2025 Julien LE CLEACH</p>
             </footer>
         </div>
 

supvisors/ui/conciliation.html

@@ -58,7 +58,7 @@ <h2 meld:id="appli_h_mid">Applications</h2>
 
             <footer>
                 <p>Contact: <a href="mailto:[email protected]">Julien Le Cl&amp;eacute;ach</a></p>
-                <p id="copyright">&amp;copy; Copyright 2016-2024 Julien LE CLEACH</p>
+                <p id="copyright">&amp;copy; Copyright 2016-2025 Julien LE CLEACH</p>
             </footer>
         </div>
 

supvisors/ui/host_instance.html

@@ -56,7 +56,7 @@ <h2 meld:id="appli_h_mid">Applications</h2>
 
             <footer>
                 <p>Contact: <a href="mailto:[email protected]">Julien Le Cl&amp;eacute;ach</a></p>
-                <p id="copyright">&amp;copy; Copyright 2016-2024 Julien LE CLEACH</p>
+                <p id="copyright">&amp;copy; Copyright 2016-2025 Julien LE CLEACH</p>
             </footer>
         </div>
 

supvisors/ui/index.html

@@ -57,7 +57,7 @@ <h2 meld:id="appli_h_mid">Applications</h2>
 
             <footer>
                 <p>Contact: <a href="mailto:[email protected]">Julien Le Cl&amp;eacute;ach</a></p>
-                <p id="copyright">&amp;copy; Copyright 2016-2024 Julien LE CLEACH</p>
+                <p id="copyright">&amp;copy; Copyright 2016-2025 Julien LE CLEACH</p>
             </footer>
         </div>
 

supvisors/ui/proc_instance.html

@@ -57,7 +57,7 @@ <h2 meld:id="appli_h_mid">Applications</h2>
 
             <footer>
                 <p>Contact: <a href="mailto:[email protected]">Julien Le Cl&amp;eacute;ach</a></p>
-                <p id="copyright">&amp;copy; Copyright 2016-2024 Julien LE CLEACH</p>
+                <p id="copyright">&amp;copy; Copyright 2016-2025 Julien LE CLEACH</p>
             </footer>
         </div>
 

supvisors/version.txt

@@ -1 +1 @@
-version=0.18.6
+version=0.18.7