Merge pull request #131 from julien6387/dev-0.18.6

Dev 0.18.6

CHANGES.md

@@ -1,5 +1,12 @@
 # Change Log
 
+## 0.18.6 (2024-08-20)
+
+* Completion of fix about process CPU statistics when using SOLARIS mode.
+
+* Add a grid to statistics plots in the **Supvisors** Web UI.
+
+
 ## 0.18.5 (2024-08-19)
 
 * Fix process CPU statistics when using SOLARIS mode.

supvisors/statscompiler.py

@@ -262,12 +262,12 @@ def cpu_process_statistics(latest: float, ref: float, host_work: float) -> float
 class ProcStatisticsInstance:
     """ This class handles statistics for a process running on a Supervisor instance and for a given period. """
 
-    def __init__(self, namespec: str = '', identifier: str = '', period: int = 0, depth: int = 0):
+    def __init__(self, namespec: str = '', identifier: str = '', period: float = 0.0, depth: int = 0):
         """ Initialization of the attributes. """
         # parameters
         self.namespec: str = namespec
         self.identifier: str = identifier
-        self.period: int = period
+        self.period: float = period
         self.depth: int = depth
         self.ref_stats: Payload = {}
         self.ref_start_time: float = 0.0
@@ -312,6 +312,14 @@ def push_statistics(self, proc_stats: Payload) -> Payload:
             self.ref_start_time = proc_stats['now']
         return result
 
+    def copy(self, cpu_factor: int):
+        """ Return a copy of the data in a new instance. """
+        instance_copy = ProcStatisticsInstance()
+        instance_copy.times = self.times.copy()
+        instance_copy.mem = self.mem.copy()
+        instance_copy.cpu = [x / cpu_factor for x in self.cpu]
+        return instance_copy
+
 
 class ProcStatisticsHolder:
     """ This class stores process statistics for a process that may be running on all Supervisor instances.
@@ -323,10 +331,10 @@ class ProcStatisticsHolder:
         - options: the Supvisors options
         - logger: the global Supvisors logger
         - instance_map: a dictionary of ProcStatisticsInstance for all Supvisors instances where the process is running
-        and for all periods.
+                        and for all periods.
     """
 
-    IdentifierInstanceMap = Dict[str, Tuple[int, Dict[int, ProcStatisticsInstance]]]
+    IdentifierInstanceMap = Dict[str, Tuple[float, Dict[int, ProcStatisticsInstance]]]
 
     def __init__(self, namespec: str, options, logger):
         """ Initialization of the attributes. """
@@ -336,11 +344,14 @@ def __init__(self, namespec: str, options, logger):
         # {identifier: (pid, {period: ProcStatisticsInstance}}
         self.instance_map: ProcStatisticsHolder.IdentifierInstanceMap = {}
 
-    def get_stats(self, identifier: str, period: int) -> Optional[ProcStatisticsInstance]:
+    def get_stats(self, identifier: str, period: float, cpu_factor: int) -> Optional[ProcStatisticsInstance]:
         """ Return the ProcStatisticsInstance corresponding to a Supvisors instance and a period. """
         _, identifier_instance = self.instance_map.get(identifier, (0, None))
         if identifier_instance:
-            return identifier_instance.get(period)
+            proc_stats = identifier_instance.get(period)
+            if proc_stats:
+                return proc_stats.copy(cpu_factor)
+        return None
 
     def push_statistics(self, identifier: str, process_stats: Payload) -> PayloadList:
         """ Consider a new list of process statistics received from a Supvisors instance.
@@ -386,11 +397,15 @@ def __init__(self, options, logger):
         # keep a CPU core map per identifier for Solaris mode
         self.nb_cores = {}
 
-    def get_stats(self, namespec: str, identifier: str, period: int) -> Optional[ProcStatisticsInstance]:
-        """ Return the ProcStatisticsInstance corresponding to a namespec, a Supvisors instance and a period. """
+    def get_stats(self, namespec: str, identifier: str, period: float) -> Optional[ProcStatisticsInstance]:
+        """ Return a copy of the current ProcStatisticsInstance corresponding to a namespec, a Supvisors instance
+        and a period.
+        CPU values are updated if SOLARIS mode is expected.
+        """
         proc_holder = self.holder_map.get(namespec)
         if proc_holder:
-            return proc_holder.get_stats(identifier, period)
+            cpu_factor = 1 if self.options.stats_irix_mode else self.nb_cores.get(identifier, 1)
+            return proc_holder.get_stats(identifier, period, cpu_factor)
 
     def get_nb_cores(self, identifier: str) -> int:
         """ Return the number of CPU cores linked to a Supvisors instance. """

supvisors/test/etc/supervisord_alt.conf

@@ -32,7 +32,7 @@ conciliation_strategy = USER
 stats_enabled = true
 stats_periods = 5,60,600
 stats_histo = 100
-stats_irix_mode = true
+stats_irix_mode = false
 logfile = AUTO
 ;logfile = ./log/supvisors_alt.log
 ;logfile_maxbytes = 50MB

supvisors/tests/test_plot.py

@@ -20,7 +20,7 @@
 
 pytest.importorskip('matplotlib', reason='cannot test as optional matplotlib is not installed')
 
-from supvisors.plot import *
+from supvisors.web.plot import *
 from supvisors.web.viewimage import StatsImage
 
 

supvisors/tests/test_statscompiler.py

@@ -533,13 +533,27 @@ def test_proc_statistics_holder_creation(supvisors, proc_statistics_holder):
 def test_proc_statistics_holder_get_stats(proc_statistics_holder):
     """ Test the search method for process statistics. """
     # change values
-    proc_statistics_holder.instance_map = {'10.0.0.1': (os.getpid(), {5: 'proc stats 5s on 10.0.0.1'})}
+    dummy_stats = ProcStatisticsInstance(identifier='10.0.0.1', period=5)
+    dummy_stats.cpu = [2, 4, 8]
+    proc_statistics_holder.instance_map = {'10.0.0.1': (os.getpid(), {5: dummy_stats})}
     # test find method with wrong identifier
-    assert proc_statistics_holder.get_stats('10.0.0.2', 5) is None
+    assert proc_statistics_holder.get_stats('10.0.0.2', 5, 1) is None
     # test find method with correct identifier and wrong period
-    assert proc_statistics_holder.get_stats('10.0.0.1', 10) is None
-    # test find method with correct identifier and period
-    assert proc_statistics_holder.get_stats('10.0.0.1', 5) == 'proc stats 5s on 10.0.0.1'
+    assert proc_statistics_holder.get_stats('10.0.0.1', 10, 1) is None
+    # test find method with correct identifier and period and irix factor
+    stats = proc_statistics_holder.get_stats('10.0.0.1', 5, 1)
+    assert stats is not None
+    assert stats is not dummy_stats
+    assert stats.identifier == ''
+    assert stats.period == 0
+    assert stats.cpu == [2, 4, 8]
+    # test find method with correct identifier and period and solaris factor
+    stats = proc_statistics_holder.get_stats('10.0.0.1', 5, 4)
+    assert stats is not None
+    assert stats is not dummy_stats
+    assert stats.identifier == ''
+    assert stats.period == 0
+    assert stats.cpu == [0.5, 1, 2]
 
 
 def test_proc_statistics_holder_push_statistics(mocker, proc_statistics_holder):
@@ -649,18 +663,28 @@ def test_proc_statistics_compiler_creation(supvisors, proc_statistics_compiler):
     assert proc_statistics_compiler.nb_cores == {}
 
 
-def test_proc_statistics_compiler_get_stats(mocker, proc_statistics_compiler):
+def test_proc_statistics_compiler_get_stats(mocker, supvisors, proc_statistics_compiler):
     """ Test the ProcStatisticsCompiler.get_stats method """
     mocked_stats = mocker.patch('supvisors.statscompiler.ProcStatisticsHolder.get_stats')
     # test on unknown namespec
-    assert proc_statistics_compiler.get_stats('dummy_proc', '10.0.0.1', '12.5') is None
+    assert proc_statistics_compiler.get_stats('dummy_proc', '10.0.0.1', 12.5) is None
     assert not mocked_stats.called
     # fill some data
     mocked_holder = Mock(**{'get_stats.return_value': 'some stats'})
     proc_statistics_compiler.holder_map['dummy_proc'] = mocked_holder
-    # test on known namespec
-    assert proc_statistics_compiler.get_stats('dummy_proc', '10.0.0.1', '12.5') == 'some stats'
-    assert mocked_holder.get_stats.call_args_list == [call('10.0.0.1', '12.5')]
+    # test on known namespec and irix mode
+    assert proc_statistics_compiler.get_stats('dummy_proc', '10.0.0.1', 12.5) == 'some stats'
+    assert mocked_holder.get_stats.call_args_list == [call('10.0.0.1', 12.5, 1)]
+    mocked_holder.get_stats.reset_mock()
+    # test on known namespec and solaris mode (nb_cores not set)
+    supvisors.options.stats_irix_mode = False
+    assert proc_statistics_compiler.get_stats('dummy_proc', '10.0.0.1', 12.5) == 'some stats'
+    assert mocked_holder.get_stats.call_args_list == [call('10.0.0.1', 12.5, 1)]
+    mocked_holder.get_stats.reset_mock()
+    # test on known namespec and solaris mode (nb_cores set)
+    proc_statistics_compiler.nb_cores['10.0.0.1'] = 4
+    assert proc_statistics_compiler.get_stats('dummy_proc', '10.0.0.1', 12.5) == 'some stats'
+    assert mocked_holder.get_stats.call_args_list == [call('10.0.0.1', 12.5, 4)]
 
 
 def test_proc_statistics_compiler_get_nb_cores(proc_statistics_compiler):

supvisors/tests/test_viewapplication.py

@@ -300,7 +300,7 @@ def test_get_process_data(mocker, view):
     view.application = Mock(processes={process_1.process_name: process_1, process_2.process_name: process_2})
     # patch context
     mocked_stats = Mock()
-    view.view_ctx = Mock(**{'get_process_stats.return_value': (4, mocked_stats),
+    view.view_ctx = Mock(**{'get_process_stats.return_value': mocked_stats,
                             'get_process_shex.side_effect': [(True, None), (False, None)]})
     # test call
     data_1 = {'row_type': ProcessRowTypes.APPLICATION_PROCESS,
@@ -309,23 +309,23 @@ def test_get_process_data(mocker, view):
               'statename': 'STOPPED', 'statecode': ProcessStates.STOPPED, 'gravity': 'FATAL',
               'has_crashed': False, 'running_identifiers': [], 'description': 'stopped',
               'main': True, 'nb_items': 1,
-              'expected_load': 20, 'nb_cores': 4, 'proc_stats': mocked_stats,
+              'expected_load': 20, 'proc_stats': mocked_stats,
               'has_stdout': True, 'has_stderr': False}
     data_1_1 = {'row_type': ProcessRowTypes.INSTANCE_PROCESS,
                 'application_name': 'appli_1', 'process_name': '', 'namespec': 'namespec_1',
                 'identifier': '10.0.0.1', 'disabled': False, 'startable': False, 'stoppable': False,
                 'statename': 'STOPPED', 'statecode': ProcessStates.STOPPED, 'gravity': 'STOPPED',
                 'has_crashed': False, 'running_identifiers': ['10.0.0.1'], 'description': 'process_1 on 10.0.0.1',
                 'main': False, 'nb_items': 0,
-                'expected_load': 20, 'nb_cores': 4, 'proc_stats': mocked_stats,
+                'expected_load': 20, 'proc_stats': mocked_stats,
                 'has_stdout': False, 'has_stderr': True}
     data_2 = {'row_type': ProcessRowTypes.APPLICATION_PROCESS,
               'application_name': 'appli_2', 'process_name': 'process_2', 'namespec': 'namespec_2',
               'identifier': None, 'disabled': False, 'startable': True, 'stoppable': True,
               'statename': 'RUNNING', 'statecode': ProcessStates.RUNNING, 'gravity': 'RUNNING',
               'has_crashed': True, 'running_identifiers': ['10.0.0.1', '10.0.0.3'], 'description': 'conflict',
               'main': True, 'nb_items': 4,
-              'expected_load': 1, 'nb_cores': 0, 'proc_stats': None,
+              'expected_load': 1, 'proc_stats': None,
               'has_stdout': False, 'has_stderr': False}
     assert view.get_process_data() == [data_1, data_1_1, data_2]
 

supvisors/tests/test_viewcontext.py

@@ -564,14 +564,11 @@ def test_get_node_stats(supvisors, ctx):
 
 def test_get_process_stats(mocker, supvisors, ctx):
     """ Test the ViewContext.get_process_stats method. """
-    mocked_core = mocker.patch('supvisors.web.viewcontext.ViewContext.get_nb_cores', return_value=4)
     # test no result as no data stored
-    assert ctx.get_process_stats('dummy_proc', ctx.local_identifier) == (4, None)
-    mocked_core.reset_mock()
+    assert ctx.get_process_stats('dummy_proc', ctx.local_identifier) is None
     # fill some internal structures
     mocked_stats = mocker.patch.object(supvisors.process_compiler, 'get_stats', return_value='mock stats')
-    assert ctx.get_process_stats('dummy_proc', '10.0.0.1') == (4, 'mock stats')
-    assert mocked_core.call_args_list == [call('10.0.0.1')]
+    assert ctx.get_process_stats('dummy_proc', '10.0.0.1') == 'mock stats'
     assert mocked_stats.call_args_list == [call('dummy_proc', '10.0.0.1', 5.0)]