Merge branch 'xnevrk03_data_logger_stats' into 'devel'

Upgrade statistics processing for data_logger, refactor histogramer, add memory comp wraps

See merge request ndk/ndk-fpga!90

Author: jakubcabal

CHANGELOG.md

@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 - Improve documentation
 - Improve documentation looks
+- Data_logger: Added packages for statistics processing
+- Histogramer: Fixed histogram box update
 
 ## [0.7.2] - 2024-10-17
 

comp/base/mem/mem_clear/Modules.tcl

@@ -0,0 +1,11 @@
+# Modules.tcl: Components include script
+# Copyright (C) 2024 CESNET
+# Author(s): Lukas Nevrkla <[email protected]>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# Packages
+lappend PACKAGES "$OFM_PATH/comp/base/pkg/math_pack.vhd"
+lappend PACKAGES "$OFM_PATH/comp/base/pkg/type_pack.vhd"
+
+lappend MOD "$ENTITY_BASE/mem_clear.vhd"

comp/base/mem/mem_clear/mem_clear.vhd

@@ -0,0 +1,108 @@
+-- mem_clear.vhd: Unit for clearing BRAM memories
+-- Copyright (C) 2024 CESNET z. s. p. o.
+-- Author(s): Lukas Nevrkla <[email protected]>
+--
+-- SPDX-License-Identifier: BSD-3-Clause
+
+library ieee;
+use ieee.std_logic_1164.all;
+use ieee.numeric_std.all;
+
+use work.math_pack.all;
+use work.type_pack.all;
+
+entity MEM_CLEAR is
+generic (
+    DATA_WIDTH  : integer := 32;
+    ITEMS       : integer := 512;
+    -- Will disable memory clearing during RST
+    CLEAR_EN    : boolean := true
+);
+port (
+    CLK         : in  std_logic;
+    RST         : in  std_logic;
+
+    -- All addresses were generated
+    CLEAR_DONE  : out std_logic;
+    -- Clear address given by CLEAR_ADDR
+    CLEAR_WR    : out std_logic;
+    CLEAR_ADDR  : out std_logic_vector(log2(ITEMS) - 1 downto 0)
+);
+end entity;
+
+architecture FULL of MEM_CLEAR is
+
+    type FSM_STATES_T is (
+        CLEAR,
+        RUNNING
+    );
+
+    -- State machine --
+
+    signal curr_state           : FSM_STATES_T;
+    signal next_state           : FSM_STATES_T;
+
+    signal addr_i               : std_logic_vector(log2(ITEMS)-1 downto 0);
+    signal addr_r               : std_logic_vector(log2(ITEMS)-1 downto 0);
+    signal rst_r                : std_logic;
+
+ begin
+
+    CLEAR_ADDR          <= addr_i;
+
+    reg_p : process (CLK)
+    begin
+        if (rising_edge(CLK)) then
+            addr_r      <= addr_i;
+            rst_r       <= RST;
+        end if;
+    end process;
+
+    -------------------
+    -- STATE MACHINE --
+    -------------------
+
+    state_reg_p : process (CLK)
+    begin
+        if (rising_edge(CLK)) then
+            if (RST = '1') then
+                if (CLEAR_EN = true) then
+                    curr_state <= CLEAR;
+                else
+                    curr_state <= RUNNING;
+                end if;
+            else
+                curr_state <= next_state;
+            end if;
+        end if;
+    end process;
+
+    -- Output logic
+    process (all)
+    begin
+        CLEAR_DONE          <= '0';
+        CLEAR_WR            <= '0';
+        next_state          <= curr_state;
+
+        case curr_state is
+            when CLEAR =>
+                if (RST = '0') then
+                    CLEAR_wR    <= '1';
+
+                    if (rst_r = '1') then
+                        addr_i  <= (others => '0');
+                    else
+                        addr_i  <= std_logic_vector(unsigned(addr_r) + 1);
+                    end if;
+
+                    if (unsigned(addr_i) = (ITEMS - 1)) then
+                        next_state  <= RUNNING;
+                    end if;
+                end if;
+
+            when RUNNING =>
+                CLEAR_DONE      <= '1';
+        end case;
+    end process;
+
+end architecture;

comp/base/mem/mem_clear/readme.rst

@@ -0,0 +1,33 @@
+.. _mem_clear:
+
+Memory clear
+------------
+
+Simple component that will generate addresses for memory clearing when RST is asserted.
+
+Component port and generics description
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. vhdl:autoentity:: MEM_CLEAR
+   :noautogenerics:
+
+
+Instance template
+^^^^^^^^^^^^^^^^^
+
+.. code-block::
+
+    data_clear_i : entity work.MEM_CLEAR
+    generic map (
+        DATA_WIDTH  => BOX_WIDTH,
+        ITEMS       => BOX_CNT,
+        CLEAR_EN    => CLEAR_BY_RST
+    )
+    port map (
+        CLK         => CLK,
+        RST         => RST,
+
+        CLEAR_DONE  => RST_DONE,
+        CLEAR_WR    => wr_clear,
+        CLEAR_ADDR  => wr_addr_clear
+    );

comp/debug/data_logger/data_logger.vhd

@@ -350,8 +350,9 @@ begin
             generic map(
                 INPUT_WIDTH             => VALUE_WIDTH(i),
                 BOX_WIDTH               => HIST_BOX_WIDTH(i),
-                BOX_CNT                 => HIST_BOX_CNT(i)
-                --READ_PRIOR              => READ_PRIOR
+                BOX_CNT                 => HIST_BOX_CNT(i),
+                -- TODO: new value can be lost when read occurs!
+                READ_PRIOR              => true
             )
             port map(
                 CLK                     => CLK,
@@ -380,8 +381,13 @@ begin
     -- RST management --
     --------------------
 
-    rst_intern  <= RST or mi_ctrl_reg(CTRL_RST_BIT);
-    SW_RST      <= mi_ctrl_reg(CTRL_RST_BIT);
+    rst_p : process(CLK)
+    begin
+        if (rising_edge(CLK)) then
+            rst_intern  <= RST or mi_ctrl_reg(CTRL_RST_BIT);
+            SW_RST      <= mi_ctrl_reg(CTRL_RST_BIT);
+        end if;
+    end process;
 
     rst_done_g : if (VALUE_CNT > 0) generate
         rst_done_intern <= and rst_done_vec;

comp/debug/data_logger/mem_logger/mem_logger.vhd

@@ -54,7 +54,7 @@ generic (
     MI_ADDR_WIDTH           : integer := 32;
 
     -- Specify read latency histogram precision
-    HISTOGRAM_BOXES         : integer := 255;
+    HISTOGRAM_BOXES         : integer := 256;
     -- Specify maximum paraller read requests
     MAX_PARALEL_READS       : integer := 128;
     -- Specify read latency ticks count width

comp/debug/data_logger/readme.rst

@@ -171,22 +171,124 @@ Instance template (full usage)
 Control SW
 ^^^^^^^^^^
 
-Folder ``data_logger/sw/`` contains ``Python3`` package that provides:
+Folder ``data_logger/sw/`` contains following ``Python3`` packages:
 
-* Module for basic interaction with ``DATA_LOGGER``
-* Modules for ``DATA_LOGGER`` wraps like ``MEM_LOGGER``
-* Simple graph generator based on `matplotlib` library
-* Simple PDF / Markdown report generator
-* Common tools
+* ``data_logger`` ... basic interaction with ``DATA_LOGGER``
+* ``mem_logger`` ... basic interaction with ``MEM_LOGGER``
+* ``logger_stats`` ... loading firmware statistics (multiple ``DATA_LOGGERS`` can be organized in tree hierarchy)
+* ``graph_tools`` ... simple plot functions for statistics from ``logger_stats``
 
 Package can be installed using this command:
 
 * You also need to install ``python nfb`` package
 
 .. code-block::
+    python3 -m pip install --upgrade pip
 
+    # Install nfb:
+    cd swbase/pynfb
+    python3 -m pip install Cython
+    python3 -m pip install .
+    cd -
+
+    # Install this package:
     cd data_logger/sw
-    python3 setup.py install --user
+    python3 -m pip install .
+
+Example usage of ``logger_stats`` (for more usage see `mem_logger/mem_logger.py`):
+
+.. code-block::
+
+
+    import logger_stats as Stats
+    from data_logger.data_logger import DataLogger
+
+    def create_stats():
+        # Create DataLoggers
+        logger_0 = DataLogger(index=0, dev='/dev/nfb0')
+        logger_1 = DataLogger(index=1, dev='/dev/nfb0')
+
+        # Create Stats hierarchy
+        stats = Stats.LoggerStats('Example stats')
+        stats_0 = Stats.LoggerStats('Logger 0 stats', logger=logger_0)
+        stats_1 = Stats.LoggerStats('Logger 1 stats', logger=logger_1)
+        stats.add_stat(stats_0)
+        stats.add_stat(stats_1)
+
+        # Add basic statistics
+        stats_0.add_stat(Stats.Constant(index=7, name='X'))
+        stats_0.add_stat(Stats.Counter(index=7, name='Y'))
+        stats_0.add_stat(Stats.Value(index=7, name='Z'))
+
+        # FSM state statistic
+        def fms_convert(v):
+            states = [
+                'IDLE',
+                ...
+            ]
+            if v >= len(states):
+                return "???"
+            else:
+                return states[int(v)]
+
+        fsm_format = Stats.FormatDefaultValue(format=Stats.FormatNone)
+        stats_1.add_stat(Stats.Value(2, 'FSM states', convert=fms_convert, format=fsm_format))
+
+        # Latency statistic
+        FREQ = 200 * 10**6
+        time_conv = Stats.ConvertTime(FREQ)
+        time_form = Stats.FormatDefaultValue(units='ns')
+        stats_1.add_stat(Stats.Value(9, 'Latency', convert=time_conv, format=time_form))
+
+        # Add value statistic which includes multiple commands
+        CMDS = [
+            'CMD_A',
+            ...
+        ]
+        stats_1.add_stat(Stats.ValueCMD(7, 'Latency of CMDs', cmd_width=2, cmds=CMDS, convert=time_conv, format=time_form))
+
+        # Add multiple counters
+        counters = [
+            'Counter A',
+            ...
+        ]
+        stats_1.add_stats(
+            name='Counters',
+            names=counters,
+            indexes=list(range(len(counters))),
+            constructor=lambda i, n: Stats.Counter(i, n)
+        )
+
+    return stats
+
+
+    stats = create_stats()
+    stats.load()
+    print(stats.to_str())
+    stats.save('stats.npz')
+
+
+Example usage of ``graph_tools``:
+
+    from graph_tools.graph_tools import load_data, plot_counter, plot_value, plot_value_2d
+
+    stats = load_data('stats.npz')
+
+    node = pd.DataFrame.from_dict(stats['Stats A']['Counters'])
+    selected = ['Counter A', 'Counter B']
+
+    # Plot single counter
+    plot_counter(node['Counter X'], 'Time', 'Requests', 'Plot title')
+
+    # Plot multiple counters
+    plot_counter(node[selected], 'Time', 'Requests', 'Plot title')
+
+    # Plot histogram of the value interface
+    plot_value(node['Value A'], 'Time', 'Blocks', 'Title' log=True)
+
+    # Plot 2D histogram of the value interface history
+    plot_value_2d(node['Value A'], 'Time', 'Blocks', 'Title' log=True)
+
 
 
 MI address space

comp/debug/data_logger/sw/data_logger/__init__.py

@@ -1,3 +0,0 @@
-from data_logger import data_logger
-
-__all__ = ["data_logger"]