Add nexthop group usage check in route_check

scripts/route_check.py

@@ -55,6 +55,7 @@
 from utilities_common.general import load_db_config
 
 APPL_DB_NAME = 'APPL_DB'
+COUNTERS_DB_NAME = 'COUNTERS_DB'
 ASIC_DB_NAME = 'ASIC_DB'
 ASIC_TABLE_NAME = 'ASIC_STATE'
 ASIC_KEY_PREFIX = 'SAI_OBJECT_TYPE_ROUTE_ENTRY:'
@@ -81,6 +82,9 @@
 
 REDIS_TIMEOUT_MSECS = 0
 
+NEXT_HOP_THRESHOLD = 80
+
+
 class Level(Enum):
     ERR = 'ERR'
     INFO = 'INFO'
@@ -814,6 +818,45 @@ def _filter_out_neigh_route(routes, neighs):
     return rt_appl_miss, rt_asic_miss
 
 
+def get_crm_nexthop_group_usage(namespace):
+    """Get CRM nexthop group usage percentage."""
+
+    db = swsscommon.DBConnector(COUNTERS_DB_NAME, REDIS_TIMEOUT_MSECS, True, namespace)
+    print_message(syslog.LOG_DEBUG, "COUNTERS DB {} connected".format(namespace))
+    crm_resources = swsscommon.Table(db, 'CRM')
+    stats_result = crm_resources.get('STATS')
+    stats_found = stats_result[0] if stats_result and len(stats_result) > 0 else False
+    crm_stats_values = stats_result[1] if stats_result and len(stats_result) > 1 else []
+
+    nh_group_used = None
+    nh_group_available = None
+
+    if stats_found:
+        if isinstance(crm_stats_values, dict):
+            items = crm_stats_values.items()
+        else:
+            items = crm_stats_values
+
+        for field, value in items:
+            print_message(syslog.LOG_DEBUG, f"CRM field: {field}, value: {value}")
+            if field == "crm_stats_nexthop_group_used":
+                nh_group_used = int(value)
+            elif field == "crm_stats_nexthop_group_available":
+                nh_group_available = int(value)
+
+    if nh_group_used is None and nh_group_available is None:
+        # NOTE: CRM STATS may not ready in counters DB when device just boots up.
+        print_message(syslog.LOG_DEBUG, "Cannot get CRM stats, skipping nexthop group usage check")
+        return None
+
+    total_nexthop_groups = nh_group_used + nh_group_available
+    if total_nexthop_groups == 0:
+        print_message(syslog.LOG_DEBUG, "No nexthop groups configured")
+        return None
+    nh_group_usage = nh_group_used / total_nexthop_groups * 100
+    return nh_group_usage
+
+
 def check_routes_for_namespace(namespace):
     """
     Process a Single Namespace:
@@ -904,6 +947,11 @@ def check_routes_for_namespace(namespace):
     if rt_frr_failed:
         results["failed_FRR_routes"] = rt_frr_failed
 
+    nh_group_usage = get_crm_nexthop_group_usage(namespace)
+
+    if nh_group_usage is not None and nh_group_usage > NEXT_HOP_THRESHOLD:
+        results["exceed_nexthop_group_threshold"] = nh_group_usage
+
     if results:
         if rt_frr_miss and not rt_appl_miss and not rt_asic_miss:
             print_message(syslog.LOG_ERR, "Some routes are not set offloaded in FRR{} \

tests/route_check_test.py

@@ -8,7 +8,7 @@
 from sonic_py_common import device_info
 from unittest.mock import MagicMock, patch
 from tests.route_check_test_data import (
-    APPL_DB, MULTI_ASIC, NAMESPACE, DEFAULTNS, ARGS, ASIC_DB, CONFIG_DB,
+    APPL_DB, MULTI_ASIC, NAMESPACE, DEFAULTNS, ARGS, ASIC_DB, CONFIG_DB, COUNTERS_DB,
     DEFAULT_CONFIG_DB, APPL_STATE_DB, DESCR, OP_DEL, OP_SET, PRE, RESULT, RET, TEST_DATA,
     UPD, FRR_ROUTES
 )
@@ -82,6 +82,7 @@ def init_db_conns(namespaces):
     for ns in namespaces:
         db_conns[ns] = {
             "APPL_DB": {"namespace": ns, "name": APPL_DB},
+            "COUNTERS_DB": {"namespace": ns, "name": COUNTERS_DB},
             "ASIC_DB": {"namespace": ns, "name": ASIC_DB},
             "APPL_STATE_DB": {"namespace": ns, "name": APPL_STATE_DB},
             "CONFIG_DB": ConfigDB(ns)

tests/route_check_test_data.py

@@ -5,6 +5,7 @@
 RET = "return"
 APPL_DB = 0
 ASIC_DB = 1
+COUNTERS_DB = 2
 CONFIG_DB = 4
 APPL_STATE_DB = 14
 PRE = "pre-value"
@@ -28,6 +29,8 @@
 SEPARATOR = ":"
 DEVICE_METADATA = "DEVICE_METADATA"
 MUX_CABLE = "MUX_CABLE"
+CRM_TABLE = 'CRM'
+CRM_STATS = 'STATS'
 
 LOCALHOST = "localhost"
 
@@ -78,6 +81,14 @@
                         ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
                         ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {}
                     }
+                },
+                COUNTERS_DB: {
+                    CRM_TABLE: {
+                        CRM_STATS: {
+                            "crm_stats_nexthop_group_used": "20",
+                            "crm_stats_nexthop_group_available": "80"
+                        }
+                    }
                 }
             }
         }
@@ -771,31 +782,31 @@
                         ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
                         ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {}
                     }
-                }
+                },
             },
             ASIC1: {
-               APPL_DB: {
-                   ROUTE_TABLE: {
-                       "0.0.0.0/0": {"ifname": "portchannel0"},
-                       "10.10.196.12/31": {"ifname": "portchannel0"},
-                       "10.10.196.20/31": {"ifname": "portchannel0"},
-                       "10.10.196.30/31": {"ifname": "lo"}
-                   },
-                   INTF_TABLE: {
-                       "PortChannel1013:10.10.196.24/31": {},
-                       "PortChannel1023:2603:10b0:503:df4::5d/126": {},
-                       "PortChannel1024": {}
-                   }
-               },
-               ASIC_DB: {
-                   RT_ENTRY_TABLE: {
-                       ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
-                       ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
-                       ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
-                       ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
-                       ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {}
-                   }
-               }
+                APPL_DB: {
+                    ROUTE_TABLE: {
+                        "0.0.0.0/0": {"ifname": "portchannel0"},
+                        "10.10.196.12/31": {"ifname": "portchannel0"},
+                        "10.10.196.20/31": {"ifname": "portchannel0"},
+                        "10.10.196.30/31": {"ifname": "lo"}
+                    },
+                    INTF_TABLE: {
+                        "PortChannel1013:10.10.196.24/31": {},
+                        "PortChannel1023:2603:10b0:503:df4::5d/126": {},
+                        "PortChannel1024": {}
+                    }
+                },
+                ASIC_DB: {
+                    RT_ENTRY_TABLE: {
+                        ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
+                        ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
+                        ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
+                        ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
+                        ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {}
+                    }
+                }
            },
         }
     },
@@ -868,28 +879,28 @@
                 }
             },
             ASIC1: {
-               APPL_DB: {
-                   ROUTE_TABLE: {
-                       "0.0.0.0/0": {"ifname": "portchannel0"},
-                       "10.10.196.20/31": {"ifname": "portchannel0"},
-                       "10.10.196.30/31": {"ifname": "lo"}
-                   },
-                   INTF_TABLE: {
-                       "PortChannel1013:10.10.196.24/31": {},
-                       "PortChannel1023:2603:10b0:503:df4::5d/126": {},
-                       "PortChannel1024": {}
-                   }
-               },
-               ASIC_DB: {
-                   RT_ENTRY_TABLE: {
-                       ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
-                       ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
-                       ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
-                       ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
-                       ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {}
-                   }
-               }
-           },
+                APPL_DB: {
+                    ROUTE_TABLE: {
+                        "0.0.0.0/0": {"ifname": "portchannel0"},
+                        "10.10.196.20/31": {"ifname": "portchannel0"},
+                        "10.10.196.30/31": {"ifname": "lo"}
+                    },
+                    INTF_TABLE: {
+                        "PortChannel1013:10.10.196.24/31": {},
+                        "PortChannel1023:2603:10b0:503:df4::5d/126": {},
+                        "PortChannel1024": {}
+                    }
+                },
+                ASIC_DB: {
+                    RT_ENTRY_TABLE: {
+                        ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
+                        ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
+                        ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
+                        ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
+                        ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {}
+                    }
+                }
+            },
         },
         RESULT: {
             ASIC0: {
@@ -1704,4 +1715,50 @@
         },
         RET: -1,
     },
+    "33": {
+        DESCR: "nexthop group usage exceeds threshold",
+        MULTI_ASIC: False,
+        NAMESPACE: [''],
+        ARGS: "route_check -m INFO -i 1000",
+        PRE: {
+            DEFAULTNS: {
+                APPL_DB: {
+                    ROUTE_TABLE: {
+                        "0.0.0.0/0": {"ifname": "portchannel0"},
+                        "10.10.196.12/31": {"ifname": "portchannel0"},
+                        "10.10.196.20/31": {"ifname": "portchannel0"},
+                        "10.10.196.30/31": {"ifname": "lo"}
+                    },
+                    INTF_TABLE: {
+                        "PortChannel1013:10.10.196.24/31": {},
+                        "PortChannel1023:2603:10b0:503:df4::5d/126": {},
+                        "PortChannel1024": {}
+                    }
+                },
+                ASIC_DB: {
+                    RT_ENTRY_TABLE: {
+                        ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
+                        ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
+                        ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
+                        ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {},
+                        ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {}
+                    }
+                },
+                COUNTERS_DB: {
+                    CRM_TABLE: {
+                        CRM_STATS: {
+                            "crm_stats_nexthop_group_used": "85",
+                            "crm_stats_nexthop_group_available": "15"
+                        }
+                    }
+                }
+            }
+        },
+        RESULT: {
+            DEFAULTNS: {
+                "exceed_nexthop_group_threshold": 85.0
+            }
+        },
+        RET: -1,
+    }
 }