add metric lightllm_request_mtp_avg_token_per_step (#1169)

Author: WANDY666

lightllm/server/httpserver/manager.py

@@ -645,6 +645,9 @@ async def _wait_to_token_package(
                         )
                         self.metric_client.histogram_observe("lightllm_request_generated_tokens", out_token_counter)
                         self.metric_client.counter_inc("lightllm_request_success")
+                        self.metric_client.histogram_observe(
+                            "lightllm_request_mtp_avg_token_per_step", mtp_avg_token_per_step
+                        )
 
                         return
                 req_status.out_token_info_list.clear()

lightllm/server/httpserver_for_pd_master/manager.py

@@ -320,6 +320,7 @@ async def _wait_to_token_package(
         group_request_id = sampling_params.group_request_id
         unfinished_count = sampling_params.best_of
         is_first_token = True
+        sub_req_id_to_mtp_accepted_token_num: Dict[int, int] = {}
 
         client_mode: NodeRole = NodeRole(d_node.mode)
 
@@ -333,6 +334,7 @@ async def _wait_to_token_package(
 
             prompt_tokens = metadata["prompt_tokens"]
             out_token_counter += 1
+            sub_req_id_to_mtp_accepted_token_num[sub_req_id] = metadata.get("mtp_accepted_token_num", 0)
             if is_first_token:
                 first_token_cost_ms = (time.time() - start_time) * 1000
                 is_first_token = False
@@ -351,6 +353,9 @@ async def _wait_to_token_package(
         x_session_id = request.headers.get("X-Session-Id", "")
         prompt_cache_len = metadata.pop("prompt_cache_len", 0)
         prompt_cache_ratio = prompt_cache_len / prompt_tokens
+        mtp_avg_token_per_step = out_token_counter / max(
+            (out_token_counter - sum(sub_req_id_to_mtp_accepted_token_num.values())), 1
+        )
         format_start_time = datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%d %H:%M:%S")
         logger.info(
             f"X-Request-Id:{x_request_id} "
@@ -361,6 +366,7 @@ async def _wait_to_token_package(
             f"prompt_token_num:{prompt_tokens} "
             f"prompt_cache_len:{prompt_cache_len} "
             f"prompt_cache_ratio:{prompt_cache_ratio} "
+            f"mtp_avg_token_per_step:{mtp_avg_token_per_step} "
         )
         self.metric_client.histogram_observe("lightllm_request_inference_duration", total_cost_time_ms / 1000.0)
         self.metric_client.histogram_observe(
@@ -369,6 +375,7 @@ async def _wait_to_token_package(
         self.metric_client.histogram_observe("lightllm_request_first_token_duration", first_token_cost_ms / 1000.0)
         self.metric_client.histogram_observe("lightllm_request_generated_tokens", out_token_counter)
         self.metric_client.counter_inc("lightllm_request_success")
+        self.metric_client.histogram_observe("lightllm_request_mtp_avg_token_per_step", mtp_avg_token_per_step)
         return
 
     async def abort(

lightllm/server/metrics/metrics.py

@@ -26,6 +26,7 @@
     "lightllm_cache_length": "Length of tokens which hit prompt cache",
     "lightllm_cache_ratio": "cache length / input_length",
     "lightllm_batch_current_max_tokens": "dynamic max token used for current batch",
+    "lightllm_request_mtp_avg_token_per_step": "Average number of tokens per step",
 }
 
 
@@ -94,6 +95,9 @@ def init_metrics(self, args):
         ratio_buckets = [(i + 1) / 10.0 for i in range(-1, 10)]
         self.create_histogram("lightllm_cache_ratio", ratio_buckets)
 
+        mtp_avg_token_per_step_buckets = [i / 10.0 + 1.0 for i in range(0, 10 * args.mtp_step)]
+        self.create_histogram("lightllm_request_mtp_avg_token_per_step", mtp_avg_token_per_step_buckets)
+
     def create_histogram(self, name, buckets, labelnames=None):
         if labelnames is None:
             histogram = Histogram(name, MONITOR_INFO[name], buckets=buckets, registry=self.registry)