verl_v0.6.1_modifications.diff

diff --git a/verl/interactions/checklist_interaction.py b/verl/interactions/checklist_interaction.py
new file mode 100644
index 00000000..9da2d945
--- /dev/null
+++ b/verl/interactions/checklist_interaction.py
@@ -0,0 +1,289 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+# Copyright 2023-2024 SGLang Team
+# Copyright 2025 ModelBest Inc. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import defaultdict
+import logging
+import os
+import json
+import asyncio
+from typing import Any, Optional
+from uuid import uuid4
+import random
+
+import httpx
+from verl import DataProto
+from verl.utils.reward_score import checklist_reward
+
+from .base import BaseInteraction
+
+logger = logging.getLogger(__name__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+class ChecklistInteraction(BaseInteraction):
+    """A demo interaction for calculating the reward of gsm8k.
+
+    - `start_interaction`: start a interaction instance for a trajectory.
+    - `generate_response`: generate the response of the assistant.
+    - `calculate_score`: calculate the score of the interaction.
+    - `finalize_interaction`: finalize the interaction instance.
+    """
+
+    def __init__(self, config: dict):
+        super().__init__(config)
+
+        self._sglang_url = config.get("sglang_url", [])
+        self._sglang_model = config.get("sglang_model")
+        self._retry_times = config.get("retry_times")
+        self._semaphore_size = config.get("semaphore_size")
+        self._temperature = config.get("temperature")
+        self._top_p = config.get("top_p")
+        self._max_new_tokens = config.get("max_new_tokens")
+        self._max_tokens = config.get("max_tokens")
+        self._timeout = config.get("timeout", 120.0)
+        self._max_checklist_to_use = config.get("max_checklist_to_use",1)
+
+        self._instance_dict = {}
+
+        # Initialize shared client and semaphore
+        try:
+            self._timeout_value = float(self._timeout)
+        except Exception:
+            self._timeout_value = 120.0
+        try:
+            semaphore_size = int(self._semaphore_size) if self._semaphore_size is not None else 64
+        except Exception:
+            semaphore_size = 64
+        self._semaphore = asyncio.Semaphore(semaphore_size)
+        self._httpx_limits = httpx.Limits(
+            max_connections=semaphore_size,
+            max_keepalive_connections=semaphore_size // 2,
+        )
+        self._client = httpx.AsyncClient(
+            timeout=httpx.Timeout(
+                timeout=self._timeout_value,
+                read=self._timeout_value,
+                write=self._timeout_value,
+                connect=self._timeout_value,
+            ),
+            limits=self._httpx_limits,
+        )
+
+        
+    async def start_interaction(
+        self, instance_id: Optional[str] = None, checklist_list: list[list[list[dict[str, Any]]]] = None, **kwargs
+    ) -> str:
+        if instance_id is None:
+            instance_id = str(uuid4())
+        self._instance_dict[instance_id] = {
+            "response": "",
+            "reward": 0.0,
+            "turns": 0,
+            "checklist_list": checklist_list,
+            "num_turns": len(checklist_list[0])
+        }
+        return instance_id
+
+    async def generate_response(
+        self, instance_id: str, messages: list[dict[str, Any]], all_messages: list[dict[str, Any]], **kwargs
+    ) -> tuple[bool, str, float, dict]:
+        checklist_list = kwargs.get("checklist_list")
+        results = []
+        call_success = []
+        for checklist in checklist_list:
+            results.append(asyncio.create_task(self.generate_response_for_single_checklist(instance_id, messages, all_messages, checklist=checklist)))
+            # await asyncio.sleep(0.001)
+        results = await asyncio.gather(*results)
+
+        per_step_results_list = [result[1] for result in results] # (len(checklist_list), step, len(this_turn_checklist)),  len(this_turn_checklist) is not the same for each checklist
+        per_step_call_success_list = [result[2] for result in results]
+        should_terminate_sequence_list = [result[0] for result in results]
+        
+        if self._instance_dict[instance_id]["turns"]+1 < self._instance_dict[instance_id]["num_turns"]:
+            user_idx = 0
+            for i in range(0, len(all_messages)):
+                item = all_messages[i]
+                if item.get("role") == "user":
+                    if user_idx == self._instance_dict[instance_id]["turns"]+1:
+                        response = item.get("content")
+                        logger.debug(f"Proceeding with the next turn response: {response}")
+                        break
+                    user_idx += 1
+            assert response != ""
+            should_terminate_sequence = False
+        else:
+            response = ""
+            should_terminate_sequence = True
+        
+        # Check if more than half of the should_terminate_sequence_list are True
+        true_count = sum(should_terminate_sequence_list)
+        if true_count > len(should_terminate_sequence_list) / 2:
+            should_terminate_sequence = True
+        
+        for x in per_step_call_success_list:
+            for y in x:
+                for z in y:
+                    if z==False:
+                        logger.warning("reward gen failed, terminate seq")
+                        should_terminate_sequence = True
+
+        self._instance_dict[instance_id]["turns"] += 1
+
+        return should_terminate_sequence, response, (per_step_results_list, [self._instance_dict[instance_id]["turns"]-1]*len(per_step_results_list[0]), per_step_call_success_list), {}
+            
+    async def generate_response_for_single_checklist(
+        self, instance_id: str, messages: list[dict[str, Any]], all_messages: list[dict[str, Any]], checklist: list[list[dict[str, Any]]], **kwargs
+    ) -> tuple[bool, str, float, dict]:
+
+
+        this_turn_checklist = checklist[self._instance_dict[instance_id]["turns"]]
+        not_required_for_next_turn_list = [not single_step_checklist["required_for_next_turn"] for single_step_checklist in this_turn_checklist]
+        all_step_results_this_turn = []
+
+        args = {
+                "sglang_model": self._sglang_model,
+                "sglang_url": None,
+                "temperature": self._temperature,
+                "top_p": self._top_p,
+                "max_new_tokens": self._max_new_tokens,
+                "max_tokens": self._max_tokens,
+                "retry_times": self._retry_times
+        }
+
+        # find last user message idx
+        last_user_message_idx = -1
+        for i in range(len(messages)-1, -1, -1):
+            if messages[i].role == "user":
+                last_user_message_idx = i
+                break
+        assert last_user_message_idx != -1
+
+
+        step = 0
+        for i in range(last_user_message_idx+1, len(messages)):
+            if messages[i].role == "assistant":
+                this_step_message = [messages[i]]
+                messages_before_this_step = messages[:i]
+                this_step_message_str = checklist_reward.get_messages_str_v2(this_step_message, step)
+                messages_str_before_this_turn = checklist_reward.get_messages_str_v2(messages[:last_user_message_idx])
+                messages_str_before_this_step = checklist_reward.get_messages_str_v2(messages_before_this_step)
+                following_tool_response_str = "No following tool response"
+                this_turn_messages_util_now = messages[last_user_message_idx:i+1]
+                tool_call_failed = False
+                if i + 1 < len(messages) and messages[i + 1].role in ["observation", "tool"]:
+                    tool_messages = []
+                    j = i + 1
+                    while j < len(messages) and messages[j].role in ["observation", "tool"]:
+                        # Safely check for error_tool_call in message content
+                        try:
+                            content = messages[j].content
+                            if content and isinstance(content, str):
+                                parsed_content = json.loads(content)
+                                if isinstance(parsed_content, dict) and "error_tool_call" in parsed_content:
+                                    tool_call_failed = True
+                        except (json.JSONDecodeError, TypeError):
+                            # Content is not valid JSON or empty, skip error check
+                            pass
+                        tool_messages.append(messages[j])
+                        this_turn_messages_util_now.append(messages[j])
+                        j += 1
+                    following_tool_response_str = checklist_reward.get_messages_str_v2(tool_messages)
+                for single_step_checklist in this_turn_checklist:
+                    # input_prompt = checklist_reward.get_input_prompt(messages_str_before_this_step, this_step_message_str, following_tool_response_str, single_step_checklist)
+                    messages_str_in_this_turn_until_now = checklist_reward.get_messages_str_v2(this_turn_messages_util_now)
+                    input_prompt = checklist_reward.get_input_prompt_v2(messages_str_before_this_turn, messages_str_in_this_turn_until_now, single_step_checklist)
+                    selected_url = random.choice(self._sglang_url) if isinstance(self._sglang_url, list) and self._sglang_url else self._sglang_url
+                    args["sglang_url"] = selected_url
+
+                    async def _guarded_eval(prompt: str) -> bool:
+                        async with self._semaphore:
+                            return await checklist_reward.eval_one_check(self._client, prompt, args)
+                    async def _guarded_eval_tool_error() -> bool:
+                            return False, True  # type: ignore[arg-type]
+                    if (single_step_checklist["focus_on"]=="assistant.tool_calls" or single_step_checklist["focus_on"]=="tool.content") and tool_call_failed:
+                        all_step_results_this_turn.append(asyncio.create_task(_guarded_eval_tool_error()))
+                    else:
+                        all_step_results_this_turn.append(asyncio.create_task(_guarded_eval(input_prompt)))
+                    # await asyncio.sleep(0.001)
+                step += 1
+
+        org_flat_per_step_results: list[(bool, bool)] = await asyncio.gather(*all_step_results_this_turn) # a list of bool lenght is steps * len(this_turn_checklist), the first len(this_turn_checklist) is for step 0
+
+        assert len(org_flat_per_step_results) == len(this_turn_checklist) * (step), f"len(per_step_results) != len(this_turn_checklist) * step, {len(org_flat_per_step_results)} != {len(this_turn_checklist)} * {step}"
+
+        flat_per_step_results = [x[0] for x in org_flat_per_step_results ]
+        flat_per_step_call_success = [x[1] for x in org_flat_per_step_results ]
+
+
+        per_step_results = [flat_per_step_results[i:i+len(this_turn_checklist)] for i in range(0, len(flat_per_step_results), len(this_turn_checklist))] # (step, len(this_turn_checklist))
+        per_step_call_success = [flat_per_step_call_success[i:i+len(this_turn_checklist)] for i in range(0, len(flat_per_step_call_success), len(this_turn_checklist))] # (step, len(this_turn_checklist))
+
+
+        turn = self._instance_dict[instance_id]["turns"]
+        step = 0
+        start = 0
+        per_step_scores = []
+        this_turn_checklist_mask = [1] * len(this_turn_checklist)
+        for i in range(last_user_message_idx+1, len(messages)):
+            message = messages[i]
+            role = message.role
+            assert role != "user"
+
+            if role == "assistant":
+                # calculate the score of this step
+                # If one checklist is completed, later step can not finish it anymore
+                # also check is all required_for_next_turn checklist are satisfied
+                end = start + len(this_turn_checklist)
+                this_step_results = flat_per_step_results[start:end]
+                not_required_for_next_turn_list = [a or b for a,b in zip(not_required_for_next_turn_list, this_step_results)]
+                weights = [float(single_step_checklist["weight"]) for single_step_checklist in this_turn_checklist]
+                this_step_score = sum([weight * result * mask for weight, result, mask in zip(weights, this_step_results, this_turn_checklist_mask)])
+                this_step_score = round(this_step_score, 8)
+                this_turn_checklist_mask = [bool(int(a*(1-b))) for a,b in zip(this_turn_checklist_mask, this_step_results)]
+
+                per_step_scores.append(this_step_score)
+                start = end
+                step += 1
+
+        # reward = round(sum(per_step_scores), 4)
+
+        if self._instance_dict[instance_id]["turns"]+1 < self._instance_dict[instance_id]["num_turns"]:
+            user_idx = 0
+            for i in range(0, len(all_messages)):
+                item = all_messages[i]
+                if item.get("role") == "user":
+                    if user_idx == self._instance_dict[instance_id]["turns"]+1:
+                        response = item.get("content")
+                        logger.debug(f"Proceeding with the next turn response: {response}")
+                        break
+                    user_idx += 1
+            assert response != ""
+            should_terminate_sequence = False
+        else:
+            response = ""
+            should_terminate_sequence = True
+        
+        if not all(not_required_for_next_turn_list):
+            should_terminate_sequence = True
+        
+
+
+
+        return should_terminate_sequence, per_step_results, per_step_call_success
+
+    async def finalize_interaction(self, instance_id: str, **kwargs) -> None:
+        del self._instance_dict[instance_id]
+
diff --git a/verl/tools/base_tool.py b/verl/tools/base_tool.py
index bec813a5..92c3876c 100644
--- a/verl/tools/base_tool.py
+++ b/verl/tools/base_tool.py
@@ -38,7 +38,7 @@ class BaseTool:
         self.tool_schema = tool_schema or self.get_openai_tool_schema()
         assert self.tool_schema is not None, "Tool schema is not set!"
         self.name = self.tool_schema.function.name
-        print(json.dumps(self.tool_schema.model_dump(exclude_unset=True, exclude_none=True), indent=2))
+        # print(json.dumps(self.tool_schema.model_dump(exclude_unset=True, exclude_none=True), indent=2))
 
     def get_openai_tool_schema(self) -> OpenAIFunctionToolSchema:
         return self.tool_schema
diff --git a/verl/tools/mcp_base_tool.py b/verl/tools/mcp_base_tool.py
index 9e1f7db6..58cf8ab9 100644
--- a/verl/tools/mcp_base_tool.py
+++ b/verl/tools/mcp_base_tool.py
@@ -78,6 +78,7 @@ class MCPBaseTool(BaseTool):
             if err_msg:
                 result = err_msg
                 metadata["api_request_error"] = err_msg
+                logger.warning(f"err_msg: {err_msg}")
             else:
                 metadata["api_request_error"] = None
         return result, metadata
diff --git a/verl/tools/mcp_checklist_tool.py b/verl/tools/mcp_checklist_tool.py
new file mode 100644
index 00000000..cddc2dc0
--- /dev/null
+++ b/verl/tools/mcp_checklist_tool.py
@@ -0,0 +1,740 @@
+# Copyright 2025 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import json
+import logging
+import os
+import re
+from typing import Any, Dict, List, Tuple
+import httpx
+from collections import defaultdict
+from mcp.types import ContentBlock, Tool as MCPTool
+import threading
+import pickle
+import hashlib
+import copy
+import random
+
+from verl.tools.mcp_base_tool import MCPBaseTool
+from verl.tools.utils.mcp_clients.McpClientManager import ClientManager
+from .schemas import OpenAIFunctionToolSchema, ToolResponse
+
+logger = logging.getLogger(__name__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+# Suppress third-party library logging to prevent stack traces from being printed
+# logging.getLogger('mcp').setLevel(logging.CRITICAL)
+# logging.getLogger('httpx').setLevel(logging.CRITICAL) 
+# logging.getLogger('httpcore').setLevel(logging.CRITICAL)
+# logging.getLogger('fastmcp').setLevel(logging.CRITICAL)
+
+
+class MCPChecklistTool(MCPBaseTool):
+    # 类级别的数据缓存，键为dataset_path，值为解析后的数据
+    _dataset_cache: Dict[str, Dict[str, Any]] = {}
+    _cache_lock = threading.Lock()
+    # 共享的 HTTP 客户端与并发控制
+    _shared_client: httpx.AsyncClient | None = None
+    _shared_semaphore: asyncio.Semaphore | None = None
+    
+    def __init__(self, config: dict, tool_schema: OpenAIFunctionToolSchema):
+        super().__init__(config, tool_schema)
+        self.return_raw: bool = bool(config.get("return_raw", True))
+
+        self._dataset_path = config.get("dataset_path", None)
+        
+        # 从缓存或新建数据
+        cached_data = self._get_or_load_dataset_data(self._dataset_path)
+        self._id_by_tool_call_response = cached_data["id_by_tool_call_response"]
+        self._id_by_candidate_tools = cached_data["id_by_candidate_tools"]
+        self._id_by_candidate_tools_name = cached_data["id_by_candidate_tools_name"]
+        self._tool_by_name = cached_data["tool_by_name"]
+        self._tools = cached_data["tools"]
+
+        self._sglang_url = config.get("sglang_url", [])
+        self._sglang_model = config.get("sglang_model", None)
+        # self._system_instruction = config.get("system_instruction", None) or (
+        #     "You are a precise tool executor that learns from examples.\n"
+        #     "You will be given:\n"
+        #     "- Tool call JSON Schema\n"
+        #     "- Few-shot examples showing tool calls and their execution results\n"
+        #     "- A new tool call with specific arguments\n"
+        #     "\n"
+        #     "Your task:\n"
+        #     "1) Learn the OUTPUT FORMAT from the provided examples - follow the exact structure, data types, and response patterns\n"
+        #     "2) Ensure FACTUAL CONSISTENCY - your output should align with the factual information demonstrated in the examples\n"
+        #     "3) For the new tool call:\n"
+        #     "   - Apply the learned format to the new arguments\n"
+        #     "   - Maintain factual consistency with example patterns\n"
+        #     "   - If arguments are similar to examples, adapt the example results appropriately\n"
+        #     "   - If arguments are significantly different, generate new results following the learned format and factual patterns\n"
+        #     "   - May need to fix some type or error in the examples\n"
+        #     "4) Handle errors gracefully - if arguments are invalid or missing, return error messages in the same format as examples\n"
+        #     "\n"
+        #     "Critical constraints:\n"
+        #     "- Act as a silent function executor - NO explanations, suggestions, or hints\n"
+        #     "- NO guidance on how to fix errors or improve calls\n"
+        #     "- NO references to examples or comparisons\n"
+        #     "- Return ONLY the raw execution result as valid JSON\n"
+        #     "- For errors, return minimal error information without instructional content\n"
+        #     "\n"
+        #     "Output requirements:\n"
+        #     "- Return ONLY the execution result as valid JSON\n"
+        #     "- No explanations, markdown, or code fences\n"
+        #     "- Use correct JSON data types\n"
+        #     "- Follow the exact output structure learned from examples\n"
+        #     "- Maintain factual consistency with the example patterns\n"
+        # )
+        self._system_instruction = config.get("system_instruction", None) or (
+            "You are a precise tool executor that learns from examples.\n"
+            "You will be given:\n"
+            "- Tool call JSON Schema\n"
+            "- Few-shot examples showing tool calls and their execution results\n"
+            "- A new tool call with specific arguments\n"
+            "\n"
+            "Your task:\n"
+            "1) Learn the OUTPUT FORMAT from the provided examples - follow the exact structure, data types, and response patterns\n"
+            "2) Ensure FACTUAL CONSISTENCY - your output should align with the factual information demonstrated in the examples\n"
+            "3) For the new tool call:\n"
+            "   - Apply the learned format to the new arguments\n"
+            "   - Maintain factual consistency with example patterns\n"
+            "   - If arguments are similar to examples, adapt the example results appropriately\n"
+            "   - If arguments are significantly different, generate new results following the learned format and factual patterns\n"
+            "   - May need to fix some type or error in the examples\n"
+            "4) Handle errors gracefully - if arguments are invalid or missing, return error messages in the same format as examples\n"
+            "\n"
+            "Critical constraints:\n"
+            "- Act as a silent function executor - NO explanations, suggestions, or hints\n"
+            "- NO guidance on how to fix errors or improve calls\n"
+            "- NO references to examples or comparisons\n"
+            "- Return ONLY the raw execution result as valid JSON\n"
+            "- For errors, return minimal error information without instructional content\n"
+            "\n"
+            "Output requirements:\n"
+            "- First do some analysis on how to mock the execution results. Then return ONLY the execution result as valid JSON array or object\n"
+            "- No explanations, markdown, or code fences\n"
+            "- Follow the exact output structure learned from examples\n"
+            "- Maintain factual consistency with the example patterns\n"
+            "Format:\n"
+            "{\n"
+            "  \"analysis\": str,\n"
+            "  \"execution_result\": JSON array or object,\n"
+            "}"
+            ""
+        )
+
+        self._temperature = config.get("temperature", 0.6)
+        self._max_new_tokens = config.get("max_new_tokens", 2048)
+        self._json_retry_attempts = config.get("retry_attempts", 1)
+        self._top_p = config.get("top_p", 0.8)
+        self._max_tokens = config.get("max_tokens", 2048)
+        self._timeout = config.get("timeout", 120)
+        try:
+            self._semaphore_size = int(config.get("semaphore_size", 64))
+        except Exception:
+            self._semaphore_size = 64
+
+        # 初始化全局共享 client 和 semaphore（按首次实例的配置创建）
+        try:
+            timeout_value = float(self._timeout)
+        except Exception:
+            timeout_value = 120.0
+        limits = httpx.Limits(
+            max_connections=max(16, self._semaphore_size),
+            max_keepalive_connections=max(8, self._semaphore_size // 2),
+        )
+        if MCPChecklistTool._shared_client is None:
+            MCPChecklistTool._shared_client = httpx.AsyncClient(
+                timeout=httpx.Timeout(timeout=timeout_value, read=timeout_value, write=timeout_value, connect=timeout_value),
+                limits=limits,
+            )
+        if MCPChecklistTool._shared_semaphore is None:
+            MCPChecklistTool._shared_semaphore = asyncio.Semaphore(self._semaphore_size)
+    @classmethod
+    def _get_or_load_dataset_data(cls, dataset_path: str) -> Dict[str, Any]:
+        """获取或加载数据集数据，使用类级别缓存避免重复加载"""
+        if dataset_path is None:
+            raise ValueError("dataset_path cannot be None")
+            
+        # 使用线程锁确保线程安全
+        with cls._cache_lock:
+            if dataset_path in cls._dataset_cache:
+                logger.info(f"Using cached dataset data for path: {dataset_path}")
+                return cls._dataset_cache[dataset_path]
+            
+            logger.info(f"Loading and caching dataset data for path: {dataset_path}")
+            
+            # 可选：尝试从磁盘缓存加载（适合大数据集）
+            disk_cache_data = cls._try_load_disk_cache(dataset_path)
+            if disk_cache_data:
+                logger.info(f"Loaded dataset from disk cache: {dataset_path}")
+                cls._dataset_cache[dataset_path] = disk_cache_data
+                return disk_cache_data
+            
+            # 加载数据
+            data = cls._load_dataset_data(dataset_path)
+            cls._dataset_cache[dataset_path] = data
+            
+            # 可选：保存到磁盘缓存
+            cls._try_save_disk_cache(dataset_path, data)
+                
+            return data
+    
+    @staticmethod 
+    def _load_dataset_data(dataset_path: str) -> Dict[str, Any]:
+        """加载数据集并构建所需的数据结构"""
+        with open(dataset_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        
+        id_by_tool_call_response = {}
+        id_by_candidate_tools = {}
+        id_by_candidate_tools_name = {}
+        tool_by_name = {}
+        tools = []
+        all_tools = []
+
+        # 构建tool_call_response映射
+        for item in data:
+            tools = json.loads(item["extra_info"]["tools"])
+            all_tools.extend(tools)
+            tool_call_response_map = MCPChecklistTool._get_called_tools_and_response_static(item)
+            id_by_tool_call_response[str(item["extra_info"]["original_index"])] = tool_call_response_map
+        
+        # 构建candidate_tools映射
+        for item in data:
+            tools = json.loads(item["extra_info"]["tools"])
+            id_by_candidate_tools[str(item["extra_info"]["original_index"])] = {tool["function"]["name"]: tool for tool in tools}
+            id_by_candidate_tools_name[str(item["extra_info"]["original_index"])] = [x["function"]["name"] for x in tools]
+
+        # 构建tool映射
+        all_tools_name = set([x["function"]["name"] for x in all_tools])
+        for name in all_tools_name:
+            mcp_tool = MCPTool(name=name, description="", inputSchema={"type": "object", "properties": {}, "required": []})
+            tools.append(mcp_tool)
+            tool_by_name[name] = mcp_tool
+
+        return {
+            "id_by_tool_call_response": id_by_tool_call_response,
+            "id_by_candidate_tools": id_by_candidate_tools,
+            "id_by_candidate_tools_name": id_by_candidate_tools_name,
+            "tool_by_name": tool_by_name,
+            "tools": tools
+        }
+
+    @staticmethod
+    def _get_called_tools_and_response_static(item: Dict[str, Any]) -> List[Any]:
+        """静态版本的get_called_tools_and_response方法，供数据加载使用"""
+        if "extra_info" in item and "messages" in item["extra_info"]:
+            messages = item["extra_info"]["messages"]
+        else:
+            raise ValueError(f"No messages found in item: {item}")
+        
+        results = defaultdict(list)
+        if not isinstance(messages, list):
+            raise ValueError(f"Unexpected messages format: {type(messages)}")
+
+        for i in range(len(messages)):
+            message = messages[i]
+            if message.get("role") != "assistant":
+                continue
+
+            tool_calls = message.get("tool_calls") or []
+            if not tool_calls:
+                continue
+
+            # Collect contiguous following tool messages for this assistant turn
+            following_tool_msgs: List[Dict] = []
+            
+            for j in range(i+1, i+len(tool_calls)+1):
+                assert messages[j].get("role") == "tool" or messages[j].get("role") == "observation", f"Unexpected role: {messages[j].get('role')}"
+                following_tool_msgs.append(messages[j]["content"])
+
+            for call, content in zip(tool_calls, following_tool_msgs, strict=True):
+                results[call["function"]["name"]].append((json.loads(call["function"]["arguments"]), content))
+        return results
+
+    def get_called_tools_and_response(self, item: Dict[str, Any]) -> List[Any]:
+        """实例方法版本，调用静态方法实现"""
+        return self._get_called_tools_and_response_static(item)
+    
+    @staticmethod
+    def _sanitize_text_for_tokenizer(text: Any) -> Any:
+        """Remove invalid Unicode surrogate code points that may break fast tokenizers or downstream consumers.
+
+        This keeps valid non-ASCII characters intact while stripping only the surrogate range U+D800..U+DFFF.
+        Accepts non-str inputs and returns them unchanged for convenience.
+        """
+        if not isinstance(text, str):
+            return text
+        has_surrogate = False
+        sanitized_chars = []
+        for ch in text:
+            code = ord(ch)
+            if 0xD800 <= code <= 0xDFFF:
+                has_surrogate = True
+                continue
+            sanitized_chars.append(ch)
+        if has_surrogate:
+            try:
+                logger.debug("[MCPChecklistTool] Stripped invalid surrogate code points from text.")
+            except Exception:
+                pass
+        return "".join(sanitized_chars)
+    
+    def _format_error(self, code: str, message: str, details: Dict[str, Any] | None = None) -> str:
+        """Return a standardized JSON error string."""
+        payload: Dict[str, Any] = {
+            "error_tool_call": {
+                "code": code,
+                "message": message,
+            }
+        }
+        if details is not None:
+            payload["error_tool_call"]["details"] = details
+        # Ensure ASCII for safety with downstream JSON-only consumers
+        return json.dumps(payload, ensure_ascii=True)
+
+    def _validate_parameters_against_schema(self, original_index: str, parameters: Dict[str, Any]) -> tuple[bool, List[str]]:
+        """Lightweight validation of parameters against OpenAI-style tool schema from dataset.
+
+        Checks:
+        - required fields present
+        - basic type conformity for primitive types
+        - optional strict mode to disallow additional properties
+        """
+        errors: List[str] = []
+        tool_schema: Dict[str, Any] = self._id_by_candidate_tools[original_index][self.name]
+
+        fn = tool_schema.get("function", {}) if isinstance(tool_schema, dict) else {}
+        params_schema = fn.get("parameters", {}) if isinstance(fn, dict) else {}
+
+        if not isinstance(parameters, dict):
+            return False, ["parameters must be a JSON object"]
+
+        properties: Dict[str, Any] = params_schema.get("properties", {}) if isinstance(params_schema, dict) else {}
+        required: List[str] = params_schema.get("required", []) if isinstance(params_schema, dict) else []
+        strict: bool = bool(fn.get("strict", True))
+
+        # required fields
+        for key in required:
+            if key not in parameters:
+                errors.append(f"missing required field: {key}")
+
+        # type checks (primitive only)
+        def _matches_type(value: Any, expected: Any) -> bool:
+            if isinstance(expected, list):
+                return any(_matches_type(value, t) for t in expected)
+            if expected == "string":
+                return isinstance(value, str)
+            if expected == "number":
+                return (isinstance(value, (int, float)) and not isinstance(value, bool))
+            if expected == "integer":
+                return (isinstance(value, int) and not isinstance(value, bool))
+            if expected == "boolean":
+                return isinstance(value, bool)
+            if expected == "null":
+                return value is None
+            if expected == "object":
+                return isinstance(value, dict)
+            if expected == "array":
+                return isinstance(value, list)
+            # unknown type keywords are treated as pass
+            return True
+
+        for key, value in parameters.items():
+            if key not in properties:
+                if strict:
+                    errors.append(f"unexpected field not allowed: {key}")
+                continue
+            prop = properties.get(key, {})
+            expected_type = prop.get("type")
+            if expected_type is not None and not _matches_type(value, expected_type):
+                errors.append(f"field '{key}' type mismatch: expected {expected_type}")
+            # enum constraint
+            if "enum" in prop:
+                enum_values = prop.get("enum")
+                try:
+                    # allow int/float equivalence only if exactly equal (no bool)
+                    if isinstance(value, bool):
+                        in_enum = value in enum_values
+                    else:
+                        in_enum = value in enum_values
+                except Exception:
+                    in_enum = False
+                if not in_enum:
+                    errors.append(f"field '{key}' not in enum: {enum_values}")
+
+        return len(errors) == 0, errors
+    
+    @classmethod
+    def clear_cache(cls, dataset_path: str = None):
+        """清理缓存数据"""
+        with cls._cache_lock:
+            if dataset_path is None:
+                # 清理所有缓存
+                cls._dataset_cache.clear()
+                logger.info("Cleared all dataset cache")
+            else:
+                # 清理特定路径的缓存
+                if dataset_path in cls._dataset_cache:
+                    del cls._dataset_cache[dataset_path]
+                    logger.info(f"Cleared cache for dataset: {dataset_path}")
+    
+    @classmethod
+    def get_cache_info(cls) -> Dict[str, int]:
+        """获取缓存信息"""
+        with cls._cache_lock:
+            return {
+                "cached_datasets": len(cls._dataset_cache),
+                "dataset_paths": list(cls._dataset_cache.keys())
+            }
+    
+    @staticmethod
+    def _get_disk_cache_path(dataset_path: str) -> str:
+        """生成磁盘缓存文件路径"""
+        cache_dir = os.getenv("MCP_CACHE_DIR", "/tmp/mcp_cache")
+        os.makedirs(cache_dir, exist_ok=True)
+        
+        path_hash = hashlib.md5(dataset_path.encode()).hexdigest()
+        return os.path.join(cache_dir, f"dataset_{path_hash}.pkl")
+    
+    @staticmethod
+    def _try_load_disk_cache(dataset_path: str) -> Dict[str, Any]:
+        """尝试从磁盘缓存加载数据"""
+        try:
+            cache_path = MCPChecklistTool._get_disk_cache_path(dataset_path)
+            if not os.path.exists(cache_path):
+                return None
+            
+            # 检查缓存是否过期
+            cache_mtime = os.path.getmtime(cache_path)
+            dataset_mtime = os.path.getmtime(dataset_path)
+            if cache_mtime < dataset_mtime:
+                logger.info(f"Disk cache expired for {dataset_path}")
+                return None
+            
+            with open(cache_path, 'rb') as f:
+                return pickle.load(f)
+        except Exception as e:
+            logger.warning(f"Failed to load disk cache for {dataset_path}: {e}")
+            return None
+    
+    @staticmethod  
+    def _try_save_disk_cache(dataset_path: str, data: Dict[str, Any]):
+        """尝试保存数据到磁盘缓存"""
+        try:
+            cache_path = MCPChecklistTool._get_disk_cache_path(dataset_path)
+            with open(cache_path, 'wb') as f:
+                pickle.dump(data, f)
+            logger.info(f"Saved dataset to disk cache: {dataset_path}")
+        except Exception as e:
+            logger.warning(f"Failed to save disk cache for {dataset_path}: {e}")
+    def get_candidate_tools(self, item: Dict[str, Any]) -> List[Any]:
+        messages = item["extra_info"]["messages"]
+        
+        
+        results = defaultdict(list)
+        if not isinstance(messages, list):
+            raise ValueError(f"Unexpected messages format: {type(messages)}")
+
+        for i in range(len(messages)):
+            message = messages[i]
+            if message.get("role") != "assistant":
+                continue
+
+            tool_calls = message.get("tool_calls") or []
+            if not tool_calls:
+                continue
+
+            # Collect contiguous following tool messages for this assistant turn
+            following_tool_msgs: List[Dict] = []
+            
+            for j in range(i+1, i+len(tool_calls)+1):
+                assert messages[j].get("role") == "tool" or messages[j].get("role") == "observation", f"Unexpected role: {messages[j].get('role')}"
+                following_tool_msgs.append(messages[j]["content"])
+
+            for call, content in zip(tool_calls, following_tool_msgs, strict=True):
+                results[call["function"]["name"]].append((json.loads(call["function"]["arguments"]), content))
+        return results
+
+    async def execute(self, instance_id: str, parameters: dict[str, Any], original_index: str, **kwargs) -> tuple[ToolResponse, float, dict]:
+        original_index = str(original_index)
+        if original_index not in self._id_by_candidate_tools_name:
+            msg = self._format_error(
+                "INDEX_NOT_FOUND",
+                f"original_index {original_index} not found",
+                {"original_index": original_index},
+            )
+            logger.warning(f"[MCPTool] {msg}")
+            return ToolResponse(text=msg), 0.0, {"success": False}
+        if self.name not in self._id_by_candidate_tools_name[original_index]:
+            msg = self._format_error(
+                "TOOL_NOT_AVAILABLE",
+                f"tool {self.name} not available for original_index {original_index}",
+                {"tool": self.name, "original_index": original_index},
+            )
+            logger.warning(f"[MCPTool] {msg}")
+            return ToolResponse(text=msg), 0.0, {"success": False}
+        if original_index not in self._id_by_tool_call_response:
+            msg = self._format_error(
+                "INDEX_NOT_FOUND",
+                f"original_index {original_index} not found in call responses",
+                {"original_index": original_index},
+            )
+            logger.warning(f"[MCPTool] {msg}")
+            return ToolResponse(text=msg), 0.0, {"success": False}
+
+        if not self.name or parameters is None or not isinstance(parameters, dict):
+            msg = self._format_error(
+                "INVALID_PARAMETERS",
+                "'parameters' is missing, empty, or not a JSON object.",
+                {"tool": self.name, "parameters_type": type(parameters).__name__},
+            )
+            logger.warning(f"[MCPTool] {msg}")
+            return ToolResponse(text=msg), 0.0, {"success": False}
+
+
+        tool_call_response_map = self._id_by_tool_call_response[original_index]
+        def _canonicalize_parameters(parameters: Dict[str, Any]) -> Any:
+            # Keep ensure_ascii=False to preserve matching semantics with dataset; do not sanitize here
+            return json.dumps(parameters, sort_keys=True, ensure_ascii=True, separators=(",", ":"))
+        if self.name in tool_call_response_map:
+            tool_call_response = tool_call_response_map[self.name]
+            # find if arguments can match any of the tool call responses
+            for args, content in tool_call_response:
+                if _canonicalize_parameters(args) == _canonicalize_parameters(parameters):
+                    logger.info(f"Found match for tool {self.name} with arguments {parameters}")
+                    safe_text = (
+                        self._sanitize_text_for_tokenizer(content)
+                        if isinstance(content, str)
+                        else json.dumps(content, ensure_ascii=True)
+                    )
+                    return ToolResponse(text=safe_text), 0.0, {"success": True}
+
+        # Validate against schema before any attempt
+        ok, validation_errors = self._validate_parameters_against_schema(original_index, parameters)
+        if not ok:
+            msg = self._format_error(
+                "SCHEMA_VALIDATION_FAILED",
+                "parameters do not conform to the tool schema",
+                {"errors": validation_errors, "tool": self.name, "original_index": original_index},
+            )
+            logger.info(f"[MCPTool] Schema validation failed: {validation_errors}")
+            return ToolResponse(text=msg), 0.0, {"success": True}
+
+
+        # logger.info(f"No match for tool {self.name} with arguments {parameters}, but schema is correct, will call llm to mock the response.")
+        # if valid, call llm to mock the response.
+        # We have valid the tool name is unique, so we can get the schema from the candidate tools.
+        schema_str = json.dumps(self._id_by_candidate_tools[original_index][self.name], ensure_ascii=True, indent=0)
+        schema_str = self._sanitize_text_for_tokenizer(schema_str)
+
+        # Build few-shot examples from this original index across all tool calls
+        examples_lines = ["Previous tool calls and results (few-shot):"]
+        try:
+            for ex_tool_name, pairs in tool_call_response_map.items():
+                for ex_args, ex_content in pairs:
+                    # try:
+                    ex_args_str = json.dumps(ex_args, ensure_ascii=True, indent=0)
+                    ex_args_str = self._sanitize_text_for_tokenizer(ex_args_str)
+                    # except Exception:
+                    #     ex_args_str = str(ex_args)
+                    if isinstance(ex_content, (dict, list)):
+                        try:
+                            ex_content_str = json.dumps(ex_content, ensure_ascii=True, indent=0)
+                            ex_content_str = self._sanitize_text_for_tokenizer(ex_content_str)
+                        except Exception:
+                            raise Exception(f"Error: {ex_content}")
+                    else:
+                        assert isinstance(ex_content, str), f"Unexpected ex_content type: {type(ex_content)}"
+                        ex_content_str = self._sanitize_text_for_tokenizer(ex_content)
+                    examples_lines.append(
+                        "Tool name:\n" + ex_tool_name + "\n"
+                        + "Tool arguments:\n" + ex_args_str + "\n"
+                        + "Tool execution result:\n" + ex_content_str +"\n"
+                    )
+        except Exception:
+            logger.warning("Failed to build few-shot examples, using <unavailable> instead")
+            examples_lines = ["Previous tool calls and results (few-shot): <unavailable>"]
+
+        user_prompt = (
+            "\n".join(examples_lines)
+            + "\n\n"
+            + "Current tool name: "
+            + self.name
+            + "\n"
+            + "Current tool input schema (JSON Schema):\n"
+            + schema_str
+            + "\n"
+            + "Current arguments (JSON):\n"
+            + self._sanitize_text_for_tokenizer(json.dumps(parameters, ensure_ascii=True, indent=0))
+            + "\n"
+            + "Generate tool execution result in JSON format."
+        )
+        user_prompt = self._sanitize_text_for_tokenizer(user_prompt)
+
+        base_messages = [
+            {"role": "system", "content": self._system_instruction},
+            {"role": "user", "content": user_prompt},
+        ]
+
+        async def _single_attempt(attempt_idx: int) -> tuple[int, str]:
+            payload = {
+                "model": self._sglang_model,
+                "messages": base_messages,
+                "temperature": self._temperature,
+                "max_new_tokens": self._max_new_tokens,
+                "max_tokens": self._max_tokens,
+                "top_p": self._top_p,
+                # "json_schema": {
+                #     "type": ["object", "array"]   # 允许对象或数组
+                # },
+                "response_format": {
+                    "type": "json_schema",
+                    "json_schema": {
+                        "name": "tool_execution_response",
+                        "schema": {
+                            "type": "object",
+                            "required": ["analysis", "execution_result"],
+                            "additionalProperties": False,
+                            "properties": {
+                                "analysis": {
+                                    "type": "string",
+                                },
+                                "execution_result": {
+                                    "anyOf": [
+                                        {
+                                            "type": "object",
+                                            "minProperties": 1,
+                                            "additionalProperties": True
+                                        },
+                                        {
+                                            "type": "array",
+                                            "items": {}
+                                        }
+                                    ]
+                                }
+                            }
+                        }
+                    }
+                },
+
+                "sampling_params": {
+                    "temperature": self._temperature,
+                    "max_new_tokens": self._max_new_tokens,
+                    "top_p": self._top_p,
+                    "max_tokens": self._max_tokens,
+                },
+
+
+            }
+            try:
+                # 确保共享资源已初始化（惰性补充 + 健康检查）
+                try:
+                    timeout_value = float(self._timeout)
+                except Exception:
+                    timeout_value = 120.0
+                limits = httpx.Limits(
+                    max_connections=max(16, self._semaphore_size),
+                    max_keepalive_connections=max(8, self._semaphore_size // 2),
+                )
+                client = MCPChecklistTool._shared_client
+                # 如果 client 缺失或已关闭，则重建
+                if client is None or getattr(client, "is_closed", False):
+                    if client is not None:
+                        try:
+                            await client.aclose()
+                        except Exception:
+                            pass
+                    MCPChecklistTool._shared_client = httpx.AsyncClient(
+                        timeout=httpx.Timeout(timeout=timeout_value, read=timeout_value, write=timeout_value, connect=timeout_value),
+                        limits=limits,
+                    )
+                    client = MCPChecklistTool._shared_client
+                if MCPChecklistTool._shared_semaphore is None:
+                    MCPChecklistTool._shared_semaphore = asyncio.Semaphore(self._semaphore_size)
+
+                selected_url = random.choice(self._sglang_url) if isinstance(self._sglang_url, list) and self._sglang_url else self._sglang_url
+                async with MCPChecklistTool._shared_semaphore:
+                    try:
+                        resp = await client.post(selected_url, json=payload)
+                    except (httpx.ReadError, httpx.HTTPError, httpx.TimeoutException, httpx.ConnectError, httpx.RemoteProtocolError, RuntimeError):  # type: ignore[attr-defined]
+                        # 出现网络/关闭异常，重建 client 并重试一次
+                        await asyncio.sleep(5)
+                        try:
+                            await client.aclose()
+                        except Exception:
+                            pass
+                        MCPChecklistTool._shared_client = httpx.AsyncClient(
+                            timeout=httpx.Timeout(timeout=timeout_value, read=timeout_value, write=timeout_value, connect=timeout_value),
+                            limits=limits,
+                        )
+                        client = MCPChecklistTool._shared_client
+                        resp = await client.post(selected_url, json=payload)
+                resp.raise_for_status()
+                data = resp.json()
+                choice = data.get("choices", [{}])[0]
+                text = choice.get("message", {}).get("content", "")
+                text = MCPChecklistTool._sanitize_text_for_tokenizer(text)
+                finish_reason = choice.get("finish_reason", "unknown")
+                return attempt_idx, (text, finish_reason)
+            except Exception as e:  # type: ignore[attr-defined]
+                # Return a JSON error object so upstream never crashes expecting JSON
+                return attempt_idx, None
+
+        # 串行重试方式 - 一个成功就立即返回
+        fallback_text = None
+        fallback_finish_reason = "all failed"
+        error_message = None
+        try:
+            for attempt_idx in range(self._json_retry_attempts):
+                try:
+                    _, result = await _single_attempt(attempt_idx)
+                    if result is None:
+                        continue
+                    text, finish_reason = result
+                    # 保存第一个有效结果作为fallback
+                    # logger.warning(finish_reason)
+                    # if fallback_text is None and finish_reason == "stop":
+                    #     fallback_text = text
+                    #     fallback_finish_reason = "stop but failed to parse json"
+
+                    parsed = json.loads(text)
+                    tool_response = parsed['execution_result']
+                    normalized = json.dumps(tool_response, ensure_ascii=True)
+                    return ToolResponse(text=normalized), 0.0, {"success": True}
+                except Exception as e:
+                    error_message = str(e)
+                    # logger.warning(f"Attempt {attempt_idx} failed with error: {e}, continuing...")
+                    continue
+        except Exception as e:
+            logger.warning(f"Error during serial retry execution: {e}")
+        
+        # 所有尝试都失败了，返回错误信息
+        if not fallback_text:
+            fallback_text = f'{{"error": {error_message}}}'
+            
+        logger.warning(f"Tool {self.name} execution failed after {self._json_retry_attempts} serial attempts: {error_message}")
+        return ToolResponse(text=fallback_text), 0.0, {"success": False}
+        
+
+       
+
+        
+
+
+
+
+        
\ No newline at end of file
diff --git a/verl/tools/schemas.py b/verl/tools/schemas.py
index aa01ae72..41e946f3 100644
--- a/verl/tools/schemas.py
+++ b/verl/tools/schemas.py
@@ -21,9 +21,12 @@ from pydantic import BaseModel, Field, model_validator
 class OpenAIFunctionPropertySchema(BaseModel):
     """The schema of a parameter in OpenAI format."""
 
-    type: str
+    type: str | list[str] = None
     description: str | None = None
-    enum: list[str] | None = None
+    enum: list[str | int | float | bool] | None = None
+    default: str | int | float | bool | None = None
+    minimum: int | float | None = None
+    maximum: int | float | None = None
 
 
 class OpenAIFunctionParametersSchema(BaseModel):
diff --git a/verl/tools/utils/checklist_tool_utils.py b/verl/tools/utils/checklist_tool_utils.py
new file mode 100644
index 00000000..66cadf87
--- /dev/null
+++ b/verl/tools/utils/checklist_tool_utils.py
@@ -0,0 +1,321 @@
+import argparse
+import json
+import os
+import sys
+from typing import Any, Dict, List, Union
+from collections import defaultdict
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Transform checklist JSON: move messages/tools into extra_info and trim messages to first user."
+    )
+    parser.add_argument(
+        "--input_path",
+        help="Absolute path to the source JSON file containing a list of items.",
+    )
+    parser.add_argument(
+        "--output_path",
+        default=None,
+        help="Optional explicit output path. If omitted, writes alongside input with _forverl suffix.",
+    )
+    return parser.parse_args()
+
+
+def compute_output_path(input_path: str, explicit_output: Union[str, None]) -> str:
+    if explicit_output:
+        return explicit_output
+    directory, filename = os.path.split(input_path)
+    name, ext = os.path.splitext(filename)
+    if not ext:
+        ext = ".json"
+    output_filename = f"{name}_all_tools{ext}"
+    return os.path.join(directory, output_filename)
+
+
+def parse_tools_field(tools_value: Any) -> Any:
+    # Tools may be a JSON-encoded string or already a Python object.
+    if isinstance(tools_value, str):
+        return json.loads(tools_value)
+    elif isinstance(tools_value, list):
+        if tools_value and isinstance(tools_value[0], dict):
+            return tools_value
+        else:
+            return json.loads(tools_value)
+    elif isinstance(tools_value, dict):
+        return tools_value
+    else:
+        raise ValueError(f"Unknown tools format: {type(tools_value)}")
+
+def get_called_tools_and_response(item: Dict[str, Any]) -> List[Any]:
+    if "extra_info" in item and "messages" in item["extra_info"]:
+        messages = item["extra_info"]["messages"]
+    else:
+        raise ValueError(f"No messages found in item: {item}")
+    
+    results = defaultdict(list)
+    if not isinstance(messages, list):
+        raise ValueError(f"Unexpected messages format: {type(messages)}")
+
+    for i in range(len(messages)):
+        message = messages[i]
+        if message.get("role") != "assistant":
+            continue
+
+        tool_calls = message.get("tool_calls") or []
+        if not tool_calls:
+            continue
+
+        # Collect contiguous following tool messages for this assistant turn
+        following_tool_msgs: List[Dict] = []
+        
+        for j in range(i+1, i+len(tool_calls)+1):
+            assert messages[j].get("role") == "tool" or messages[j].get("role") == "observation", f"Unexpected role: {messages[j].get('role')}"
+            following_tool_msgs.append(messages[j]["content"])
+
+        for call, content in zip(tool_calls, following_tool_msgs, strict=True):
+            results[call["function"]["name"]].append((json.loads(call["function"]["arguments"]), content))
+    return results
+    
+def get_tools(item: Dict[str, Any]) -> List[Any]:
+    if "tools" in item:
+        tools = item.get("tools")
+    elif "extra_info" in item and "tools" in item["extra_info"]:
+        tools = item["extra_info"]["tools"]
+    else:
+        raise ValueError(f"No tools found in item: {item}")
+
+    if isinstance(tools, str):
+        tools = json.loads(tools)
+    elif not isinstance(tools, list):
+        raise ValueError(f"Unknown tools format: {type(tools)}")
+    
+    if tools is None:
+        return []
+    parsed_tools = parse_tools_field(tools)
+    if isinstance(parsed_tools, list):
+        return parsed_tools
+    elif isinstance(parsed_tools, dict):
+        return [parsed_tools]
+    else:
+        raise ValueError(f"Unknown tools format: {type(parsed_tools)}")
+
+
+def _canonicalize_json_for_compare(value: Any) -> str:
+    try:
+        return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    except TypeError:
+        return str(value)
+
+
+def json_structurally_equal(left_value: Any, right_value: Any) -> bool:
+    """
+    Return True if two JSON-like Python values are structurally equal.
+
+    This comparison ignores dictionary key ordering by comparing their
+    canonical JSON serialization with sorted keys. Lists preserve order
+    as in JSON semantics.
+    """
+    return _canonicalize_json_for_compare(left_value) == _canonicalize_json_for_compare(right_value)
+
+
+def _coerce_schema_to_dict(schema_obj: Any) -> Dict[str, Any]:
+    """
+    Convert a schema-like object into a Python dict.
+    Accepts raw dict, JSON string, or objects exposing model_dump().
+    """
+    if isinstance(schema_obj, dict):
+        return schema_obj
+    if isinstance(schema_obj, str):
+        try:
+            return json.loads(schema_obj)
+        except Exception:
+            # Not a JSON string; treat as empty schema
+            return {}
+    # Try pydantic-like model with model_dump
+    dump_fn = getattr(schema_obj, "model_dump", None)
+    if callable(dump_fn):
+        try:
+            return dump_fn()
+        except Exception:
+            return {}
+    return {}
+
+
+def _json_type_to_python_types(json_type: Any) -> tuple:
+    """
+    Map JSON Schema type(s) to acceptable Python types.
+    Supports single type string or list of type strings.
+    """
+    mapping = {
+        "string": (str,),
+        "number": (int, float),
+        "integer": (int,),
+        "boolean": (bool,),
+        "object": (dict,),
+        "array": (list,),
+        "null": (type(None),),
+    }
+    if isinstance(json_type, list):
+        py_types: tuple = tuple()
+        for t in json_type:
+            py_types += mapping.get(t, (object,))
+        return py_types or (object,)
+    return mapping.get(json_type, (object,))
+
+
+def _validate_value_against_schema(value: Any, schema: Dict[str, Any], path: str = "$") -> None:
+    # Handle enums first (only when enum is a valid iterable of candidates)
+    if isinstance(schema, dict):
+        enum_values = schema.get("enum", None)
+        if enum_values is not None and isinstance(enum_values, (list, tuple, set)):
+            if value not in enum_values:
+                raise ValueError(f"{path}: value {value!r} not in enum {enum_values!r}")
+
+    schema_type = schema.get("type") if isinstance(schema, dict) else None
+
+    # If no type given, accept any value (best-effort)
+    if not schema_type:
+        return
+
+    expected_types = _json_type_to_python_types(schema_type)
+    if not isinstance(value, expected_types):
+        raise ValueError(f"{path}: expected type {schema_type}, got {type(value).__name__}")
+
+    if schema_type == "object":
+        properties: Dict[str, Any] = schema.get("properties", {}) if isinstance(schema, dict) else {}
+        required_fields = schema.get("required", []) if isinstance(schema, dict) else []
+
+        # Check required fields
+        for key in required_fields:
+            if key not in value:
+                raise ValueError(f"{path}: missing required property '{key}'")
+
+        # Validate present properties
+        for key, val in value.items():
+            prop_schema = properties.get(key, {})
+            _validate_value_against_schema(val, prop_schema, path=f"{path}.{key}")
+
+    elif schema_type == "array":
+        items_schema = schema.get("items", {}) if isinstance(schema, dict) else {}
+        for idx, elem in enumerate(value):
+            _validate_value_against_schema(elem, items_schema, path=f"{path}[{idx}]")
+
+
+def validate_arguments(arguments: Dict[str, Any], input_schema: Any) -> None:
+    """
+    Validate tool arguments against a JSON Schema-like object.
+
+    - Ensures required properties exist for object schemas
+    - Performs shallow type checks for primitives, objects, arrays
+    - Supports nested validation for objects/arrays and enum constraints
+    - Raises ValueError with a concise message on first failure
+    """
+    schema = _coerce_schema_to_dict(input_schema)
+
+    # If schema explicitly specifies object, ensure arguments is dict
+    schema_type = schema.get("type") if isinstance(schema, dict) else None
+    if schema_type == "object" and not isinstance(arguments, dict):
+        raise ValueError(f"$: expected type object, got {type(arguments).__name__}")
+
+    # Validate using recursive helper
+    _validate_value_against_schema(arguments, schema, path="$")
+
+
+def _extract_name_and_params(tool: Any) -> Union[None, tuple]:
+    if not isinstance(tool, dict):
+        return None
+    func = tool.get("function")
+    if isinstance(func, dict):
+        name = func.get("name")
+        params = func.get("parameters")
+        return (name, params)
+    # Fallback to top-level name/parameters if present
+    name = tool.get("name")
+    params = tool.get("parameters")
+    return (name, params)
+
+
+def dedupe_and_validate_tools(tools_list: List[Any]) -> List[Any]:
+    name_to_params_canon: Dict[str, str] = {}
+    name_to_tool: Dict[str, Any] = {}
+    seen_full_json: set = set()
+    result: List[Any] = []
+
+    for tool in tools_list:
+        extracted = _extract_name_and_params(tool)
+        if not extracted or extracted[0] is None:
+            # No name available: dedupe by full JSON content
+            full_key = _canonicalize_json_for_compare(tool)
+            if full_key in seen_full_json:
+                continue
+            seen_full_json.add(full_key)
+            result.append(tool)
+            continue
+
+        name, params = extracted
+        params_canon = _canonicalize_json_for_compare(params)
+        if name in name_to_params_canon:
+            if name_to_params_canon[name] != params_canon:
+                # Conflicting schema for same tool name
+                raise ValueError(
+                    f"Conflicting tool definitions for name '{name}': parameter schemas differ.\n"
+                    f"First: {name_to_params_canon[name]}\nNew:   {params_canon}"
+                )
+            # Identical schema -> skip duplicate
+            continue
+        name_to_params_canon[name] = params_canon
+        name_to_tool[name] = tool
+        result.append(tool)
+
+    return result
+
+
+def main() -> None:
+    args = parse_args()
+    input_path = args.input_path
+    output_path = compute_output_path(input_path, args.output_path)
+
+    # Read input JSON
+    try:
+        with open(input_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+    except Exception as exc:
+        print(f"Failed to read JSON from {input_path}: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+    if not isinstance(data, list):
+        print(
+            "Input JSON is expected to be a list of items.",
+            file=sys.stderr,
+        )
+        sys.exit(2)
+
+    all_tools = []
+    # Transform all items
+    for item in data:
+        tools = get_tools(item)
+        all_tools.extend(tools)
+
+    # Dedupe and validate
+    try:
+        all_tools = dedupe_and_validate_tools(all_tools)
+    except ValueError as exc:
+        print(str(exc), file=sys.stderr)
+        sys.exit(4)
+
+
+    # Write output JSON
+    try:
+        with open(output_path, "w", encoding="utf-8") as f:
+            json.dump(all_tools, f, ensure_ascii=False, indent=2)
+            f.write("\n")
+    except Exception as exc:
+        print(f"Failed to write JSON to {output_path}: {exc}", file=sys.stderr)
+        sys.exit(3)
+
+    print(f"Wrote transformed file to: {output_path}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/verl/trainer/ppo/core_algos.py b/verl/trainer/ppo/core_algos.py
index cb09503f..ef6b570d 100644
--- a/verl/trainer/ppo/core_algos.py
+++ b/verl/trainer/ppo/core_algos.py
@@ -34,6 +34,11 @@ from verl.utils import as_torch_index, group_mean_std
 from verl.utils.import_utils import deprecated
 from verl.workers.config import ActorConfig
 
+import logging
+import os
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
 PolicyLossFn = Callable[
     [
         torch.Tensor,  # old_log_prob
@@ -105,6 +110,7 @@ class AdvantageEstimator(str, Enum):
     GPG = "gpg"
     RLOO_VECTORIZED = "rloo_vectorized"
     GRPO_VECTORIZED = "grpo_vectorized"
+    CHECKLIST = "checklist"
 
 
 ADV_ESTIMATOR_REGISTRY: dict[str, Any] = {}
@@ -260,6 +266,189 @@ def compute_gae_advantage_return(
     return advantages, returns
 
 
+@register_adv_est(AdvantageEstimator.CHECKLIST)  # or simply: @register_adv_est("checklist")
+def compute_checklist_step_advantage(
+    token_level_rewards: torch.Tensor,
+    response_mask: torch.Tensor,
+    index: np.ndarray,
+    epsilon: float = 1e-6,
+    norm_adv_by_std_in_grpo: bool = False,
+    checklist_info: dict[str, Any] = None,
+    config: Optional[AlgoConfig] = None,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute advantage for Checklist, operating on Step reward
+    (with only one scalar reward for each response).
+
+    Args:
+        token_level_rewards: `(torch.Tensor)`
+            shape is (bs, response_length)
+        response_mask: `(torch.Tensor)`
+            shape is (bs, response_length)
+        index: `(np.ndarray)`
+            index array for grouping
+        epsilon: `(float)`
+            small value to avoid division by zero
+        norm_adv_by_std_in_checklist: `(bool)`
+            whether to scale the Checklist advantage
+        checklist_info: `(dict[str, Any])`
+            checklist information
+        config: `(Optional[AlgoConfig])`
+            algorithm configuration object
+
+    Note:
+        If norm_adv_by_std_in_grpo is True, the advantage is scaled by the std, as in the original Checklist.
+        If False, the advantage is not scaled, as in Dr.GRPO (https://arxiv.org/abs/2503.20783).
+
+    Returns:
+        advantages: `(torch.Tensor)`
+            shape is (bs, response_length)
+        Returns: `(torch.Tensor)`
+            shape is (bs, response_length)
+    """
+    # scores = token_level_rewards.sum(dim=-1)
+    # step level
+    if checklist_info is not None:
+        if checklist_info.get("adv_tensor", None) is not None:
+            adv_tensor = checklist_info["adv_tensor"]
+            # convert to torch tensor
+            if not isinstance(checklist_info["adv_tensor"], torch.Tensor):
+                adv_tensor = torch.tensor(adv_tensor)
+            adv_tensor = adv_tensor.to(token_level_rewards.device, dtype=token_level_rewards.dtype)
+            return adv_tensor, adv_tensor
+
+    reward_model_call_success = [
+        all([
+            metric["success"]
+            for metrics_list in req_metrics.values()
+            for metric in metrics_list
+            if "success" in metric
+        ])
+        for req_metrics in checklist_info["metrics"]
+    ]
+    # turn level
+    turn_end_tensor = checklist_info.get("turn_end_tensor", None)
+    max_num_turns = checklist_info.get("max_num_turns", None)
+    tool_call_success = checklist_info.get("tool_call_success", None)
+    if turn_end_tensor is None:
+        raise ValueError("turn_end_tensor is required for Checklist step normalization")
+
+    if max_num_turns is None:
+        raise ValueError("max_num_turns is required for Checklist step normalization")
+
+    # Convert turn_end_tensor to torch tensor if it's not already
+    if not isinstance(turn_end_tensor, torch.Tensor):
+        turn_end_tensor = torch.tensor(turn_end_tensor)
+    turn_end_tensor = turn_end_tensor.to(token_level_rewards.device, dtype=token_level_rewards.dtype)
+
+    # Convert max_num_turns to torch tensor if it's not already
+    if not isinstance(max_num_turns, torch.Tensor):
+        max_num_turns = torch.tensor(max_num_turns)
+    max_num_turns = max_num_turns.to(token_level_rewards.device, dtype=token_level_rewards.dtype)
+
+    # Use provided non-normalized reward tensor; if not provided, approximate by undoing per-turn normalization by max_num_turns
+    reward_tensor_non_norm = token_level_rewards * max_num_turns.unsqueeze(-1)
+
+    beta = config.get("beta", 0)
+    with torch.no_grad():
+        device = reward_tensor_non_norm.device
+        dtype = reward_tensor_non_norm.dtype
+        bsz, seq_len = reward_tensor_non_norm.shape
+
+        # Build group -> sample indices mapping
+        id2indices: dict[Any, list[int]] = defaultdict(list)
+        for i in range(bsz):
+            if tool_call_success[i] and reward_model_call_success[i]:
+                id2indices[index[i]].append(i)
+
+        # For each sample, collect end positions and end rewards (ragged lists)
+        sample_end_positions: list[torch.Tensor] = []
+        sample_end_rewards: list[torch.Tensor] = []
+        for i in range(bsz):
+            end_pos_i = torch.nonzero(turn_end_tensor[i], as_tuple=False).squeeze(-1)
+            sample_end_positions.append(end_pos_i)
+            if end_pos_i.numel() > 0:
+                sample_end_rewards.append(reward_tensor_non_norm[i, end_pos_i])
+            else:
+                sample_end_rewards.append(torch.empty((0,), device=device, dtype=dtype))
+
+        # Prepare container for normalized per-turn rewards per sample
+        normalized_end_rewards: list[torch.Tensor] = [torch.empty_like(sample_end_rewards[i]) for i in range(bsz)]
+        total = 0
+        ignored = 0
+        # Per-id, per-position normalization
+        for gid, sample_ids in id2indices.items():
+            # Determine max number of turns observed in this group
+            max_k = 0
+            for si in sample_ids:
+                max_k = max(max_k, sample_end_rewards[si].numel())
+
+            # For each turn position k, compute mean/std over samples that have that k
+            for k in range(max_k):
+                vals_k: list[torch.Tensor] = []
+                holders: list[int] = []
+                for si in sample_ids:
+                    if sample_end_rewards[si].numel() > k:
+                        vals_k.append(sample_end_rewards[si][k].unsqueeze(0))
+                        holders.append(si)
+                if len(vals_k) == 0:
+                    continue
+                vals_k_tensor = torch.cat(vals_k, dim=0)
+                mean_k = torch.mean(vals_k_tensor)
+                if norm_adv_by_std_in_grpo and vals_k_tensor.numel() > 1:
+                    std_k = torch.std(vals_k_tensor)
+                else:
+                    std_k = torch.tensor(1.0, device=device, dtype=dtype)
+                total+=1
+                if len(holders)<=4:
+                    ignored+=1
+                # Write normalized values back
+                for idx_holder, si in enumerate(holders):
+                    if len(holders) <= 4:
+                        norm_val = torch.tensor(float('nan'))
+                    else:
+                        if norm_adv_by_std_in_grpo:
+                            norm_val = (vals_k_tensor[idx_holder] - mean_k) / (std_k + epsilon)
+                            # n = len(holders)
+                            # norm_val *= torch.sqrt(torch.tensor(n*(n-4)/(n-1)/(n-2), device=device, dtype=dtype))
+                            # norm_val /= torch.pow(torch.tensor(n, device=device, dtype=dtype), beta)
+                        else:
+                            norm_val = vals_k_tensor[idx_holder] - mean_k
+                    # Ensure tensor exists and can be indexed
+                    # if normalized_end_rewards[si].numel() == 0:
+                    #     normalized_end_rewards[si] = torch.empty_like(sample_end_rewards[si])
+                    normalized_end_rewards[si][k] = norm_val
+        if total==0:
+            total==-1
+        logger.warning(f"{ignored}/{total} ({ignored/total*100}%) is ignored because group <= 4")
+        # Scatter normalized end rewards across token ranges between consecutive turn ends (left open, right closed)
+        advantages = torch.zeros_like(reward_tensor_non_norm)
+        for i in range(bsz):
+            if not tool_call_success[i] or not reward_model_call_success[i]:
+                # response_mask[i]=False
+                continue
+            ends_i = sample_end_positions[i]
+            if ends_i.numel() == 0:
+                continue
+            prev_end = -1
+            for t_idx, end_pos in enumerate(ends_i.tolist()):
+                # val = normalized_end_rewards[i][t_idx] if normalized_end_rewards[i].numel() > t_idx else torch.tensor(0.0, device=device, dtype=dtype)
+                val = normalized_end_rewards[i][t_idx]
+                # Range (prev_end, end_pos] -> indices prev_end+1 .. end_pos inclusive
+                start = prev_end + 1
+                end = end_pos + 1
+                assert start<end
+                if not torch.isnan(val):
+                    advantages[i, start:end] = val
+                prev_end = end_pos
+
+        # Respect response_mask
+        advantages = advantages * response_mask
+
+    print(advantages)
+    return advantages, advantages
+
+
 # NOTE(sgm): this implementation only consider outcome supervision, where the reward is a scalar.
 @register_adv_est(AdvantageEstimator.GRPO)  # or simply: @register_adv_est("grpo")
 def compute_grpo_outcome_advantage(
diff --git a/verl/trainer/ppo/ray_trainer.py b/verl/trainer/ppo/ray_trainer.py
index eccda70d..7a551561 100644
--- a/verl/trainer/ppo/ray_trainer.py
+++ b/verl/trainer/ppo/ray_trainer.py
@@ -251,6 +251,10 @@ def compute_advantage(
             adv_kwargs["index"] = data.non_tensor_batch["uid"]
         if "reward_baselines" in data.batch:  # optional
             adv_kwargs["reward_baselines"] = data.batch["reward_baselines"]
+        # Pass checklist_info when using CHECKLIST estimator so it can consume precomputed advantages
+        if adv_estimator == AdvantageEstimator.CHECKLIST:
+            adv_kwargs["norm_adv_by_std_in_grpo"] = norm_adv_by_std_in_grpo
+            adv_kwargs["checklist_info"] = data.non_tensor_batch
 
         # calculate advantage estimator
         advantages, returns = adv_estimator_fn(**adv_kwargs)
@@ -424,7 +428,34 @@ class RayPPOTrainer:
                     self.config.critic.optim.total_training_steps = total_training_steps
         except Exception as e:
             print(f"Warning: Could not set total_training_steps in config. Structure missing? Error: {e}")
-
+    def to_serializable(self, x):
+        """将 torch.Tensor / numpy.ndarray / numpy scalar 等转换成 JSON 可序列化的 Python 原生类型。"""
+        # Torch tensor
+        if isinstance(x, torch.Tensor):
+            # detach, move to cpu, convert to numpy
+            x = x.detach().cpu()
+            # 0-d tensor -> python scalar
+            if x.dim() == 0:
+                return x.item()
+            # 否则转成 list（可能是嵌套 list）
+            return x.numpy().tolist()
+        # numpy ndarray
+        if isinstance(x, np.ndarray):
+            if x.shape == ():
+                return x.item()
+            return x.tolist()
+        # numpy scalar (e.g. numpy.int64, numpy.bool_)
+        if isinstance(x, (np.generic, np.number, np.bool_)):
+            return x.item()
+        # python builtins: list/tuple -> 递归处理元素
+        if isinstance(x, (list, tuple)):
+            return [self.to_serializable(i) for i in x]
+        # dict -> 递归处理键值
+        if isinstance(x, dict):
+            return {k: self.to_serializable(v) for k, v in x.items()}
+        # torch device / dtype / other objects -> 转成字符串（备用）
+        # 如果不想字符串化这些，删除下面一行改为 raise 或其它策略
+        return x
     def _dump_generations(self, inputs, outputs, gts, scores, reward_extra_infos_dict, dump_path):
         """Dump rollout/validation samples as JSONL."""
         os.makedirs(dump_path, exist_ok=True)
@@ -441,7 +472,7 @@ class RayPPOTrainer:
 
         for k, v in reward_extra_infos_dict.items():
             if len(v) == n:
-                base_data[k] = v
+                base_data[k] = self.to_serializable(v)
 
         lines = []
         for i in range(n):
@@ -535,6 +566,7 @@ class RayPPOTrainer:
         sample_outputs = []
         sample_gts = []
         sample_scores = []
+        sample_nonnorm_scores = []
         sample_turns = []
         sample_uids = []
 
@@ -611,10 +643,18 @@ class RayPPOTrainer:
             scores = reward_tensor.sum(-1).cpu().tolist()
             sample_scores.extend(scores)
 
+            max_num_turns = result["reward_extra_info"]["max_num_turns"]
+            tool_call_success = result["reward_extra_info"]["tool_call_success"]
+            nonnorm_score = (reward_tensor*max_num_turns.unsqueeze(-1)*tool_call_success.unsqueeze(-1)).sum(-1).cpu().tolist()
+            sample_nonnorm_scores.extend(nonnorm_score)
+
             reward_extra_infos_dict["reward"].extend(scores)
             if "reward_extra_info" in result:
+                not_include_keys = ["num_turns", "turn_end_tensor", "adv_tensor", "max_num_turns", "tool_call_success"]
                 for key, lst in result["reward_extra_info"].items():
-                    reward_extra_infos_dict[key].extend(lst)
+                    if key not in not_include_keys:
+                        reward_extra_infos_dict[key].extend(lst)
+                        print(f"len reward_extra_infos_dict['{key}']: {len(reward_extra_infos_dict[key])}")
 
             # collect num_turns of each prompt
             if "__num_turns__" in test_batch.non_tensor_batch:
@@ -665,6 +705,9 @@ class RayPPOTrainer:
             metric_dict["val-aux/num_turns/max"] = sample_turns.max()
             metric_dict["val-aux/num_turns/mean"] = sample_turns.mean()
 
+        metric_dict["val-core/reward_norm/mean"] = np.mean(sample_nonnorm_scores)
+        metric_dict["val-core/reward_non_norm/mean"] = np.mean(sample_scores)
+
         return metric_dict
 
     def init_workers(self):
@@ -1161,6 +1204,24 @@ class RayPPOTrainer:
                         if reward_extra_infos_dict:
                             batch.non_tensor_batch.update({k: np.array(v) for k, v in reward_extra_infos_dict.items()})
 
+                        # compute sequence-level reward sums and non-zero counts
+                        max_num_turns = reward_extra_infos_dict["max_num_turns"]
+                        reward_tensor_non_norm = (reward_tensor*max_num_turns.unsqueeze(-1)).sum(dim=-1)
+                        reward_tensor_sum = reward_tensor.sum(dim=-1)
+                        metrics.update(
+                            {
+                                "reward/reward/mean": reward_tensor_sum.mean().item(),
+                                "reward/reward/max": reward_tensor_sum.max().item(),
+                                "reward/reward/min": reward_tensor_sum.min().item(),
+                                "reward/reward_non_norm/mean": reward_tensor_non_norm.mean().item(),
+                                "reward/reward_non_norm/max": reward_tensor_non_norm.max().item(),
+                                "reward/reward_non_norm/min": reward_tensor_non_norm.min().item(),
+                                "reward/max_num_turns/mean": max_num_turns.mean().item(),
+                                "reward/max_num_turns/max": max_num_turns.max().item(),
+                                "reward/max_num_turns/min": max_num_turns.min().item(),
+                            }
+                        )
+
                         # compute rewards. apply_kl_penalty if available
                         if self.config.algorithm.use_kl_in_reward:
                             batch, kl_metrics = apply_kl_penalty(
diff --git a/verl/trainer/runtime_env.yaml b/verl/trainer/runtime_env.yaml
index 63750cd7..8a799eb8 100644
--- a/verl/trainer/runtime_env.yaml
+++ b/verl/trainer/runtime_env.yaml
@@ -1,5 +1,6 @@
 working_dir: ./
-excludes: ["/.git/"]
+excludes: ["/.git/","./mem_snapshots"]
 env_vars:
   TORCH_NCCL_AVOID_RECORD_STREAMS: "1"
   CUDA_DEVICE_MAX_CONNECTIONS: "1"
+  ENABLE_CHECKLIST: "1"
diff --git a/verl/utils/dataset/__init__.py b/verl/utils/dataset/__init__.py
index 6032d68c..f9d1d98e 100644
--- a/verl/utils/dataset/__init__.py
+++ b/verl/utils/dataset/__init__.py
@@ -15,5 +15,6 @@
 from .rl_dataset import RLHFDataset
 from .rm_dataset import RMDataset
 from .sft_dataset import SFTDataset
+from .checklist_dataset import ChecklistDataset
 
-__all__ = ["RLHFDataset", "RMDataset", "SFTDataset"]
+__all__ = ["RLHFDataset", "RMDataset", "SFTDataset", "ChecklistDataset"]
diff --git a/verl/utils/dataset/checklist_dataset.py b/verl/utils/dataset/checklist_dataset.py
new file mode 100644
index 00000000..a8e87a85
--- /dev/null
+++ b/verl/utils/dataset/checklist_dataset.py
@@ -0,0 +1,462 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+# Copyright 2023-2024 SGLang Team
+# Copyright 2025 ModelBest Inc. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import logging
+import os
+import re
+import json
+from collections import defaultdict
+from typing import Optional
+
+import datasets
+import numpy as np
+import torch
+from omegaconf import DictConfig, ListConfig
+from torch.utils.data import Dataset
+from transformers import PreTrainedTokenizer, ProcessorMixin
+
+import verl.utils.torch_functional as verl_F
+from verl.utils.model import compute_position_id_with_mask
+from verl.tools.schemas import OpenAIFunctionToolSchema
+from pydantic import ValidationError
+
+logger = logging.getLogger(__name__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+def collate_fn(data_list: list[dict]) -> dict:
+    """
+    Collate a batch of sample dicts into batched tensors and arrays.
+
+    Args:
+        data_list: List of dicts mapping feature names to torch.Tensor or other values.
+
+    Returns:
+        Dict where tensor entries are stacked into a torch.Tensor of shape
+        (batch_size, dims) and non-tensor entries are converted to
+        np.ndarray of dtype object with shape (batch_size,).
+    """
+    tensors = defaultdict(list)
+    non_tensors = defaultdict(list)
+
+    for data in data_list:
+        for key, val in data.items():
+            if isinstance(val, torch.Tensor):
+                tensors[key].append(val)
+            else:
+                non_tensors[key].append(val)
+
+    for key, val in tensors.items():
+        tensors[key] = torch.stack(val, dim=0)
+
+    for key, val in non_tensors.items():
+        non_tensors[key] = np.fromiter(val, dtype=object, count=len(val))
+
+    return {**tensors, **non_tensors}
+
+
+class ChecklistDataset(Dataset):
+    """
+    Load and preprocess RLHF data from Parquet files.
+
+    - Caches files locally.
+    - Reads into a HuggingFace Dataset and tokenizes prompts.
+    - Optionally handles images/videos via a ProcessorMixin.
+    - Filters prompts over a max length.
+    - Supports resuming from checkpoints.
+
+    Args:
+        data_files (str or list): Path(s) to Parquet file(s).
+        tokenizer (PreTrainedTokenizer): For the tokenization of text to token IDs.
+        config (DictConfig): Options like cache_dir, prompt_key, max_prompt_length, truncation, etc.
+        processor (ProcessorMixin, optional): Multimodal preprocessor for images/videos.
+    """
+
+    def __init__(
+        self,
+        data_files: str | list[str],
+        tokenizer: PreTrainedTokenizer,
+        config: DictConfig,
+        processor: Optional[ProcessorMixin] = None,
+        max_samples: int = -1,
+    ):
+        if not isinstance(data_files, list | ListConfig):
+            data_files = [data_files]
+
+        self.data_files = copy.deepcopy(data_files)
+        self.original_data_files = copy.deepcopy(data_files)  # use for resume
+        self.tokenizer = tokenizer
+        self.processor = processor
+        self.config = config
+
+        self.cache_dir = os.path.expanduser(config.get("cache_dir", "~/.cache/verl/rlhf"))
+        self.prompt_key = config.get("prompt_key", "prompt")
+        self.image_key = config.get("image_key", "images")
+        self.video_key = config.get("video_key", "videos")
+        self.max_prompt_length = config.get("max_prompt_length", 1024)
+        self.return_raw_chat = config.get("return_raw_chat", False)
+        self.return_full_prompt = config.get("return_full_prompt", False)
+        self.truncation = config.get("truncation", "error")
+        self.filter_overlong_prompts = config.get("filter_overlong_prompts", True)
+        self.apply_chat_template_kwargs = config.get("apply_chat_template_kwargs", {})
+        self.max_samples = max_samples
+        self.max_num_checklist = config.get("max_num_checklist", None)
+        if self.max_num_checklist is not None:
+            assert self.max_num_checklist > 0, "max_num_checklist must be greater than 0"
+        logger.warning(f"max_num_checklist is set to {self.max_num_checklist}")
+
+        self.num_workers = config.get("filter_overlong_prompts_workers", max(1, os.cpu_count() // 4))
+        self.num_workers = min(self.num_workers, os.cpu_count())
+        self.use_shm = config.get("use_shm", False)
+        self.chat_template_func = config.get("chat_template_func", None)
+        self.need_tools_kwargs = config.get("need_tools_kwargs", False)
+        self.filter_prompts = config.get("filter_prompts", True)
+        self.serialize_dataset = False
+        self.return_multi_modal_inputs = config.get("return_multi_modal_inputs", True)
+
+        self._download()
+        self._read_files_and_tokenize()
+
+    def _download(self, use_origin_parquet=False):
+        from verl.utils.fs import copy_to_local
+
+        data_files = self.data_files if not use_origin_parquet else self.original_data_files
+        for i, parquet_file in enumerate(data_files):
+            self.data_files[i] = copy_to_local(src=parquet_file, cache_dir=self.cache_dir, use_shm=self.use_shm)
+
+    def _read_files_and_tokenize(self):
+        dataframes = []
+        for parquet_file in self.data_files:
+            # read parquet files and cache
+            dataframe = datasets.load_dataset("parquet", data_files=parquet_file)["train"]
+            dataframes.append(dataframe)
+        self.dataframe: datasets.Dataset = datasets.concatenate_datasets(dataframes)
+
+        print(f"dataset len: {len(self.dataframe)}")
+
+        self.dataframe = self._filter_invalid_tool_schemas(self.dataframe)
+
+        self.dataframe = self.maybe_filter_out_long_prompts(self.dataframe)
+
+        # Optionally cap the dataset size for training efficiency or controlled runs
+        if self.max_samples is not None:
+            try:
+                max_samples_int = int(self.max_samples)
+            except Exception as exc:
+                raise ValueError(f"Invalid max_samples value: {self.max_samples}") from exc
+
+            if max_samples_int > 0 and len(self.dataframe) > max_samples_int:
+                indices = np.random.permutation(len(self.dataframe))[:max_samples_int].tolist()
+                self.dataframe = self.dataframe.select(indices)
+                print(f"cap dataset len: {len(self.dataframe)} (random subset)")
+
+    def maybe_filter_out_long_prompts(self, dataframe: datasets.Dataset = None):
+
+        # filter out too long prompts
+        if self.filter_overlong_prompts:
+            tokenizer = self.tokenizer
+            processor = self.processor
+            prompt_key = self.prompt_key
+            image_key = self.image_key
+            video_key = self.video_key
+
+            if processor is not None:
+                from verl.utils.dataset.vision_utils import process_image, process_video
+
+                def doc2len(doc) -> int:
+                    messages = self._build_messages(doc)
+                    tools_data = doc.get("extra_info", {}).get("tools", [])
+                    # Validate and normalize tool schemas to OpenAIFunctionToolSchema dicts
+                    tools = None
+                    if tools_data:
+                        if isinstance(tools_data, str):
+                            import json
+                            tools_data = json.loads(tools_data)
+                        if isinstance(tools_data, list) and len(tools_data) > 0:
+                            validated_tools = []
+                            for tool in tools_data:
+                                try:
+                                    schema = OpenAIFunctionToolSchema.model_validate(tool)
+                                    validated_tools.append(schema.model_dump())
+                                except ValidationError:
+                                    pass
+                            tools = validated_tools if len(validated_tools) > 0 else None
+                    
+                    raw_prompt = self.processor.apply_chat_template(
+                        messages, tools=tools, add_generation_prompt=True, tokenize=False, **self.apply_chat_template_kwargs
+                    )
+                    images = [process_image(image) for image in doc[image_key]] if image_key in doc else None
+                    videos = [process_video(video) for video in doc[video_key]] if video_key in doc else None
+
+                    return len(processor(text=[raw_prompt], images=images, videos=videos)["input_ids"][0])
+
+            else:
+
+                def doc2len(doc) -> int:
+                    tools_data = doc.get("extra_info", {}).get("tools", [])
+                    # Validate and normalize tool schemas to OpenAIFunctionToolSchema dicts
+                    tools = None
+                    if tools_data:
+                        if isinstance(tools_data, str):
+                            import json
+                            tools_data = json.loads(tools_data)
+                        if isinstance(tools_data, list) and len(tools_data) > 0:
+                            validated_tools = []
+                            for tool in tools_data:
+                                try:
+                                    schema = OpenAIFunctionToolSchema.model_validate(tool)
+                                    validated_tools.append(schema.model_dump(exclude_none=True))
+                                except ValidationError:
+                                    pass
+                            tools = validated_tools if len(validated_tools) > 0 else None
+                    
+                    return len(
+                        tokenizer.apply_chat_template(
+                            doc[prompt_key], tools=tools, add_generation_prompt=True, **self.apply_chat_template_kwargs
+                        )
+                    )
+
+            dataframe = dataframe.filter(
+                lambda doc: doc2len(doc) <= self.max_prompt_length-500,
+                num_proc=self.num_workers,
+                desc=f"Filtering prompts longer than {self.max_prompt_length} tokens",
+            )
+
+            print(f"filter dataset len: {len(dataframe)}")
+        return dataframe
+    def _filter_invalid_tool_schemas(self, dataframe):
+        """过滤掉不符合 OpenAIFunctionToolSchema 的样本"""
+        from pydantic import ValidationError
+
+        def is_valid_tool_schema(tools_data):
+            if not tools_data:
+                return True  # 没有工具的样本保留
+            try:
+                if isinstance(tools_data, str):
+                    tools_data = json.loads(tools_data)
+                if not isinstance(tools_data, list):
+                    return False
+                for tool in tools_data:
+                    # 尝试用 Pydantic 校验每个工具
+                    _ = OpenAIFunctionToolSchema.model_validate(tool)
+                return True
+            except (ValidationError, Exception):
+                return False
+
+        def should_keep(doc):
+            tools_data = doc.get("extra_info", {}).get("tools", [])
+            return is_valid_tool_schema(tools_data)
+
+        print("Filtering invalid tool schemas...")
+        filtered_df = dataframe.filter(
+            should_keep,
+            num_proc=1,
+            desc="Filtering invalid tool schemas",
+        )
+        print(f"After filtering invalid tool schemas: {len(filtered_df)} samples remain.")
+        return filtered_df
+
+    def resume_dataset_state(self):
+        self.serialize_dataset = not hasattr(self, "original_data_files")
+        # resume dataframe if not it's serialized in data.pt
+        if not self.serialize_dataset:
+            self._download(use_origin_parquet=True)  # download and resume from original parquet files
+            self._read_files_and_tokenize()
+        else:
+            print(r"old dataloader ckpt file is used, please train from scratch for better ckpt performance")
+
+    def __len__(self):
+        return len(self.dataframe)
+
+    def _build_messages(self, example: dict):
+        messages: list = example.pop(self.prompt_key)
+
+        if self.image_key in example or self.video_key in example:
+            for message in messages:
+                content = message["content"]
+                content_list = []
+                segments = re.split("(<image>|<video>)", content)
+                segments = [item for item in segments if item != ""]
+                for segment in segments:
+                    if segment == "<image>":
+                        content_list.append({"type": "image"})
+                    elif segment == "<video>":
+                        content_list.append({"type": "video"})
+                    else:
+                        content_list.append({"type": "text", "text": segment})
+
+                message["content"] = content_list
+
+        return messages
+
+    def __getitem__(self, item):
+        """
+        Note that we also return the raw_input_ids so that it can be combined with other chat template
+        """
+        row_dict: dict = self.dataframe[item]
+        messages = self._build_messages(row_dict)
+        model_inputs = {}
+
+        if self.processor is not None:
+            from verl.utils.dataset.vision_utils import process_image, process_video
+
+            raw_prompt = self.processor.apply_chat_template(
+                messages, add_generation_prompt=True, tokenize=False, **self.apply_chat_template_kwargs
+            )
+            multi_modal_data = {}
+
+            images = None
+            if self.image_key in row_dict and row_dict.get(self.image_key, None) is not None:
+                images = [process_image(image) for image in row_dict.pop(self.image_key)]
+
+                # due to the image key is "image" instead of "images" in vllm, we need to use "image" here
+                # link: https://github.com/vllm-project/vllm/blob/3c545c0c3b98ee642373a308197d750d0e449403/vllm/multimodal/parse.py#L205
+                multi_modal_data["image"] = images
+
+            videos = None
+            if self.video_key in row_dict and row_dict.get(self.video_key, None) is not None:
+                videos = [process_video(video) for video in row_dict.pop(self.video_key)]
+
+                # due to the video key is "video" instead of "videos" in vllm, we need to use "video" here
+                # link: https://github.com/vllm-project/vllm/blob/3c545c0c3b98ee642373a308197d750d0e449403/vllm/multimodal/parse.py#L205
+                multi_modal_data["video"] = [video.numpy() for video in videos]
+
+            model_inputs = self.processor(text=[raw_prompt], images=images, videos=videos, return_tensors="pt")
+
+            input_ids = model_inputs.pop("input_ids")
+            attention_mask = model_inputs.pop("attention_mask")
+
+            if "second_per_grid_ts" in model_inputs:
+                model_inputs.pop("second_per_grid_ts")
+
+            # There's a trap here, multi_modal_inputs has to be a dict, not BatchFeature
+            row_dict["multi_modal_data"] = multi_modal_data
+
+            # We will do batch.union() in the trainer,
+            # so we cannot have "multi_modal_inputs" in row_dict if rollout generates new multi_modal_inputs
+            if self.return_multi_modal_inputs:
+                row_dict["multi_modal_inputs"] = dict(model_inputs)
+
+                # second_per_grid_ts isn't used for training, just for mrope
+                row_dict["multi_modal_inputs"].pop("second_per_grid_ts", None)
+
+        else:
+            raw_prompt = self.tokenizer.apply_chat_template(
+                messages, add_generation_prompt=True, tokenize=False, **self.apply_chat_template_kwargs
+            )
+            model_inputs = self.tokenizer(raw_prompt, return_tensors="pt", add_special_tokens=False)
+            input_ids = model_inputs.pop("input_ids")
+            attention_mask = model_inputs.pop("attention_mask")
+
+        input_ids, attention_mask = verl_F.postprocess_data(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            max_length=self.max_prompt_length,
+            pad_token_id=self.tokenizer.pad_token_id,
+            left_pad=True,
+            truncation=self.truncation,
+        )
+
+        if self.processor is not None and "Qwen2VLImageProcessor" in self.processor.image_processor.__class__.__name__:
+            from verl.models.transformers.qwen2_vl import get_rope_index
+
+            position_ids = [
+                get_rope_index(
+                    self.processor,
+                    input_ids=input_ids[0],
+                    image_grid_thw=model_inputs.get("image_grid_thw"),
+                    video_grid_thw=model_inputs.get("video_grid_thw"),
+                    second_per_grid_ts=model_inputs.get("second_per_grid_ts"),
+                    attention_mask=attention_mask[0],
+                )
+            ]  # (1, 3, seq_len)
+
+        else:
+            position_ids = compute_position_id_with_mask(attention_mask)
+
+        row_dict["input_ids"] = input_ids[0]
+        row_dict["attention_mask"] = attention_mask[0]
+        row_dict["position_ids"] = position_ids[0]
+
+        raw_prompt_ids = self.tokenizer.encode(raw_prompt, add_special_tokens=False)
+        if len(raw_prompt_ids) > self.max_prompt_length:
+            if self.truncation == "left":
+                raw_prompt_ids = raw_prompt_ids[-self.max_prompt_length :]
+            elif self.truncation == "right":
+                raw_prompt_ids = raw_prompt_ids[: self.max_prompt_length]
+            elif self.truncation == "middle":
+                left_half = self.max_prompt_length // 2
+                right_half = self.max_prompt_length - left_half
+                raw_prompt_ids = raw_prompt_ids[:left_half] + raw_prompt_ids[-right_half:]
+            elif self.truncation == "error":
+                raise RuntimeError(f"Prompt length {len(raw_prompt_ids)} is longer than {self.max_prompt_length}.")
+
+        row_dict["raw_prompt_ids"] = raw_prompt_ids
+        # encode prompts without chat template
+        if self.return_raw_chat:
+            row_dict["raw_prompt"] = messages
+
+        # get prompts with chat template
+        if self.return_full_prompt:
+            row_dict["full_prompts"] = raw_prompt  # array of strings
+
+        # add index for each prompt
+        index = row_dict.get("extra_info").get("index")
+        original_index = row_dict.get("extra_info").get("original_index")
+        tools_kwargs = {}
+
+        tools = row_dict.get("extra_info").get("tools")
+        if isinstance(tools, str):
+            tools = json.loads(tools)
+        elif not isinstance(tools, list):
+            raise ValueError(f"Unknown tools format: {type(tools)}")
+
+
+        for tool in tools:
+            tool_name = tool.get("function").get("name")
+            assert tool_name is not None, f"Tool {tool} is not a valid function"
+            tools_kwargs[tool_name] = {
+                "create_kwargs": {
+                },
+                "execute_kwargs": {
+                    "original_index": original_index,
+                },
+                "calc_reward_kwargs": {},
+                "release_kwargs": {},
+            }
+
+        interaction_kwargs = row_dict.get("extra_info").get("interaction_kwargs")
+        interaction_kwargs["all_messages"] = row_dict.get("extra_info").get(self.prompt_key)
+
+        if self.max_num_checklist is not None:
+            interaction_kwargs["checklist_list"] = interaction_kwargs["checklist_list"][:self.max_num_checklist]
+
+        row_dict["index"] = index
+        row_dict["tools_kwargs"] = tools_kwargs
+        row_dict["interaction_kwargs"] = interaction_kwargs
+        row_dict["tools"] = tools
+        return row_dict
+
+    def __getstate__(self):
+        if not self.serialize_dataset:
+            state = self.__dict__.copy()
+
+            if "dataframe" in state:
+                del state["dataframe"]
+            return state
+
+        return self.__dict__.copy()
diff --git a/verl/utils/reward_score/checklist_reward.py b/verl/utils/reward_score/checklist_reward.py
new file mode 100644
index 00000000..d013e8df
--- /dev/null
+++ b/verl/utils/reward_score/checklist_reward.py
@@ -0,0 +1,646 @@
+import asyncio
+import json
+from collections import defaultdict
+from typing import Any
+import os
+import re
+import httpx
+import torch
+from verl import DataProto
+
+import logging
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+def extract_last_json(text, parse=True):
+    FENCE_JSON_RE = re.compile(
+        r"```(?:\s*json)?\s*(.*?)\s*```",
+        re.IGNORECASE | re.DOTALL
+    )
+    matches = FENCE_JSON_RE.findall(text)
+    if not matches:
+        return None
+    raw = matches[-1].strip()
+    if not parse:
+        return None
+    try:
+        return json.loads(raw)
+    except Exception:
+        return None  # 解析失败就返回原始字符串
+
+
+async def eval_one_check(client: httpx.AsyncClient, user_prompt: str, args: dict) -> bool:
+
+    sglang_model = args.get("sglang_model")
+    sglang_url = args.get("sglang_url")
+    temperature = args.get("temperature")
+    top_p = args.get("top_p")
+    max_new_tokens = args.get("max_new_tokens")
+    max_tokens = args.get("max_tokens")
+    retry_times = args.get("retry_times")
+
+    payload = {
+        "model": sglang_model,
+        "messages": [{"role": "user", "content": user_prompt}],
+        "temperature": temperature,
+        "top_p": top_p,
+        "max_new_tokens": max_new_tokens,
+        "max_tokens": max_tokens,
+        "sampling_params": {
+            "temperature": temperature,
+            "max_new_tokens": max_new_tokens,
+            "top_p": top_p,
+            "max_tokens": max_tokens,
+        },
+        # "json_schema": {
+        #     "type": "object",
+        #     "properties": {
+        #         "result": {"type": "boolean"}
+        #     },
+        #     "required": ["result"]
+        # },
+        # "response_format": {
+        #     "type": "json_schema",
+        #     "json_schema": {
+        #         "name": "foo",
+        #         "schema":{
+        #             "type": "object",
+        #             "properties": {
+        #                 "result": {"type": "boolean"}
+        #             },
+        #             "required": ["result"]
+        #         }
+        #     }
+        # }
+        "response_format": {
+            "type": "json_schema",
+            "json_schema": {
+                "name": "evaluation_verdict",
+                "strict": True,
+                "schema": {
+                    "type": "object",
+                    "title": "Checklist Evaluation Verdict",
+                    "description": "Structured evaluation result for a single assistant turn against checklist criteria.",
+                    "properties": {
+                        "high_level_understanding_of_the_question": {
+                            "type": "string",
+                        },
+                        "analysis_of_if_focus_on": {
+                            "type": "string",
+                        },
+                        "analysis_of_pass_condition": {
+                            "type": "string",
+                        },
+                        "analysis_of_failure_examples": {
+                            "type": "string",
+                        },
+                        "answer": {
+                            "type": "boolean",
+                        }
+                    },
+                    "required": [
+                        "high_level_understanding_of_the_question",
+                        "analysis_of_if_focus_on",
+                        "analysis_of_pass_condition",
+                        "analysis_of_failure_examples",
+                        "answer"
+                    ],
+                    "additionalProperties": False
+                }
+            }
+        }
+    }
+
+    try:
+        resp = await _post_with_retries(client, sglang_url, payload, retry_times)
+        data = resp.json()
+        text = data.get("choices", [{}])[0].get("message", {}).get("content", "")
+        try:
+            # parsed = json.loads(text)
+            parsed = json.loads(text)
+            ans = parsed['answer']
+            if ans not in [True, False]:
+                raise ValueError("answer is not boolen")
+            return ans, True
+        except Exception as e:
+            logger.warning(f"text can not be parsed in reward (call passed): {repr(e)}")
+            return False, False
+    except Exception as e:
+        logger.warning(f"text can not be parsed in reward (call not passed): {repr(e)}")
+        return False, False
+
+def get_input_prompt(messages_str_before_this_step: str, this_step_message_str: str, following_tool_response_str: str, this_turn_checklist: list[dict[str, Any]]) -> str:
+    reference_snippet = [evidence['snippet'] for evidence in this_turn_checklist['evidence']]
+    input_prompt = (
+        "# Instructions\n"
+        "You are a strict checklist evaluator.\n"
+        "You will be give a checklist for the assistant's new response.\n"
+        "Checklist contains question, focus_on, pass_condition, failure_examples and reference snippet.\n"
+        "Focus on is the part of the assistant's new response that the question is about.\n"
+        "If the assistant's response follows the checklist's question, return true. Otherwise, return false.\n"
+        "If the focus on is not in the assistant's new response or following tool response, return false.\n"
+        "\n"
+        "# Checklist:\n"
+        f"Question: {this_turn_checklist['question']}\n"
+        f"Focus on: {this_turn_checklist['focus_on']}\n"
+        f"Pass condition: {this_turn_checklist['pass_condition']}\n"
+        f"Failure examples: {json.dumps(this_turn_checklist['failure_examples'], ensure_ascii=False, indent=0)}\n"
+        f"Reference snippet: {json.dumps(reference_snippet, ensure_ascii=False, indent=0)}\n"
+        "\n"
+        "# Previous messages:\n" + messages_str_before_this_step + "\n"
+        "\n"
+        "# Assistant's new response:\n" + this_step_message_str + "\n"
+        "\n"
+        "# Following tool response:\n" + following_tool_response_str + "\n"
+        "# Response format:\n"
+        "Return in JSON format: {'result': true/false}"
+    )
+    return input_prompt
+
+def get_input_prompt_v2(messages_str_before_this_turn, messages_str_in_this_turn: str, this_turn_checklist: list[dict[str, Any]]) -> str:
+    reference_snippet = [evidence['snippet'] for evidence in this_turn_checklist['evidence']]
+    # input_prompt = (
+    #     "# Instructions\n"
+    #     "You are a strict checklist evaluator.\n"
+    #     "You will be give messages between user, assistant and tools. And you will also be given a checklist for the assistant's response.\n"
+    #     "Checklist contains question, focus_on, pass_condition, failure_examples and reference snippet.\n"
+    #     "Focus on is the part of the assistant's response that the question is about.\n"
+    #     "If the assistant's response follows the checklist's question and pass condition, return true. Otherwise, return false.\n"
+    #     "If the focus on is not in the messages, return false.\n"
+    #     "\n"
+    #     "# Checklist:\n"
+    #     f"Question: {this_turn_checklist['question']}\n"
+    #     f"Focus on: {this_turn_checklist['focus_on']}\n"
+    #     f"Pass condition: {this_turn_checklist['pass_condition']}\n"
+    #     f"Failure examples: {json.dumps(this_turn_checklist['failure_examples'], ensure_ascii=False, indent=0)}\n"
+    #     f"Reference snippet: {json.dumps(reference_snippet, ensure_ascii=False, indent=0)}\n"
+    #     "\n"
+    #     "# Messages:\n" + messages_str_in_this_turn + "\n"
+    #     "# Response format:\n"
+    #     "Return only in JSON format: {'result': true/false}"
+    # )
+    input_prompt = (
+        "# Role\n"
+        "You are a precise checklist evaluator. Your sole task is to judge whether the messages between user, assistant and tool satisfie the provided criteria.\n"
+        "\n"
+        "# Objective\n"
+        "Produce a strict JSON verdict (no extra text) based on the instructions below.\n"
+        "\n"
+        "# Criteria\n"
+        f"**Question:** {this_turn_checklist['question']}\n"
+        f"**Focus on:** {this_turn_checklist['focus_on']}\n"
+        f"**Pass condition:** {this_turn_checklist['pass_condition']}\n"
+        f"**Failure examples:** {json.dumps(this_turn_checklist['failure_examples'], ensure_ascii=True, indent=2)}\n"
+        f"**Reference snippet:** {json.dumps(reference_snippet, ensure_ascii=True, indent=2)}\n"
+        "\n"
+        "# Previous Messages\n"
+        + messages_str_before_this_turn +
+        "# Current Messages to Evaluate\n"
+        + messages_str_in_this_turn +
+        "\n"
+        "# Special rule of tool call\n"
+        "If there is no tool call in tool_call part but there are some tool calls in content.thinking part, it means these tools' format are not correct and all tool calls are not valid."
+        "If there is error in tool response. The previous tool calls in latest assistant (only the latest one) are not valid."
+        "# Evaluation Process (Align each step to a JSON output field)\n"
+        "1. high_level_understanding_of_the_question:\n"
+        "   - Briefly restate what is being evaluated (the intent of the question + what compliance means here).\n"
+        "2. analysis_of_if_focus_on:\n"
+        "   - Check whether Focus on part presents in the Current Messages.\n"
+        "3. analysis_of_pass_condition:\n"
+        "   - Determine if the 'Pass condition' is fully satisfied.\n"
+        "4. analysis_of_failure_examples:\n"
+        "   - For EACH failure example pattern: state clearly 'triggered' or 'not triggered' with a brief justification.\n"
+        "5. answer:\n"
+        "   - Return true ONLY IF:\n"
+        "     * Focus on part is present.\n"
+        "     * The 'Pass condition' is fully met.\n"
+        "     * No failure example pattern is triggered.\n"
+        "   - Otherwise return false.\n"
+        "\n"
+        "# Output Format\n"
+        "Return ONLY a single JSON object with exactly these keys:\n"
+        "{\n"
+        "  \"high_level_understanding_of_the_question\": str,\n"
+        "  \"analysis_of_if_focus_on\": str,\n"
+        "  \"analysis_of_pass_condition\": str,\n"
+        "  \"analysis_of_failure_examples\": str,\n"
+        "  \"answer\": bool\n"
+        "}"
+    )
+
+    # input_prompt = (
+    #     "# Task\n"
+    #     "You are a precise checklist evaluator that determines whether the messages between user, assistant and tool meet specific criteria.\n"
+    #     "\n"
+    #     "# Evaluation\n"
+    #     "1. Locate the 'Focus on' content in the messages\n"
+    #     "2. Check if the messages satisfies the 'Pass condition' of the question\n"
+    #     "3. Compare against 'Failure examples' to avoid common mistakes\n"
+    #     "4. Use 'Reference snippet' as a benchmark for expected quality if needed\n"
+    #     "\n"
+    #     "# Evaluation Criteria\n"
+    #     f"**Question:** {this_turn_checklist['question']}\n"
+    #     f"**Focus on:** {this_turn_checklist['focus_on']}\n"
+    #     f"**Pass condition:** {this_turn_checklist['pass_condition']}\n"
+    #     f"**Failure examples:** {json.dumps(this_turn_checklist['failure_examples'], ensure_ascii=True, indent=2)}\n"
+    #     f"**Reference snippet:** {json.dumps(reference_snippet, ensure_ascii=True, indent=2)}\n"
+    #     "\n"
+    #     "# Decision Rules\n"
+    #     "- Return `true` ONLY if:\n"
+    #     "  * The 'Focus on' content is present in the assistant's response\n"
+    #     "  * The response meets the 'Pass condition'\n"
+    #     "  * The response avoids patterns shown in 'Failure examples'\n"
+    #     "- Return `false` if:\n"
+    #     "  * The 'Focus on' content is missing\n"
+    #     "  * The 'Pass condition' is not satisfied\n"
+    #     "  * The response matches any 'Failure examples'\n"
+    #     "# Special criteria: Tool call format\n"
+    #     "Each tool call made by the assistant must strictly adhere to the following format:\n"
+    #     "<tool_call>\n{\"name\": \"tool_name\", \"arguments\": { \"key\": \"value\" }}\n</tool_call>\n"
+    #     "If a tool call deviates from this format, this tool call is not a valid tool call and should be considered incorrect.\n"
+    #     "\n"
+    #     "# Previous Messages\n"
+    #     + messages_str_before_this_turn + 
+    #     "# Messages to Evaluate\n"
+    #     + messages_str_in_this_turn + 
+    #     "\n\n# Output\n"
+        
+    #     "{\"high_level_understanding_of_the_question\": str,
+    # "analysis_of_if_focus_on": str,
+    # "analysis_of_pass_condition": str,
+    # "analysis_of_failure_examples": str,
+    # "answer": bool}"    
+    
+    return input_prompt
+
+def get_messages_str_v1(messages: list[dict[str, Any]], step_num: int=None, max_length: int=40000) -> str:
+
+    # TODO: limit the length of the messages_str
+    if step_num is not None and messages[0].role == "assistant":
+        assert len(messages) == 1, "Only one message is allowed when step_num is not None"
+    turn = -1
+    step = 0
+    thinking_regex = re.compile(r"<think>(.*?)</think>", re.DOTALL)
+    tool_call_regex = re.compile(r"<tool_call>(.*?)</tool_call>", re.DOTALL)
+
+    messages_str = ""
+
+    for i, message in enumerate(messages):
+        role = message.role
+        content = message.content
+        if role == "assistant":
+            if_thinking = thinking_regex.search(content)
+            if if_thinking:
+                thinking = if_thinking.group(1)
+                user_visible_reply = content.split("</think>")[1].strip()
+                if user_visible_reply == "":
+                    user_visible_reply = "None"
+            else:
+                thinking = content
+                user_visible_reply = "None"
+            tool_calls = json.dumps([fm.model_dump() for fm in message.tool_calls] if message.tool_calls else [])
+        
+        if role == "system":
+            messages_str += f"Role: system\ncontent: {content}\n"
+            step = 0
+        elif role == "user":
+            turn += 1
+            step = 0
+            messages_str += f"# Turn: {turn}\nRole: user\ncontent: {content}\n"
+        elif role == "assistant":
+            if step_num is not None:
+                _step = step_num
+            else:
+                _step = step
+            this_step_message = f"## Step: {_step}\nRole: assistant\ncontent.thinking: {thinking}\ncontent.user_visible_reply: {user_visible_reply}\ntool_call: {tool_calls}\n"
+            # for single_checklist in checklist[turn]:
+            #     user_prompt = get_user_prompt_per_step(messages_before_this_step, this_step_message, single_checklist)
+            #     all_step_results.append(eval_one_check(client, user_prompt, args))
+            messages_str += this_step_message
+            step += 1
+        elif role == "observation" or role == "tool":
+            messages_str += f"Role: tool\ncontent: {content}\n"
+    return messages_str
+
+
+def get_messages_str_v2(messages: list[dict[str, Any]], step_num: int=None, max_length: int=40000) -> str:
+
+    # TODO: limit the length of the messages_str
+    if step_num is not None and messages[0].role == "assistant":
+        assert len(messages) == 1, "Only one message is allowed when step_num is not None"
+    turn = -1
+    step = 0
+    thinking_regex = re.compile(r"<think>(.*?)</think>", re.DOTALL)
+    tool_call_regex = re.compile(r"<tool_call>(.*?)</tool_call>", re.DOTALL)
+
+    messages_str = ""
+
+    for i, message in enumerate(messages):
+        role = message.role
+        content = message.content
+        if role == "assistant":
+            if_thinking = thinking_regex.search(content)
+            if if_thinking:
+                thinking = if_thinking.group(1)
+                user_visible_reply = content.split(thinking+"</think>")[1].strip()
+                if user_visible_reply == "":
+                    user_visible_reply = "None"
+            else:
+                thinking = content
+                user_visible_reply = "None"
+            if message.tool_calls is not None:
+                tool_calls = json.dumps([fm.model_dump() for fm in message.tool_calls] if message.tool_calls else [])
+            elif "<tool_call>" in user_visible_reply or "</tool_call>" in user_visible_reply:
+                tool_calls = user_visible_reply.replace("<tool_call>", "<|tool_call_start|>").replace("</tool_call>", "<|tool_call_end|>")
+                user_visible_reply = "None"
+            else:
+                tool_calls = "None"
+        
+        if role == "system":
+            messages_str += f"Role: system\ncontent: {content}\n"
+            step = 0
+        elif role == "user":
+            turn += 1
+            step = 0
+            messages_str += f"# Turn: {turn}\nRole: user\ncontent: {content}\n"
+        elif role == "assistant":
+            if step_num is not None:
+                _step = step_num
+            else:
+                _step = step
+            this_step_message = f"## Step: {_step}\nRole: assistant\ncontent.thinking: {thinking}\ncontent.user_visible_reply: {user_visible_reply}\ntool_call: {tool_calls}\n"
+            # for single_checklist in checklist[turn]:
+            #     user_prompt = get_user_prompt_per_step(messages_before_this_step, this_step_message, single_checklist)
+            #     all_step_results.append(eval_one_check(client, user_prompt, args))
+            messages_str += this_step_message
+            step += 1
+        elif role == "observation" or role == "tool":
+            messages_str += f"Role: tool\ncontent: {content}\n"
+    return messages_str
+
+# async def get_checklist_scores_per_step(messages_before_this_step: list[dict[str, Any]], this_step_message: str, this_turn_checklist: list[dict[str, Any]], args: dict) -> tuple[list[float], list[int]]:
+#     turn = -1
+#     step = 0
+#     thinking_regex = re.compile(r"<think>(.*?)</think>", re.DOTALL)
+#     tool_call_regex = re.compile(r"<tool_call>(.*?)</tool_call>", re.DOTALL)
+
+#     messages_before_this_step = ""
+#     all_step_results = []
+#     # semaphore = asyncio.Semaphore(self._semaphore_size)
+#     async with httpx.AsyncClient(timeout=httpx.Timeout(timeout=1200.0, read=1200.0, write=1200.0, connect=1200.0)) as client:
+#         for i, message in enumerate(messages_before_this_step):
+#             role = message.role
+#             content = message.content
+#             if role == "assistant":
+#                 if_thinking = thinking_regex.search(content)
+#                 if if_thinking:
+#                     thinking = if_thinking.group(1)
+#                     user_visible_reply = content.split("</think>")[1]
+#                 else:
+#                     thinking = content
+#                     user_visible_reply = ""
+#                 tool_calls = json.dumps([fm.model_dump() for fm in message.tool_calls] if message.tool_calls else [])
+            
+#             if role == "system":
+#                 messages_before_this_step += f"Role: system\nContent: {content}\n"
+#                 step = 0
+#             elif role == "user":
+#                 turn += 1
+#                 step = 0
+#                 messages_before_this_step += f"# Turn: {turn}\nRole: user\nContent: {content}\n"
+#             elif role == "assistant":
+#                 this_step_message = f"## Step: {step}\nRole: assistant\nThinking: {thinking}\nUser visible reply: {user_visible_reply}\nTool calls: {tool_calls}\n"
+#                 for single_checklist in checklist[turn]:
+#                     user_prompt = get_user_prompt_per_step(messages_before_this_step, this_step_message, single_checklist)
+#                     all_step_results.append(eval_one_check(client, user_prompt, args))
+#                 messages_before_this_step += this_step_message
+#                 step += 1
+#             elif role == "observation" or role == "tool":
+#                 messages_before_this_step += f"Role: tool response\nContent: {content}\n"
+
+    
+#         all_step_results = await asyncio.gather(*all_step_results)
+
+#     turn = -1
+#     step = 0
+#     start = 0
+#     all_step_scores = []
+#     turns = []
+#     for i, message in enumerate(messages):
+#         role = message.role
+        
+#         if role == "user":
+#             step = 0
+#             turn += 1
+#             this_turn_checklist_mask = [1] * len(checklist[turn])
+#         elif role == "assistant":
+#             this_turn_checklist = checklist[turn]
+#             end = start + len(this_turn_checklist)
+#             this_step_results = all_step_results[start:end]
+#             weights = [float(single_checklist["weight"]) for single_checklist in this_turn_checklist]
+#             this_step_score = sum([weight * result * mask for weight, result, mask in zip(weights, this_step_results, this_turn_checklist_mask)])
+#             this_turn_checklist_mask = [a*(1-b) for a,b in zip(this_turn_checklist_mask, this_step_results)]
+#             this_step_score = round(this_step_score, 4)
+#             all_step_scores.append(this_step_score)
+#             turns.append(turn)
+#             start = end
+#             step += 1
+
+#     return all_step_scores, turns
+
+
+
+# def get_user_prompt_per_step(messages_before_this_step, this_step_message, this_turn_checklist):
+#     user_prompt = (
+#         "If the assistant's response follows the checklist, return True. Otherwise, return False.\n"
+#         + "Checklist: " + json.dumps(this_turn_checklist, ensure_ascii=False, indent=0) + "\n"
+#         + "Previous messages: " + json.dumps(messages_before_this_step, ensure_ascii=False, indent=0) + "\n"
+#         + "Assistant's new response: " + json.dumps(this_step_message, ensure_ascii=False, indent=0) + "\n"
+#         + "Return in JSON format: {'result': True/False}"
+#     )
+#     return user_prompt
+
+async def get_checklist_scores_multiturn_multistep(
+    messages,
+    checklist,
+    args: dict,
+    calculated_rewards: list[float] | None = None,
+    calculated_turns: list[int] | None = None,
+    calculated_call_success: list[int] | None = None,
+    client: httpx.AsyncClient | None = None,
+    semaphore: asyncio.Semaphore | None = None,
+) -> tuple[list[float], list[int]]:
+    # Count the number of assistant messages
+    assistant_count = sum(1 for message in messages if message.role == "assistant")
+    if calculated_rewards is not None:
+        if assistant_count == len(calculated_rewards):
+            return calculated_rewards, calculated_turns, calculated_call_success
+
+    turns = []
+    all_step_scores = []
+    call_success = []
+
+    # if calculated_turns is not None:
+    #     if len(calculated_turns) == 0:
+    #         start_turn = 0
+    #     else:
+    #         start_turn = calculated_turns[-1]
+
+    #     for i, calculated_turn in enumerate(calculated_turns):
+    #         if calculated_turn < start_turn:
+    #             all_step_scores.append(calculated_rewards[i])
+    #             turns.append(calculated_turn)
+    #         else:
+    #             break
+
+    if calculated_rewards is not None:
+        if len(calculated_rewards) == 0:
+            start_step = 0
+        else:
+            start_step = len(calculated_rewards)
+
+        for i, calculated_reward in enumerate(calculated_rewards):
+            all_step_scores.append(calculated_reward)
+            turns.append(calculated_turns[i])
+            call_success.append(calculated_call_success[i])
+
+    turn = -1
+    step = 0
+    
+    # Setup shared client and semaphore if not provided by caller
+    # local_client: httpx.AsyncClient | None = None
+    # if client is None:
+    #     semaphore_size = int(args.get("semaphore_size", 64))
+    #     timeout_seconds = float(args.get("timeout_seconds", 120.0))
+    #     limits = httpx.Limits(
+    #         max_connections=max(16, semaphore_size),
+    #         max_keepalive_connections=max(8, semaphore_size // 2),
+    #     )
+    #     timeout = httpx.Timeout(timeout=timeout_seconds, read=timeout_seconds, write=timeout_seconds, connect=timeout_seconds)
+    #     local_client = httpx.AsyncClient(timeout=timeout, limits=limits)
+    #     client = local_client
+    # if semaphore is None:
+    #     semaphore = asyncio.Semaphore(int(args.get("semaphore_size", 64)))
+
+    all_step_results = []
+    global_assistant_count = 0
+    try:
+        for i, message in enumerate(messages):
+            role = message.role
+            
+            if role == "system":
+                step = 0
+            elif role == "user":
+                turn += 1
+                step = 0
+            elif role == "assistant":
+                if calculated_rewards is not None and start_step > global_assistant_count:
+                        pass # already calculated
+                else:
+                    this_step_message = [messages[i]]
+                    messages_before_this_step = messages[:i]
+                    this_step_message_str = get_messages_str_v2(this_step_message, step)
+                    messages_str_before_this_step = get_messages_str_v2(messages_before_this_step)
+                    # Get following tool response if next message is a tool
+                    following_tool_response_str = "No following tool response"
+                    last_user_message_idx = -1
+                    for i in range(len(messages)-1, -1, -1):
+                        if messages[i].role == "user":
+                            last_user_message_idx = i
+                            break
+                    assert last_user_message_idx != -1
+                    messages_str_before_this_turn = get_messages_str_v2(messages[:last_user_message_idx])
+                    this_turn_messages_util_now = messages[last_user_message_idx:i+1]
+                    tool_call_failed = False
+                    if i + 1 < len(messages) and messages[i + 1].role in ["observation", "tool"]:
+                        tool_messages = []
+                        j = i + 1
+                        while j < len(messages) and messages[j].role in ["observation", "tool"]:
+                            if "error_tool_call" in json.loads(messages[j].content):
+                                tool_call_failed = True
+                            tool_messages.append(messages[j])
+                            this_turn_messages_util_now.append(messages[j])
+                            j += 1
+                        following_tool_response_str = get_messages_str_v2(tool_messages)
+                    for single_step_checklist in checklist[turn]:
+                        # input_prompt = get_input_prompt(messages_str_before_this_step, this_step_message_str, following_tool_response_str, single_step_checklist)
+                        messages_str_in_this_turn_until_now = get_messages_str_v2(this_turn_messages_util_now)
+                        input_prompt = get_input_prompt_v2(messages_str_before_this_turn, messages_str_in_this_turn_until_now, single_step_checklist)
+                        async def _guarded_eval(prompt: str) -> bool:
+                            async with semaphore:  # type: ignore[arg-type]
+                                return await eval_one_check(client, prompt, args)  # type: ignore[arg-type]
+                        async def _guarded_eval_tool_error() -> bool:
+                            return False, True  # type: ignore[arg-type]
+                        if (single_step_checklist["focus_on"]=="assistant.tool_calls" or single_step_checklist["focus_on"]=="tool.content") and tool_call_failed:
+                            all_step_results.append(_guarded_eval_tool_error())
+                        else:
+                            all_step_results.append(_guarded_eval(input_prompt))
+                step += 1
+                global_assistant_count += 1
+            elif role == "observation" or role == "tool":
+                pass
+
+        all_step_results = await asyncio.gather(*all_step_results)
+    finally:
+        # if local_client is not None:
+        #     await local_client.aclose()
+        pass
+
+
+    turn = -1
+    step = 0
+    start = 0
+    global_assistant_count = 0
+
+    for i, message in enumerate(messages):
+        role = message.role
+        if role == "system":
+            step = 0
+        elif role == "user":
+            step = 0
+            turn += 1
+            this_turn_checklist_mask = [1] * len(checklist[turn])
+        elif role == "assistant":
+            if calculated_rewards is not None and start_step > global_assistant_count:
+                pass
+            else:
+                this_turn_checklist = checklist[turn]
+                end = start + len(this_turn_checklist)
+                this_step_results = all_step_results[start:end]
+                all_step_scores.append([ x[0] for x in this_step_results])
+                call_success.append([ x[1] for x in this_step_results])
+                # weights = [float(single_checklist["weight"]) for single_checklist in this_turn_checklist]
+                # this_step_score = sum([weight * result * mask for weight, result, mask in zip(weights, this_step_results, this_turn_checklist_mask)])
+                # this_turn_checklist_mask = [a*(1-b) for a,b in zip(this_turn_checklist_mask, this_step_results)]
+                # this_step_score = round(this_step_score, 8)
+                # all_step_scores.append(this_step_score)
+                turns.append(turn)
+                start = end
+            step += 1
+            global_assistant_count += 1
+
+    return all_step_scores, turns, call_success
+
+async def _post_with_retries(client: httpx.AsyncClient, url: str, json_payload: dict, retry_times: int = 3) -> httpx.Response:
+    """Post with retries and basic backoff. Caller should handle failures."""
+
+
+    last_exc: Exception | None = None
+    for attempt in range(max(1, int(retry_times))):
+        try:
+            resp = await client.post(url, json=json_payload)
+            resp.raise_for_status()
+            data = resp.json()
+            text = data.get("choices", [{}])[0].get("message", {}).get("content", "")
+            # print(text,flush=True)
+            parsed = json.loads(text)
+            return resp
+        except Exception as e:  # type: ignore[attr-defined]
+            last_exc = e
+            try:
+                await asyncio.sleep(min((2 ** attempt)/10, 1))
+            except Exception:
+                pass
+    # If all retries failed, re-raise the last exception
+    assert last_exc is not None
+    raise last_exc
\ No newline at end of file
diff --git a/verl/workers/reward_manager/__init__.py b/verl/workers/reward_manager/__init__.py
index 566631b4..7b3606e9 100644
--- a/verl/workers/reward_manager/__init__.py
+++ b/verl/workers/reward_manager/__init__.py
@@ -17,6 +17,7 @@ from .batch import BatchRewardManager
 from .dapo import DAPORewardManager
 from .naive import NaiveRewardManager
 from .prime import PrimeRewardManager
+from .checklist import ChecklistRewardManager
 
 # Note(haibin.lin): no need to include all reward managers here in case of complicated dependencies
 __all__ = [
@@ -26,4 +27,5 @@ __all__ = [
     "PrimeRewardManager",
     "register",
     "get_reward_manager_cls",
+    "ChecklistRewardManager",
 ]
diff --git a/verl/workers/reward_manager/checklist.py b/verl/workers/reward_manager/checklist.py
new file mode 100644
index 00000000..cc84259a
--- /dev/null
+++ b/verl/workers/reward_manager/checklist.py
@@ -0,0 +1,644 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import defaultdict
+from typing import Any
+import os
+import torch
+import re
+import math
+import httpx
+import asyncio
+import json
+import logging
+import numpy as np
+import random
+import torch.distributed as dist
+
+from verl import DataProto
+from verl.utils.reward_score import checklist_reward, default_compute_score
+from verl.workers.reward_manager import register
+from verl.workers.reward_manager.abstract import AbstractRewardManager
+
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+@register("checklist")
+class ChecklistRewardManager(AbstractRewardManager):
+    """The reward manager."""
+
+    def __init__(self, tokenizer, num_examine, compute_score=None, reward_fn_key="data_source", **kwargs: Any) -> None:
+        """
+        Initialize the ChecklistRewardManager instance.
+
+        Args:
+            tokenizer: The tokenizer used to decode token IDs into text.
+            num_examine: The number of batches of decoded responses to print to the console for debugging purpose.
+            compute_score: A function to compute the reward score. If None, `default_compute_score` will be used.
+            reward_fn_key: The key used to access the data source in the non-tensor batch data. Defaults to
+                "data_source".
+        """
+        self._sglang_url = kwargs.get("sglang_url", [])
+        self._sglang_model = kwargs.get("sglang_model", "Qwen/Qwen3-30B-A3B-Instruct-2507")
+        self._retry_times = kwargs.get("retry_times", 3)
+        self._semaphore_size = kwargs.get("semaphore_size", 20)
+        self._timeout_seconds = kwargs.get("timeout_seconds", 120.0)
+        self._temperature = kwargs.get("temperature", 0.6)
+        self._top_p = kwargs.get("top_p", 0.8)
+        self._max_new_tokens = kwargs.get("max_new_tokens", 100)
+        self._max_tokens = kwargs.get("max_tokens", 2048)
+        self._eta = kwargs.get("eta", 1.0)
+        self._do_norm_in_adv = kwargs.get("do_norm_in_adv", False)
+        self._reward_level = kwargs.get("reward_level", "turn")
+        self._threthod_num = int(kwargs.get("threthod_num", 4))
+
+        self.tokenizer = tokenizer  # Store the tokenizer for decoding token IDs
+        self.num_examine = num_examine  # the number of batches of decoded responses to print to the console
+        self.compute_score = compute_score or default_compute_score
+        self.reward_fn_key = reward_fn_key  # Store the key for accessing the data source
+        self.thinking_regex = re.compile(r"<think>(.*?)</think>", re.DOTALL)
+        self.tool_call_regex = re.compile(r"<tool_call>(.*?)</tool_call>", re.DOTALL)
+
+
+    def print_data(self, data: DataProto, i2scores, i2turn_counts):
+        already_print_data_sources = {}
+
+        for i in range(len(data)):
+            data_item = data[i]  # DataProtoItem
+            # scores = scores_list[i]
+
+            prompt_ids = data_item.batch["prompts"]
+
+            prompt_length = prompt_ids.shape[-1]
+
+            valid_prompt_length = data_item.batch["attention_mask"][:prompt_length].sum()
+            valid_prompt_ids = prompt_ids[-valid_prompt_length:]
+
+            response_ids = data_item.batch["responses"]
+            valid_response_length = data_item.batch["attention_mask"][prompt_length:].sum()
+            valid_response_ids = response_ids[:valid_response_length]
+
+            # decode
+            prompt_str = self.tokenizer.decode(valid_prompt_ids, skip_special_tokens=False)
+            response_str = self.tokenizer.decode(valid_response_ids, skip_special_tokens=False)
+
+            data_source = data_item.non_tensor_batch[self.reward_fn_key]
+            extra_info = data_item.non_tensor_batch.get("extra_info", {})
+
+            if data_source not in already_print_data_sources:
+                already_print_data_sources[data_source] = 0
+
+            if already_print_data_sources[data_source] < self.num_examine:
+                already_print_data_sources[data_source] += 1
+                print("[prompt]", prompt_str)
+                print("[response]", response_str)
+                print("[checklists]", data.non_tensor_batch['extra_info'][i]['interaction_kwargs']['checklist_list'])
+                print("[scores]", i2scores[i])
+                print("[turn_counts]", i2turn_counts[i])
+                # print("[scores]", scores)
+
+    def __call__(self, data: DataProto, return_dict: bool = False) -> torch.Tensor | dict[str, Any]:
+        """Synchronous entrypoint that internally runs the async pipeline.
+
+        Returns a torch.Tensor or a dict with keys 'reward_tensor' and 'reward_extra_info'.
+        """
+        
+        result = asyncio.run(self._async_call(data, return_dict))
+        return result
+
+    async def _async_call(self, data: DataProto, return_dict: bool = False) -> torch.Tensor | dict[str, Any]:
+
+        # data.batch = TensorDict(
+        #     fields={
+        #         attention_mask: Tensor(shape=torch.Size([32, 10000]), device=cpu, dtype=torch.int64, is_shared=False),
+        #         input_ids: Tensor(shape=torch.Size([32, 10000]), device=cpu, dtype=torch.int64, is_shared=False),
+        #         position_ids: Tensor(shape=torch.Size([32, 10000]), device=cpu, dtype=torch.int64, is_shared=False),
+        #         prompts: Tensor(shape=torch.Size([32, 2000]), device=cpu, dtype=torch.int64, is_shared=False),
+        #         response_mask: Tensor(shape=torch.Size([32, 8000]), device=cpu, dtype=torch.int64, is_shared=False),
+        #         responses: Tensor(shape=torch.Size([32, 8000]), device=cpu, dtype=torch.int64, is_shared=False)},
+        #     batch_size=torch.Size([32]),
+        #     device=None,
+        #     is_shared=False)
+
+        logger.info("Getting checklist scores")
+        # each element in scores_list[i] is a list of scores per assistant step for sample i
+        def get_args_with_random_url():
+            selected_url = random.choice(self._sglang_url) if self._sglang_url else None
+            return {
+                "sglang_model": self._sglang_model,
+                "sglang_url": selected_url,
+                "temperature": self._temperature,
+                "top_p": self._top_p,
+                "max_new_tokens": self._max_new_tokens,
+                "max_tokens": self._max_tokens,
+                "retry_times": self._retry_times,
+            }
+
+        semaphore_size = int(self._semaphore_size)
+        limits = httpx.Limits(
+            max_connections=semaphore_size,
+            max_keepalive_connections=semaphore_size // 2,
+        )
+        timeout = httpx.Timeout(timeout=float(self._timeout_seconds), read=float(self._timeout_seconds), write=float(self._timeout_seconds), connect=float(self._timeout_seconds))
+        semaphore = asyncio.Semaphore(semaphore_size)
+
+
+        # tool_call_success = [
+        #     all([
+        #         metric["success"]
+        #         for metrics_list in req_metrics.values()
+        #         for metric in metrics_list
+        #         if "success" in metric
+        #     ])
+        #     for req_metrics in data.non_tensor_batch["metrics"]
+        # ]
+        # logger.warning("{}/{} success".format(sum(tool_call_success), len(tool_call_success)))
+
+        user_turn_rewards = [ x["user_turn_rewards"] for x in data.non_tensor_batch["reward_scores"]]
+
+        # get reward that are not calculated in the last turn
+        async with httpx.AsyncClient(timeout=timeout, limits=limits) as shared_client:
+            results = []
+            for i in range(len(data)):
+                this_data_checklist = data.non_tensor_batch['extra_info'][i]['interaction_kwargs']['checklist_list'] # 3 * turns * steps
+                num_checklists = len(this_data_checklist)
+                this_data_user_turn_rewards = user_turn_rewards[i] # turns * (reward, turn, call_success)
+                this_data_rewards = [reward for reward, turn, success in this_data_user_turn_rewards] # turns * num_checklists * steps
+                this_data_turns = [turn for reward, turn, success in this_data_user_turn_rewards] # turns * steps
+                this_data_call_success = [success for reward, turn, success in this_data_user_turn_rewards] # turns * num_checklists * steps
+
+                for j, this_checklist in enumerate(this_data_checklist):
+
+                    this_checklist_this_data_rewards = [x[j] for x in this_data_rewards] # turns * steps
+                    this_checklist_this_data_turns = this_data_turns # turns * steps
+                    this_checklist_this_data_call_success = [x[j] for x in this_data_call_success] # turns * steps
+
+                    flat_this_checklist_this_data_rewards = [item for sublist in this_checklist_this_data_rewards for item in sublist]
+                    flat_this_checklist_this_data_turns = [item for sublist in this_checklist_this_data_turns for item in sublist]
+                    flat_this_checklist_this_data_success = [item for sublist in this_checklist_this_data_call_success for item in sublist]
+
+                    args = get_args_with_random_url()
+                    # also pass semaphore size down for internal defaulting
+                    args["semaphore_size"] = semaphore_size
+                    args["timeout_seconds"] = float(self._timeout_seconds)
+                    results.append(asyncio.create_task(checklist_reward.get_checklist_scores_multiturn_multistep(
+                        data.non_tensor_batch["messages"][i]["messages"],
+                        this_checklist,
+                        args,
+                        flat_this_checklist_this_data_rewards,
+                        flat_this_checklist_this_data_turns,
+                        flat_this_checklist_this_data_success,
+                        client=shared_client,
+                        semaphore=semaphore,
+                    )))
+                    # await asyncio.sleep(0.001)
+            results = await asyncio.gather(*results)
+
+        tool_call_success = [
+            all([
+                metric["success"]
+                for metrics_list in req_metrics.values()
+                for metric in metrics_list
+                if "success" in metric
+            ])
+            for req_metrics in data.non_tensor_batch["metrics"]
+        ]
+
+        global_idx = 0
+        for i in range(len(data)):
+            this_data_checklist = data.non_tensor_batch['extra_info'][i]['interaction_kwargs']['checklist_list'] # 3 * turns * steps
+            sample_base_idx = global_idx
+            # this_data_result_list = []
+            uuid = data.non_tensor_batch['extra_info'][i]["original_index"]
+            should_block = False
+            for j, this_checklist in enumerate(this_data_checklist):
+                this_checklist_this_data_results = results[sample_base_idx + j]
+                per_step_call_success = this_checklist_this_data_results[2] # list[list[bool]] per step
+                for x in per_step_call_success:
+                    for y in x:
+                        if y == False:
+                            should_block = True
+                            break
+                    if should_block:
+                        break
+            global_idx += len(this_data_checklist)
+            if should_block:
+                tool_call_success[i] = False
+
+        logger.warning("{}/{} success (reward manager)".format(sum(tool_call_success), len(tool_call_success)))
+
+        # merge all results to uuid2results
+        global_idx = 0
+        results_list = [] # len(data) * num_checklists * (scores_list, turns_list)
+        uuid2results = {} # batchsize * num_checklist * (i, rewards, turns, dependence)
+        uuid2idx = {} # uuid2idx[uuid] = [idx1, idx2, ...]
+        for i in range(len(data)):
+            this_data_checklist = data.non_tensor_batch['extra_info'][i]['interaction_kwargs']['checklist_list'] # 3 * turns * steps
+            sample_base_idx = global_idx
+            if not tool_call_success[i]:
+                global_idx += len(this_data_checklist)
+                # data.batch["response_mask"][i] = 0   
+                continue
+            # this_data_result_list = []
+            uuid = data.non_tensor_batch['extra_info'][i]["original_index"]
+            if uuid not in uuid2results or len(uuid2results[uuid])==0:
+                uuid2results[uuid] = [[] for _ in range(len(this_data_checklist))]
+            if uuid not in uuid2idx:
+                uuid2idx[uuid] = []
+            uuid2idx[uuid].append(i)
+            for j, this_checklist in enumerate(this_data_checklist):
+                this_checklist_this_data_results = results[sample_base_idx + j]
+                all_turn_dependence_list = [self.get_dependence_per_turn_checklist(this_turn_this_checklist) for this_turn_this_checklist in this_checklist]
+                # compute per-turn counts and aggregated standards
+                per_step_standards = this_checklist_this_data_results[0]  # list[list[bool]] per step
+                per_step_turn_indices = this_checklist_this_data_results[1]  # list[int] per step's turn index
+                per_step_call_success = this_checklist_this_data_results[2] # list[list[bool]] per step
+                num_turns = len(this_checklist)
+                per_turn_counts = [0] * num_turns
+                per_turn_aggregated_standards = [[False] * len(this_checklist[t]) for t in range(num_turns)]
+                per_turn_weights = [[y['weight']for y in x] for x in this_checklist]
+                for step_standards, turn_idx in zip(per_step_standards, per_step_turn_indices):
+                    if 0 <= turn_idx < num_turns:
+                        per_turn_counts[turn_idx] += 1
+                        assert len(step_standards) == len(per_turn_aggregated_standards[turn_idx]), f"{len(step_standards)}, {len(per_turn_aggregated_standards[turn_idx])}"
+                        for k in range(len(step_standards)):
+                            if step_standards[k]:
+                                per_turn_aggregated_standards[turn_idx][k] = True
+                # compute weighted per-turn scores: sum(weight_k * standard_k)
+                per_turn_scores = []
+                for t in range(num_turns):
+                    if per_turn_counts[t] == 0:
+                        per_turn_scores.append(0.0)
+                        continue
+                    standards_t = per_turn_aggregated_standards[t]
+                    # already normalized to 1.0
+                    weights_t = per_turn_weights[t]
+                    numer = 0.0
+                    for s_val, w_val in zip(standards_t, weights_t):
+                        numer += (1.0 if bool(s_val) else 0.0) * float(w_val)
+                    per_turn_scores.append(numer)
+
+                uuid2results[uuid][j].append((i, this_checklist_this_data_results[0], this_checklist_this_data_results[1], all_turn_dependence_list, per_turn_counts, per_turn_scores, per_turn_aggregated_standards, per_turn_weights)) # batchsize * num_checklist * n * (succes_turn, success_turn, all_turn, per_turn_counts, per_turn_aggregated, per_turn_weights, per_turn_scores)
+            global_idx += len(this_data_checklist)
+
+        # implement logic.txt: compute effective ranges, pass rates and per-step rewards
+        # 将二进制mask中连续的1段转换为[(start, end)]区间（end为开区间）
+        # 用于把assistant的每段回复映射到对应的末token位置
+        def one_segments_torch(mask):
+            m = mask.to(torch.int8) if isinstance(mask, torch.Tensor) else torch.tensor(mask, dtype=torch.int8)
+            diff = torch.diff(torch.cat([torch.tensor([0], dtype=torch.int8), m, torch.tensor([0], dtype=torch.int8)]))
+            starts = (diff == 1).nonzero(as_tuple=True)[0]
+            ends = (diff == -1).nonzero(as_tuple=True)[0]
+            return list(zip(starts.tolist(), ends.tolist()))
+        
+        reward_tensor = torch.zeros_like(data.batch["responses"], dtype=torch.float32)
+        turn_end_tensor = torch.zeros_like(data.batch["responses"], dtype=torch.bool)
+        adv_tensor = torch.zeros_like(data.batch["responses"], dtype=torch.float32)
+
+        i2scores = [None]*len(data)
+        i2turn_counts = [None]*len(data)
+        for uuid, all_checklist_results in uuid2results.items():
+            for this_checklist_results in all_checklist_results:
+                for this_checklist_result in this_checklist_results:
+                    i = this_checklist_result[0]
+                    per_turn_counts = this_checklist_result[4]
+                    per_turn_scores = this_checklist_result[5]
+                    i2scores[i] = per_turn_scores
+                    i2turn_counts[i] = per_turn_counts
+                    accumulate_steps = 0
+                    segments = one_segments_torch(data.batch["response_mask"][i])
+                    for count, score in zip(per_turn_counts, per_turn_scores):
+                        if count == 0:
+                            break
+                        if accumulate_steps+count-1 >= len(segments):
+                            break
+                        end = segments[accumulate_steps+count-1][1]
+                        reward_tensor[i, end-1] += float(score)
+                        turn_end_tensor[i, end-1] = True
+                        accumulate_steps += count
+        self.print_data(data,i2scores,i2turn_counts)
+        # norm by num of checklists
+        num_checklists = torch.tensor(
+            [len(data.non_tensor_batch['extra_info'][i]['interaction_kwargs']['checklist_list']) for i in range(len(data))],
+            device=reward_tensor.device,
+            dtype=reward_tensor.dtype,
+        )
+        reward_tensor /= num_checklists.unsqueeze(1)
+
+        # norm by num of max turns
+        max_num_turns_tensor = torch.tensor(
+            [len(data.non_tensor_batch['extra_info'][i]['interaction_kwargs']['checklist_list'][0]) for i in range(len(data))],
+            device=reward_tensor.device,
+            dtype=reward_tensor.dtype,
+        )
+        reward_tensor /= max_num_turns_tensor.unsqueeze(1)
+
+        num_turns_tensor = torch.tensor(
+            [sum(1 for m in data.non_tensor_batch["messages"][i]["messages"] if m.role=="user") for i in range(len(data))],
+            device=reward_tensor.device,
+            dtype=reward_tensor.dtype,
+        )
+
+        tool_call_success = torch.tensor(
+            tool_call_success,
+            device=reward_tensor.device,
+            dtype=torch.bool,
+        )
+        # reward_tensor *= tool_call_success.unsqueeze(1)
+
+        if self._reward_level == "step":
+            logger.warning("calculate step level reward")
+            # 步骤1：计算每个uuid的每个checklist的每个turn的每条标准的平均值、标准差
+            uuid_checklist_turn_standard_stats = {}  # uuid -> checklist_idx -> turn_idx -> standard_idx -> (mean, std)
+
+            for uuid, all_checklist_results in uuid2results.items():
+                uuid_checklist_turn_standard_stats[uuid] = {}
+
+                for checklist_idx, this_checklist_results in enumerate(all_checklist_results):
+                    if len(this_checklist_results) == 0:
+                        continue
+
+                    # 获取依赖关系（所有sample的依赖关系相同）
+                    dependence_list = this_checklist_results[0][3]
+                    num_turns = len(dependence_list)
+
+                    uuid_checklist_turn_standard_stats[uuid][checklist_idx] = {}
+
+                    # 收集每个turn的每条标准的所有有效分数
+                    turn_standard_scores = {}  # turn_idx -> standard_idx -> list of scores
+
+                    for this_checklist_result_idx, this_checklist_result in enumerate(this_checklist_results):
+                        step_standards = this_checklist_result[1]  # list[list[bool]] per step
+                        step_turn_indices = this_checklist_result[2]  # list[int] per step's turn index
+
+                        # 对每个turn，跟踪已完成的标准
+                        turn_completed_standards = {}  # turn_idx -> set of completed standard_idx
+
+                        # 按step顺序处理，跟踪动态依赖关系
+                        previous_turn = -1
+                        fist_step_this_turn = -1
+                        for step_idx, (step_standard_results, turn_idx) in enumerate(zip(step_standards, step_turn_indices)):
+                            # if turn_idx >= num_turns or turn_idx < 0:
+                            #     continue
+                            assert 0 <= turn_idx < num_turns
+                            if turn_idx!=previous_turn:
+                                previous_turn = turn_idx
+                                fist_step_this_turn = step_idx
+
+                            if turn_idx not in turn_completed_standards:
+                                turn_completed_standards[turn_idx] = {}
+
+                            turn_dependence = dependence_list[turn_idx]
+                            for standard_idx, standard_result in enumerate(step_standard_results):
+                                if standard_idx >= len(turn_dependence):
+                                    continue
+
+                                # 检查该标准的依赖是否在这个turn中已经被前面的step完成了
+                                dependencies_met = True
+                                max_step_dependencies_met = -1
+                                if len(turn_dependence[standard_idx])==0:
+                                    max_step_dependencies_met = fist_step_this_turn-1
+                                for dep_idx in turn_dependence[standard_idx]:
+                                    if dep_idx not in turn_completed_standards[turn_idx]:
+                                        dependencies_met = False
+                                        break
+                                    else:
+                                        max_step_dependencies_met = max(max_step_dependencies_met, turn_completed_standards[turn_idx][dep_idx])
+                                    
+
+                                # 如果依赖满足且该标准在此turn中尚未完成，收集该标准的分数
+                                if dependencies_met and standard_idx not in turn_completed_standards[turn_idx]:
+                                    if turn_idx not in turn_standard_scores:
+                                        turn_standard_scores[turn_idx] = {}
+                                    if standard_idx not in turn_standard_scores[turn_idx]:
+                                        turn_standard_scores[turn_idx][standard_idx] = [np.nan] * len(this_checklist_results)
+                                    if standard_result:
+                                        standard_score = 1.0 * (self._eta**(step_idx-max_step_dependencies_met-1))
+                                        turn_standard_scores[turn_idx][standard_idx][this_checklist_result_idx] = standard_score
+                                    else:
+                                        if np.isnan(turn_standard_scores[turn_idx][standard_idx][this_checklist_result_idx]):
+                                            turn_standard_scores[turn_idx][standard_idx][this_checklist_result_idx] = 0.0
+
+                                    # 如果标准完成了且依赖项都已完成，才标记为已完成
+                                    if standard_result:
+                                        # turn_completed_standards[turn_idx].add(standard_idx)
+                                        turn_completed_standards[turn_idx][standard_idx] = step_idx
+
+                    # 计算每个turn每条标准的平均值和标准差
+                    for turn_idx in range(num_turns):
+                        uuid_checklist_turn_standard_stats[uuid][checklist_idx][turn_idx] = {}
+
+                        if turn_idx in turn_standard_scores:
+                            for standard_idx, scores in turn_standard_scores[turn_idx].items():
+                                not_nan_score = [ x for x in scores if not np.isnan(x)]
+                                if len(not_nan_score) > 0:
+                                    mean_score = np.mean(not_nan_score)
+                                    std_score = np.std(not_nan_score) if len(not_nan_score) > 1 else 1.0  # 避免一个元素报错
+                                    std_score = max(std_score, 1e-6)  # 避免标准差为0
+                                    uuid_checklist_turn_standard_stats[uuid][checklist_idx][turn_idx][standard_idx] = (mean_score, std_score, len(not_nan_score))
+
+            all_std_num = 0
+            all_std_num_lt_4 = 0
+            threthod_num = self._threthod_num
+            # 步骤2和3：计算每个数据的每个step的advantage并分配到tokens
+            for uuid, all_checklist_results in uuid2results.items():
+                # 按数据索引分组，将同一数据的所有checklist的结果组合在一起
+                data_idx_to_checklist_results = {}
+                for checklist_idx, this_checklist_results in enumerate(all_checklist_results):
+                    for this_checklist_result in this_checklist_results:
+                        i = this_checklist_result[0]  # 数据索引
+                        if i not in data_idx_to_checklist_results:
+                            data_idx_to_checklist_results[i] = []
+                        data_idx_to_checklist_results[i].append((checklist_idx, this_checklist_result))
+
+                # 对每个数据计算advantage
+                for i, checklist_results in data_idx_to_checklist_results.items():
+                    # 获取response的segments
+                    segments = one_segments_torch(data.batch["response_mask"][i])
+
+                    # 计算总共的step数量
+                    total_steps = len(checklist_results[0][1][1])  # 取第一个checklist的step数量
+
+                    # 对每个step计算advantage
+                    for step_idx in range(total_steps):
+                        if step_idx >= len(segments):
+                            break
+
+                        # 收集所有checklist在这个step的标准分数、均值、标准差
+                        all_actual_scores = []
+                        all_mean_scores = []
+                        all_std_scores = []
+                        all_weights = []
+
+                        for checklist_idx, this_checklist_result in checklist_results:
+                            step_standards = this_checklist_result[1]
+                            step_turn_indices = this_checklist_result[2]
+                            dependence_list = this_checklist_result[3]
+                            per_turn_weights = this_checklist_result[7]  # 获取权重信息
+
+                            if step_idx >= len(step_standards):
+                                continue
+
+                            step_standard_results = step_standards[step_idx]
+                            turn_idx = step_turn_indices[step_idx]
+
+                            if turn_idx < 0 or turn_idx >= len(dependence_list):
+                                continue
+
+                            # 检查该step在统计数据中是否存在
+                            if (uuid not in uuid_checklist_turn_standard_stats or 
+                                checklist_idx not in uuid_checklist_turn_standard_stats[uuid] or
+                                turn_idx not in uuid_checklist_turn_standard_stats[uuid][checklist_idx]):
+                                continue
+                            
+                            turn_stats = uuid_checklist_turn_standard_stats[uuid][checklist_idx][turn_idx]
+                            turn_dependence = dependence_list[turn_idx]
+                            turn_weights = per_turn_weights[turn_idx]
+                            
+                            # 跟踪这个turn中已完成的标准（重新模拟前面的步骤）
+                            turn_completed_standards = set()
+                            for prev_step_idx in range(step_idx):
+                                if prev_step_idx >= len(step_standards):
+                                    break
+                                prev_step_results = step_standards[prev_step_idx]
+                                prev_turn_idx = step_turn_indices[prev_step_idx]
+                                if prev_turn_idx == turn_idx:
+                                    for std_idx, std_result in enumerate(prev_step_results):
+                                        if std_result and std_idx < len(turn_dependence):
+                                            # 检查该标准的依赖是否都已完成
+                                            dependencies_met = True
+                                            for dep_idx in turn_dependence[std_idx]:
+                                                if dep_idx not in turn_completed_standards:
+                                                    dependencies_met = False
+                                                    break
+                                            # 只有当标准完成且其依赖项也都完成时才标记为已完成
+                                            if dependencies_met:
+                                                turn_completed_standards.add(std_idx)
+                            
+                            # 检查整个turn中该标准是否被完成（包括当前step之后的step），需要考虑依赖关系
+                            # turn_standard_completion = {}  # standard_idx -> bool
+                            turn_completed_for_full_check = {}  # 用于完整turn检查的已完成标准集合
+                            
+                            # 按step顺序重新模拟整个turn，考虑依赖关系
+                            first_step_this_turn = -1
+                            previous_turn = -1
+                            for check_step_idx in range(len(step_standards)):
+                                check_turn_idx = step_turn_indices[check_step_idx]
+                                if check_turn_idx != previous_turn:
+                                    previous_turn = check_turn_idx
+                                    first_step_this_turn = check_step_idx
+                                if check_turn_idx == turn_idx:
+                                    check_step_results = step_standards[check_step_idx]
+                                    for std_idx, std_result in enumerate(check_step_results):
+                                        if std_result:
+                                            # 检查该标准的依赖是否都已完成
+                                            dependencies_met = True
+                                            max_step_dependencies_met = -1
+                                            if len(turn_dependence[std_idx])==0:
+                                                max_step_dependencies_met = first_step_this_turn-1
+                                            for dep_idx in turn_dependence[std_idx]:
+                                                if dep_idx not in turn_completed_for_full_check:
+                                                    dependencies_met = False
+                                                    break
+                                                else:
+                                                    max_step_dependencies_met = max(max_step_dependencies_met, turn_completed_for_full_check[dep_idx])
+
+                                            # 只有当标准完成且依赖项都满足时，才标记为真正完成
+                                            if dependencies_met:
+                                                # turn_standard_completion[std_idx] = True
+                                                turn_completed_for_full_check[std_idx]=check_step_idx-max_step_dependencies_met-1
+                            
+                            # 检查当前step中每个标准的依赖和有效性
+                            for standard_idx, standard_result in enumerate(step_standard_results):
+                                if (standard_idx >= len(turn_dependence) or 
+                                    standard_idx not in turn_stats):
+                                    continue
+                                
+                                # 检查该标准的依赖是否在这个turn中已经被前面的step完成了
+                                dependencies_met = True
+                                for dep_idx in turn_dependence[standard_idx]:
+                                    if dep_idx not in turn_completed_standards:
+                                        dependencies_met = False
+                                        break
+                                
+                                # 如果依赖满足，且该标准在此turn中尚未完成，收集该标准的分数
+                                if dependencies_met and standard_idx not in turn_completed_standards:
+                                    mean_score, std_score, num_scores = turn_stats[standard_idx]
+                                    # 修改判断逻辑：如果依赖项都完成了，之前没完成，并且这个turn内完成了这个标准（可能在后面完成），都算对
+                                    all_std_num += 1
+                                    if num_scores > threthod_num:
+                                        actual_score = 1.0 * (self._eta ** turn_completed_for_full_check[standard_idx]) if standard_idx in turn_completed_for_full_check else 0.0
+                                    else:
+                                        all_std_num_lt_4 += 1
+                                        actual_score = mean_score
+
+                                    # 获取该标准的权重
+                                    weight = turn_weights[standard_idx]
+                                    
+                                    # 按权重加权收集分数
+                                    all_actual_scores.append(actual_score * weight)
+                                    all_mean_scores.append(mean_score * weight)
+                                    all_std_scores.append(std_score * weight)
+                                    all_weights.append(weight)
+                        # 计算该step的advantage：所有标准分数之和 - 所有均值之和，除以所有标准差的几何平均值
+                        if len(all_actual_scores) > 0:
+                            total_actual = sum(all_actual_scores)
+                            total_mean = sum(all_mean_scores)
+                            total_weight = sum(all_weights)
+                            total_weight = total_weight if total_weight != 0 else 1.0
+                            geometric_mean_std = np.sqrt(np.sum(np.square(all_std_scores)))  # 平方求和再开根号
+                            geometric_mean_std = geometric_mean_std if geometric_mean_std != 0 else 1.0
+                            
+                            # final_advantage = (total_actual - total_mean) / (geometric_mean_std if self._do_norm_in_adv else total_weight)
+                            final_advantage = (total_actual - total_mean) / (total_weight if self._do_norm_in_adv else 1.0)
+                            
+                            # 将advantage分配到该step的所有tokens
+                            segment_start, segment_end = segments[step_idx]
+                            adv_tensor[i, segment_start:segment_end] = final_advantage
+            if all_std_num==0:
+                all_std_num=1
+            logger.warning(f"step adv: {all_std_num_lt_4}/{all_std_num} ({all_std_num_lt_4/all_std_num*100}%) is ignored because group <= {threthod_num}")
+        if return_dict:
+            if self._reward_level == "step":
+                reward_extra_info = {
+                        "adv_tensor": adv_tensor,
+                        "turn_end_tensor": turn_end_tensor,
+                        "max_num_turns": max_num_turns_tensor,
+                        "num_turns": num_turns_tensor,
+                        "tool_call_success": tool_call_success
+                    }
+            else:
+                reward_extra_info = {
+                    "turn_end_tensor": turn_end_tensor,
+                    "max_num_turns": max_num_turns_tensor,
+                    "num_turns": num_turns_tensor,
+                    "tool_call_success": tool_call_success
+                }
+            return {
+                "reward_tensor": reward_tensor,
+                "reward_extra_info": reward_extra_info
+            }
+        else:
+            return reward_tensor
+
+    
+    def get_dependence_per_turn_checklist(self, this_turn_checklist: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        id2idex = {item["id"]: idx for idx, item in enumerate(this_turn_checklist)}
+        dependence_list = []
+        for idx, item in enumerate(this_turn_checklist):
+            dependence = item["dependence"]
+            dependence_list.append([id2idex[dep] for dep in dependence])
+        return dependence_list
diff --git a/verl/workers/rollout/schemas.py b/verl/workers/rollout/schemas.py
index b640ba64..93b52416 100644
--- a/verl/workers/rollout/schemas.py
+++ b/verl/workers/rollout/schemas.py
@@ -146,7 +146,7 @@ class AsyncRolloutRequest(BaseModel):
             values["multi_modal_inputs"] = {}
 
         tools = (
-            [tool.model_dump() for tool in tool_schemas] if (tool_schemas := values.get("tool_schemas", [])) else None
+            [tool.model_dump(exclude_none=True) for tool in tool_schemas] if (tool_schemas := values.get("tool_schemas", [])) else None
         )
 
         multi_modal_data = values["multi_modal_data"]
@@ -242,7 +242,13 @@ class AsyncRolloutRequest(BaseModel):
                 logger.warning(
                     "There is multi_modal_data but you are not using a processor. Multi-modal data will be ignored."
                 )
-            model_inputs = processing_class(text=[raw_prompt], return_tensors="pt")
+            try:
+                model_inputs = processing_class(text=[raw_prompt], return_tensors="pt")
+            except:
+                def clean(s: str) -> str:
+                    return ''.join(ch for ch in s if not 0xD800 <= ord(ch) <= 0xDFFF)
+                logger.warning("UnicodeEncodeError in input text")
+                model_inputs = processing_class(text=[clean(raw_prompt)], return_tensors="pt")
         elif isinstance(processing_class, ProcessorMixin):
             # When we update multi_model_keys, we also need to update this logic
             images = images if len(images := multi_modal_data.get("image", [])) > 0 else None
@@ -363,7 +369,7 @@ class AsyncRolloutRequest(BaseModel):
 
         if self.use_inference_chat_template:
             messages = [msg.model_dump() for msg in self.messages]
-            tools = [tool.model_dump() for tool in self.tool_schemas] if self.tool_schemas else None
+            tools = [tool.model_dump(exclude_none=True) for tool in self.tool_schemas] if self.tool_schemas else None
             generation_prompt_ids = self._handle_apply_chat_template(
                 processing_class,
                 messages,
@@ -383,7 +389,7 @@ class AsyncRolloutRequest(BaseModel):
     ) -> None:
         self.messages.append(Message(role="user", content=content))
         messages = [*BASE_CHAT_HISTORY, self.messages[-1]]
-        tools = [tool.model_dump() for tool in self.tool_schemas] if self.tool_schemas else None
+        tools = [tool.model_dump(exclude_none=True) for tool in self.tool_schemas] if self.tool_schemas else None
 
         # We don't need to pass multi_modal_data here because we don't have any multi-modal data from Engine
         # Inference, it is pure text.
@@ -402,13 +408,16 @@ class AsyncRolloutRequest(BaseModel):
         self.messages.append(Message(role="assistant", content=content, tool_calls=tool_calls))
         if content_ids is None:
             messages = [*BASE_CHAT_HISTORY, self.messages[-1]]
-            tools = [tool.model_dump() for tool in self.tool_schemas] if self.tool_schemas else None
+            tools = [tool.model_dump(exclude_none=True) for tool in self.tool_schemas] if self.tool_schemas else None
 
             # We don't need to pass multi_modal_data here because we don't have any multi-modal data from Engine
             # Inference, it is pure text.
             content_ids = self._handle_apply_chat_template(
                 processing_class, messages, multi_modal_data={}, tools=tools, add_generation_prompt=False, tokenize=True
             )[..., self.base_conv_with_gen_prompt_end_pos :]
+        if content_ids[0][-1] != torch.tensor([processing_class.encode("<|im_end|>")])[0][-1]:
+            content_ids = torch.cat([content_ids, torch.tensor([processing_class.encode("<|im_end|>")])], dim=-1)
+        content_ids = torch.cat([content_ids, torch.tensor([processing_class.encode("\n")])], dim=-1)
         self._update_input_ids(processing_class, content_ids, attention_mask=True, loss_mask=True)
 
     def add_tool_response_messages(
@@ -438,7 +447,7 @@ class AsyncRolloutRequest(BaseModel):
                 self.messages.append(Message(role="tool", content=content_list))
 
         messages = [*BASE_CHAT_HISTORY, *self.messages[-len(contents) :]]
-        tools = [tool.model_dump() for tool in self.tool_schemas] if self.tool_schemas else None
+        tools = [tool.model_dump(exclude_none=True) for tool in self.tool_schemas] if self.tool_schemas else None
 
         for key in self.multi_modal_keys:
             if len(delta_multi_modal_data[key]) > 0:
@@ -568,7 +577,7 @@ class AsyncRolloutRequest(BaseModel):
             diff_surrounding_chars = 10
 
             messages = [msg.model_dump() for msg in self.messages]
-            tools = [tool.model_dump() for tool in self.tool_schemas] if self.tool_schemas else None
+            tools = [tool.model_dump(exclude_none=True) for tool in self.tool_schemas] if self.tool_schemas else None
             full_prompt_info = self._handle_apply_chat_template(
                 processing_class,
                 messages,
@@ -604,7 +613,7 @@ class AsyncRolloutRequest(BaseModel):
                         f"This may lead to unexpected behavior during training."
                         f"Please review your multi_modal_inputs logic."
                     )
-
+            log_warning = None
             if diffs := self._get_prompt_diffs(
                 processing_class, full_prompt_ids, self.input_ids, diff_surrounding_chars=diff_surrounding_chars
             ):
@@ -637,7 +646,11 @@ class AsyncRolloutRequest(BaseModel):
                             f"current_prompt_chunk: {repr(d['current_prompt_chunk'])}"
                         )
                     diff_details = "\n".join(diff_details_list)
-                    logger.warning(f"Found differences:\n{diff_details}")
+                    logger.warning(f"Found differences:\n{diff_details}\nSet loss mask to 0")
+                    
+                    self.loss_mask.zero_()
+                    if self.response_loss_mask is not None:
+                        self.response_loss_mask.zero_()
 
         if finish_reason_type == FinishReasonTypeEnum.STOP:
             pass
diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py
index fa5ca879..60a52ab1 100644
--- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py
+++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py
@@ -54,7 +54,7 @@ from verl.interactions.base import BaseInteraction
 from verl.interactions.utils.interaction_registry import initialize_interactions_from_config
 from verl.third_party.sglang import parallel_state as sglang_ps
 from verl.tools.base_tool import BaseTool
-from verl.tools.schemas import OpenAIFunctionCallSchema, OpenAIFunctionParsedSchema, OpenAIFunctionToolCall
+from verl.tools.schemas import OpenAIFunctionCallSchema, OpenAIFunctionParsedSchema, OpenAIFunctionToolCall, OpenAIFunctionToolSchema
 from verl.tools.utils.tool_registry import initialize_tools_from_config
 from verl.utils.device import get_visible_devices_keyword
 from verl.utils.import_utils import deprecated
@@ -88,6 +88,13 @@ try:
 except ImportError:
     from sglang.srt.utils import get_local_ip_auto as get_ip
 
+from verl.tools.schemas import ToolResponse
+from sglang.srt.function_call.core_types import (
+    ToolCallItem,
+)
+import json
+import re
+
 logger = logging.getLogger(__file__)
 logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
 
@@ -190,8 +197,8 @@ def _extract_logprob_from_output(output):
             *[(log_prob, token_ids) for log_prob, token_ids, _ in input_token_logprobs[1:]], strict=False
         )
         return torch.tensor(output_token_ids), torch.tensor(log_probs)
-
-    output_token_ids, log_probs = _map_each_response(output)
+    # TODO: check if this is correct to only use the first output
+    output_token_ids, log_probs = _map_each_response(output[0])
     return output_token_ids, log_probs
 
 
@@ -441,6 +448,7 @@ class SGLangRollout(BaseRollout):
         backend = attention_backend if attention_backend is not None else "fa3"
         sglang_port = int(os.getenv("SGLANG_PORT", "30000")) + (dist.get_rank() * 2)
         if effective_first:
+            rank = dist.get_rank()
             os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "0"
             args = {
                 "model_path": actor_module,
@@ -463,7 +471,7 @@ class SGLangRollout(BaseRollout):
                 # NOTE(Chenyang): if you want to debug the SGLang engine output
                 # please set the following parameters
                 # Otherwise, it will make the engine run too slow
-                "log_level": "info",
+                "log_level": "warn",
                 # "log_level": "error",
                 # log_requests=True,
                 # log_requests_level=2,
@@ -807,6 +815,11 @@ class SGLangRollout(BaseRollout):
             # we will recompute old log prob with actor
             batch["rollout_log_probs"] = rollout_log_probs
 
+        # free cache engine
+        if self._engine is not None and self._tp_rank == 0:
+            loop = asyncio.get_event_loop()
+            loop.run_until_complete(self._engine.flush_cache())
+
         return DataProto(batch=batch, non_tensor_batch=non_tensor_batch)
 
     async def _async_rollout_a_request(
@@ -856,6 +869,51 @@ class SGLangRollout(BaseRollout):
 
         # Update with any additional kwargs
         request_sampling_params.update(kwargs)
+        def parse_base_json(action):
+            if not isinstance(action, list):
+                action = [action]
+
+            results = []
+            error_message = None
+            for act in action:
+                # Safely extract fields from possibly malformed objects
+                name = act.get("name") if isinstance(act, dict) else None
+                arguments = act.get("parameters") if isinstance(act, dict) else None
+                if arguments is None and isinstance(act, dict):
+                    arguments = act.get("arguments", None)
+
+                # Ensure arguments are JSON-serializable
+                if arguments is None:
+                    error_message = json.dumps({"error_tool_call": "No argumets found in one tool call. Use empty dict if no argument."})
+                    break
+
+                try:
+                    _ = json.dumps(arguments, ensure_ascii=False)
+                except Exception as e:
+                    error_message = json.dumps({"error_tool_call": "NON_SERIALIZABLE_ARGUMENTS in one tool call", "detail": str(e)})
+                    break
+
+                # Validate tool name; fallback to error_tool_call if invalid
+                if not isinstance(name, str) or not name.strip():
+                    error_message = json.dumps(
+                                {
+                                    "error_tool_call": "INVALID_TOOL_NAME in one tool call",
+                                    "raw_name": str(name),
+                                    "arguments": str(arguments),
+                                },
+                                ensure_ascii=True,
+                            )
+                    break
+
+                results.append(
+                    ToolCallItem(
+                        tool_index=-1,  # Caller should update this based on the actual tools array called
+                        name=name.strip(),
+                        parameters=json.dumps(arguments, ensure_ascii=True),
+                    )
+                )
+
+            return results, error_message
 
         while current_turns < self.config.multi_turn.max_assistant_turns:
             if _req.state == AsyncRolloutRequestStateEnum.PENDING:
@@ -866,20 +924,46 @@ class SGLangRollout(BaseRollout):
                     parsed_tool_calls = _req.messages[-1].tool_calls
                     if self.config.skip_tokenizer_init:
                         _req.messages[-1].tool_calls = None
-                    tool_call_results = await asyncio.gather(
-                        *[
-                            self._tool_map[tool_call.function.name].execute(
-                                _req.request_id,
-                                tool_call.function.arguments,
-                                **_req.tools_kwargs.get(tool_call.function.name, {}).get("execute_kwargs", {}),
+                    # Execute tools with validation; return error responses for unknown tool names or missing kwargs
+                    tasks = []
+                    async def _make_error_response(name: str, reason: str):
+                        msg = f"tool name '{name}' {reason}. Please check your function call and use provided tools."
+                        return ToolResponse(text=json.dumps({"error_tool_call": msg})), 0.0, {"success": True, "error_tool_call": "INVALID_TOOL_NAME", "tool": name, "reason": reason}
+
+                    for tool_call in parsed_tool_calls:
+                        tool_name = tool_call.function.name
+                        missing_in_map = tool_name not in self._tool_map
+                        missing_in_kwargs = _req.tools_kwargs is None or tool_name not in _req.tools_kwargs
+
+                        if missing_in_map or missing_in_kwargs:
+                            tasks.append(_make_error_response(tool_name, "is not found."))
+                        else:
+                            execute_kwargs = _req.tools_kwargs[tool_name].get("execute_kwargs", {})
+                            tasks.append(
+                                self._tool_map[tool_name].execute(
+                                    _req.request_id,
+                                    tool_call.function.arguments,
+                                    **execute_kwargs,
+                                )
                             )
-                            for tool_call in parsed_tool_calls
-                        ]
-                    )
+                    tool_call_results = await asyncio.gather(*tasks)
                     _req.add_tool_response_messages(self.processing_class, [resp for resp, _, _ in tool_call_results])
+                    tool_call_error = False
                     for tool_call, (resp, reward, metrics) in zip(parsed_tool_calls, tool_call_results, strict=True):
                         _req.update_metrics(metrics, tool_call.function.name)
-                    if _req.input_ids.size(-1) >= self.config.max_model_len:
+                    for tool_call_result, _, _ in tool_call_results:
+                        if "error_tool_call" in tool_call_result.text:
+                            tool_call_error = True
+                    if tool_call_error:
+                        # logger.warning("Tool call schema error, stopping the conversation")
+                        finish_reason_type = FinishReasonTypeEnum.STOP
+                        break
+                    all_tool_call_success = all(metrics['success'] for resp, reward, metrics in tool_call_results)
+                    if not all_tool_call_success:
+                        # logger.warning("Tool call failed, stopping the conversation")
+                        finish_reason_type = FinishReasonTypeEnum.STOP
+                        break
+                    if len(_req.input_ids) >= self.config.max_model_len:
                         finish_reason_type = FinishReasonTypeEnum.STOP
                         break
                     _req.state = AsyncRolloutRequestStateEnum.RUNNING
@@ -914,7 +998,11 @@ class SGLangRollout(BaseRollout):
                 output = await self._handle_engine_call(_req, request_sampling_params, image_data=image_data)
                 if self.config.skip_tokenizer_init:
                     content_ids = output["output_ids"]
-                    content = self.processing_class.decode(content_ids, skip_special_tokens=True)
+                    # content = self.processing_class.decode(content_ids, skip_special_tokens=True)
+                    content = self.processing_class.decode(content_ids, skip_special_tokens=False)
+                    if content.endswith("<|im_end|>"):
+                        content = content[:-len("<|im_end|>")]
+                    
                     content_ids = torch.tensor(
                         content_ids, dtype=_req.input_ids.dtype, device=_req.input_ids.device
                     ).unsqueeze(0)
@@ -931,15 +1019,53 @@ class SGLangRollout(BaseRollout):
                     if self._function_call_parser and self._function_call_parser.has_tool_call(content):
                         finish_reason_type = FinishReasonTypeEnum.TOOL_CALL
                         _req.state = AsyncRolloutRequestStateEnum.TOOL_CALLING
+                        tool_calls = []
+                        error_message = None
                         try:
-                            normed_content, tool_calls = self._function_call_parser.parse_non_stream(content)
-                        except JSONDecodeError:
-                            normed_content = content
-                            tool_calls = []
-                        except AttributeError:
+                            # normed_content, tool_calls = self._function_call_parser.parse_non_stream(content)
+                            bot_token = "\n<tool_call>\n"
+                            eot_token = "\n</tool_call>"
+                            idx = content.find(bot_token)
+                            normal_text = content[:idx] if idx != -1 else content
+                            if bot_token not in content:
+                                normed_content = normal_text
+                                tool_calls = []
+
+                            # Find all \n<tool_call>\n...\n</tool_call> blocks
+                            pattern = rf"{re.escape(bot_token)}(.*?){re.escape(eot_token)}"
+                            match_result_list = re.findall(pattern, content, re.DOTALL)
+
+                            for match_result in match_result_list:
+                                try:
+                                    parsed_call = json.loads(match_result.strip())
+                                except json.JSONDecodeError as e:
+                                    error_message = json.dumps({"error_tool_call": f"One tool call can not be parsed as JSON: {match_result.strip()}"})
+                                    break
+                                parsed_results, error_message = parse_base_json(parsed_call)
+                                if error_message is None:
+                                    tool_calls.extend(parsed_results)
+                                else:
+                                    break
+  
+                            if tool_calls and error_message is None:
+                                normed_content = normal_text
+                                tool_calls = tool_calls
+                            else:
+                                normed_content = content
+                                tool_calls = []
+                        except Exception as e:
+                            error_message = json.dumps({"error_tool_call": "Tool call parse error"})
                             normed_content = content
                             tool_calls = []
+                        if error_message is not None:
+                            _req.add_assistant_message(self.processing_class, content, content_ids=content_ids)
+                            _req.add_tool_response_messages(self.processing_class, [ToolResponse(text=error_message)])
+                            finish_reason_type = FinishReasonTypeEnum.STOP
+                            _req.state = AsyncRolloutRequestStateEnum.COMPLETED
+                            # logger.warning(f"Tool call parse error, force to end: {error_message}")
+                            break
                         parsed_tool_calls = []
+                        has_decode_error = False
                         for tool_call in tool_calls:
                             function, has_decode_error = OpenAIFunctionCallSchema.from_openai_function_parsed_schema(
                                 OpenAIFunctionParsedSchema(
@@ -949,22 +1075,33 @@ class SGLangRollout(BaseRollout):
                             )
                             # Drop the tool call if its arguments has decode error
                             if has_decode_error:
-                                continue
+                                # logger.warning(f"has decode error: Tool call: {tool_call.name}, arguments: {tool_call.parameters}")
+                                _req.add_assistant_message(self.processing_class, content, content_ids=content_ids)
+                                error_message = json.dumps({"error_tool_call": "Tool call tag detected but no one success. Please check your output format."})
+                                _req.add_tool_response_messages(self.processing_class, [ToolResponse(text=error_message)])
+                                finish_reason_type = FinishReasonTypeEnum.STOP
+                                _req.state = AsyncRolloutRequestStateEnum.COMPLETED
+                                break
                             parsed_tool_calls.append(
                                 OpenAIFunctionToolCall(
                                     id=str(tool_call.tool_index),
                                     function=function,
                                 )
                             )
+                        if has_decode_error:
+                            break
                         if len(parsed_tool_calls) > 0:
                             _req.add_assistant_message(
                                 # since the content is updated, we just pass the content not content_ids
                                 self.processing_class,
-                                content=normed_content,
+                                content=content,
+                                content_ids=content_ids,
                                 tool_calls=parsed_tool_calls,
                             )
                         else:
                             _req.add_assistant_message(self.processing_class, content=content, content_ids=content_ids)
+                            error_message = json.dumps({"error_tool_call": "No tool call found. Please check your output format. The correct (<tool_call>\n{\"name\": ..., \"arguments\": { ... }}\n</tool_call>)."})
+                            _req.add_tool_response_messages(self.processing_class, [ToolResponse(text=error_message)])
                             finish_reason_type = FinishReasonTypeEnum.STOP
                             _req.state = AsyncRolloutRequestStateEnum.COMPLETED
                             break
@@ -988,7 +1125,7 @@ class SGLangRollout(BaseRollout):
                             break
             elif _req.state == AsyncRolloutRequestStateEnum.INTERACTING:
                 user_turns += 1
-                messages = [{"role": x.role, "content": x.content} for x in _req.messages]
+                messages = _req.messages
 
                 # Get interaction by name from interaction_kwargs
                 interaction_name = _req.interaction_kwargs.get(
@@ -1017,6 +1154,7 @@ class SGLangRollout(BaseRollout):
                     break
                 else:
                     _req.add_user_message(self.processing_class, content)
+                    # TODO: check if this is correct to use input_ids.size(-1) instead of len(_req.input_ids)
                     if _req.input_ids.size(-1) >= self.config.max_model_len:
                         finish_reason_type = FinishReasonTypeEnum.STOP
                         break
@@ -1025,6 +1163,9 @@ class SGLangRollout(BaseRollout):
 
         if current_turns >= self.config.multi_turn.max_assistant_turns:
             finish_reason_type = FinishReasonTypeEnum.STOP
+            if _req.messages[-1].tool_calls is not None:
+                if self.config.skip_tokenizer_init:
+                    _req.messages[-1].tool_calls = None
 
         # Calculate the reward for each tool
         async def calc_reward_and_release_fn(name: str, tool: BaseTool):
@@ -1038,15 +1179,38 @@ class SGLangRollout(BaseRollout):
             tool_reward_tasks.append(calc_reward_and_release_fn(name, tool))
         tool_reward_scores = await asyncio.gather(*tool_reward_tasks)
         tool_reward_scores = dict(tool_reward_scores)
-        all_rewards = {**tool_reward_scores, **{"user_turn_rewards": user_turn_rewards}}
+        all_rewards = {**tool_reward_scores, **{"user_turn_rewards": user_turn_rewards, "user_turns": user_turns}}
         _req.finalize(self.processing_class, all_rewards, finish_reason_type)
 
         if self.config.calculate_log_probs:
             debug_sampling_params = {**self.sampling_params}
             debug_sampling_params["max_new_tokens"] = 0
+            def _to_engine_input_ids(x):
+                # Accept torch.Tensor / np.ndarray / list-like and return list[list[int]]
+                if isinstance(x, torch.Tensor):
+                    lst = x.detach().cpu().tolist()
+                elif isinstance(x, np.ndarray):
+                    lst = x.tolist()
+                elif isinstance(x, list):
+                    lst = x
+                else:
+                    raise TypeError(f"Unsupported input_ids type for engine: {type(x)}")
+
+                # Empty -> return as-is (edge case)
+                if len(lst) == 0:
+                    return lst
+
+                # If first element is list -> assume already batch of sequences
+                first = lst[0]
+                if isinstance(first, list):
+                    return lst
+                else:
+                    # Single sequence -> wrap into a batch
+                    return [lst]
+            input_ids_for_engine = _to_engine_input_ids(_req.input_ids)
             output = await self._engine.async_generate(
                 prompt=None,
-                input_ids=_req.input_ids,
+                input_ids=input_ids_for_engine,
                 sampling_params=debug_sampling_params,
                 return_logprob=True,
                 logprob_start_len=0,
@@ -1198,6 +1362,10 @@ class SGLangRollout(BaseRollout):
             src=self._device_mesh_cpu["tp"].mesh[0].item(),
             force_cpu_device=False,
         )
+        # free cache engine
+        if self._engine is not None and self._tp_rank == 0:
+            loop = asyncio.get_event_loop()
+            loop.run_until_complete(self._engine.flush_cache())
         # Construct the batch data
         prompt_ids, response_ids = [], []
         prompt_attention_mask, response_attention_mask = [], []
@@ -1251,8 +1419,8 @@ class SGLangRollout(BaseRollout):
             request_ids.append(req.request_id)
             if self.config.calculate_log_probs:
                 # extract output log_probs
-                output_logprobs.append(req.rollout_log_probs[-len(req.response_ids) :])
-                rollout_output_token_ids.append(req.output_token_ids[-len(req.response_ids) :])
+                output_logprobs.append(req.rollout_log_probs[-len(req.response_ids[0]) :])
+                rollout_output_token_ids.append(req.output_token_ids[-len(req.response_ids[0]) :])
 
         prompt_ids = pad_sequence(
             prompt_ids,
@@ -1318,13 +1486,13 @@ class SGLangRollout(BaseRollout):
             output_logprobs = pad_sequence(output_logprobs, padding_value=0.0, batch_first=True)
             output_logprobs = pad_sequence_to_length(
                 output_logprobs, pad_token_id=0.0, max_seq_len=response_ids.shape[-1]
-            ).to(tgt_device)
+            ).to("cpu")
             rollout_output_token_ids = pad_sequence(
                 rollout_output_token_ids, padding_value=self.pad_token_id, batch_first=True
             )
             rollout_output_token_ids = pad_sequence_to_length(
                 rollout_output_token_ids, pad_token_id=self.pad_token_id, max_seq_len=response_ids.shape[-1]
-            ).to(tgt_device)
+            ).to("cpu")
 
         input_ids = torch.cat((prompt_ids, response_ids), dim=-1)
         attention_mask = torch.cat((prompt_attention_mask, response_attention_mask), dim=-1)
@@ -1346,11 +1514,27 @@ class SGLangRollout(BaseRollout):
             batch["rollout_log_probs"] = output_logprobs
             batch["rollout_output_token_ids"] = rollout_output_token_ids
 
+        # free cache engine
+        if self._engine is not None and self._tp_rank == 0:
+            loop = asyncio.get_event_loop()
+            loop.run_until_complete(self._engine.flush_cache())
+
         non_tensor_batch = {
             "messages": np.array(messages),
             "reward_scores": np.array(reward_scores),
             "request_id": np.array(request_ids),
+            # expose per-request metrics to downstream reward managers (e.g. checklist)
+            "metrics": np.array([req.metrics for req in sorted_output_req_list], dtype=object),
         }
+        # Provide per-request user turn counts to downstream metric utils
+        # so that compute_data_metrics() can log num_turns/{min,max,mean} to wandb
+        try:
+            user_turn_counts = [
+                (rs.get("user_turns", 0) if isinstance(rs, dict) else 0) for rs in reward_scores
+            ]
+            non_tensor_batch["__num_turns__"] = np.array(user_turn_counts)
+        except Exception:
+            pass
 
         is_multimodal = isinstance(self.processing_class, ProcessorMixin) and (
             hasattr(self.processing_class, "image_processor") or hasattr(self.model_hf_config, "vision_config")
@@ -1445,7 +1629,21 @@ class SGLangRollout(BaseRollout):
         ):
             if self._tool_schemas:
                 _tools_kwargs = prompts.non_tensor_batch["tools_kwargs"][data_idx]
-                _tool_schemas = [self._tool_map[k].get_openai_tool_schema() for k in _tools_kwargs.keys()]
+                # Prefer per-sample tool schemas if provided in non_tensor_batch["tools"][data_idx]
+                _tool_schemas = None
+                tools_arr = prompts.non_tensor_batch.get("tools", None)
+                if tools_arr is not None:
+                    per_sample_tools = tools_arr[data_idx]
+                    if per_sample_tools is not None:
+                        per_sample_tools_list = per_sample_tools if isinstance(per_sample_tools, list) else [per_sample_tools]
+                        parsed_tool_schemas = []
+                        for t in per_sample_tools_list:
+                            if isinstance(t, OpenAIFunctionToolSchema):
+                                parsed_tool_schemas.append(t)
+                            else:
+                                parsed_tool_schemas.append(OpenAIFunctionToolSchema.model_validate(t))
+                        if len(parsed_tool_schemas) > 0:
+                            _tool_schemas = parsed_tool_schemas
                 _input_ids = None
                 _attention_mask = None
             else: