From e96b3d440d7fac4405b69d9f39a9ee4851ee254c Mon Sep 17 00:00:00 2001 From: WANDY666 <1060304770@qq.com> Date: Thu, 8 Jan 2026 11:21:58 +0000 Subject: [PATCH 1/2] update openai_api --- lightllm/server/api_models.py | 16 +++++++++++++--- lightllm/server/api_openai.py | 3 ++- lightllm/server/build_prompt.py | 6 +++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/lightllm/server/api_models.py b/lightllm/server/api_models.py index 7b9cdd501..df684d4dd 100644 --- a/lightllm/server/api_models.py +++ b/lightllm/server/api_models.py @@ -1,7 +1,7 @@ import time import uuid -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from typing import Any, Dict, List, Optional, Union, Literal, ClassVar from transformers import GenerationConfig @@ -21,6 +21,14 @@ class Message(BaseModel): content: Union[str, List[MessageContent]] +class CharacterMessage(BaseModel): + """Message format for character-based chat, where role is inferred from name.""" + + name: str + content: Union[str, List[MessageContent]] + role: Optional[str] = None # Optional, can be inferred from role_setting + + class Function(BaseModel): """Function descriptions.""" @@ -105,7 +113,7 @@ def _normalize_role(cls, v): raise ValueError("'role' must be a string") -ChatCompletionMessageParam = Union[ChatCompletionMessageGenericParam, Message] +ChatCompletionMessageParam = Union[ChatCompletionMessageGenericParam, Message, CharacterMessage] class CompletionRequest(BaseModel): @@ -176,6 +184,8 @@ def apply_loaded_defaults(cls, data: Any): class ChatCompletionRequest(BaseModel): + model_config = ConfigDict(populate_by_name=True) + model: str messages: List[ChatCompletionMessageParam] function_call: Optional[str] = "none" @@ -216,7 +226,7 @@ class ChatCompletionRequest(BaseModel): top_k: Optional[int] = -1 repetition_penalty: Optional[float] = 1.0 ignore_eos: Optional[bool] = False - role_settings: Optional[Dict[str, str]] = None + role_settings: Optional[Dict[str, str]] = Field(default=None, alias="role_setting") character_settings: Optional[List[Dict[str, str]]] = None # Class variables to store loaded default values diff --git a/lightllm/server/api_openai.py b/lightllm/server/api_openai.py index 6a8c232dc..cdac4ab5c 100644 --- a/lightllm/server/api_openai.py +++ b/lightllm/server/api_openai.py @@ -105,7 +105,8 @@ def _get_history_tool_calls_cnt(request: ChatCompletionRequest) -> int: messages = getattr(request, "messages", []) idx = 0 for msg in messages: - if msg.role == "assistant": + role = getattr(msg, "role", None) + if role == "assistant": tool_calls = getattr(msg, "tool_calls", None) idx += len(list(tool_calls)) if tool_calls is not None else 0 # noqa return idx diff --git a/lightllm/server/build_prompt.py b/lightllm/server/build_prompt.py index f770459a5..131ee76b4 100644 --- a/lightllm/server/build_prompt.py +++ b/lightllm/server/build_prompt.py @@ -17,7 +17,11 @@ async def build_prompt(request, tools) -> str: global tokenizer # pydantic格式转成dict， 否则，当根据tokenizer_config.json拼template时，Jinja判断无法识别 messages = [m.model_dump(by_alias=True, exclude_none=True) for m in request.messages] - kwargs = {"conversation": messages} + kwargs = { + "conversation": messages, + # 假设 request 对象里有这个字段，或者你想传空 + "system_instruction": getattr(request, "system_instruction", ""), + } if request.character_settings: kwargs["character_settings"] = request.character_settings if request.role_settings: From 40382c89dd86a55b9d56ea212afce9e64203f793 Mon Sep 17 00:00:00 2001 From: WANDY666 <1060304770@qq.com> Date: Thu, 8 Jan 2026 11:58:40 +0000 Subject: [PATCH 2/2] fix --- lightllm/server/api_models.py | 1 + lightllm/server/build_prompt.py | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lightllm/server/api_models.py b/lightllm/server/api_models.py index df684d4dd..de6d6ba20 100644 --- a/lightllm/server/api_models.py +++ b/lightllm/server/api_models.py @@ -228,6 +228,7 @@ class ChatCompletionRequest(BaseModel): ignore_eos: Optional[bool] = False role_settings: Optional[Dict[str, str]] = Field(default=None, alias="role_setting") character_settings: Optional[List[Dict[str, str]]] = None + system_instruction: Optional[str] = None # Class variables to store loaded default values _loaded_defaults: ClassVar[Dict[str, Any]] = {} diff --git a/lightllm/server/build_prompt.py b/lightllm/server/build_prompt.py index 131ee76b4..cff2ec127 100644 --- a/lightllm/server/build_prompt.py +++ b/lightllm/server/build_prompt.py @@ -17,15 +17,12 @@ async def build_prompt(request, tools) -> str: global tokenizer # pydantic格式转成dict， 否则，当根据tokenizer_config.json拼template时，Jinja判断无法识别 messages = [m.model_dump(by_alias=True, exclude_none=True) for m in request.messages] - kwargs = { - "conversation": messages, - # 假设 request 对象里有这个字段，或者你想传空 - "system_instruction": getattr(request, "system_instruction", ""), - } + kwargs = {"conversation": messages} if request.character_settings: kwargs["character_settings"] = request.character_settings if request.role_settings: kwargs["role_setting"] = request.role_settings + kwargs["system_instruction"] = request.system_instruction if request.chat_template_kwargs: kwargs.update(request.chat_template_kwargs)