💥👽 Update ErrorResponse and supported vllm versions (#102)

* ⬆️ Bump vllm support

Signed-off-by: Evaline Ju <[email protected]>

* 👽⬆️ Breaking ErrorResponse changes with vllm 0.10.1

Signed-off-by: Evaline Ju <[email protected]>

* ✨ Conform to original detectors API

Signed-off-by: Evaline Ju <[email protected]>

* 🔧 Update Dockerfile vllm version

Signed-off-by: Evaline Ju <[email protected]>

* 🥅 Handle request validation

Signed-off-by: Evaline Ju <[email protected]>

* 🥅 Keep current request validation handling for other endpoints

Signed-off-by: Evaline Ju <[email protected]>

* 🐛 Fix import

Signed-off-by: Evaline Ju <[email protected]>

* 🐛♻️ Format validation errors

Signed-off-by: Evaline Ju <[email protected]>

---------

Signed-off-by: Evaline Ju <[email protected]>

Author: evaline-ju

Dockerfile

@@ -8,7 +8,7 @@ ARG BASE_UBI_IMAGE_TAG=9.6
 ARG PYTHON_VERSION=3.12
 
 ### Build layer
-FROM quay.io/vllm/vllm-cuda:0.10.0.2 as build
+FROM quay.io/vllm/vllm-cuda:0.11.0.1 as build
 
 ARG PYTHON_VERSION
 ENV PYTHON_VERSION=${PYTHON_VERSION}

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vllm-detector-adapter"
-version = "0.8.0"
+version = "0.9.0"
 authors = [
     { name = "Gaurav Kumbhat", email = "[email protected]" },
     { name = "Evaline Ju", email = "[email protected]" },
@@ -16,8 +16,8 @@ dependencies = ["orjson>=3.10.16,<3.11"]
 vllm-tgis-adapter = ["vllm-tgis-adapter>=0.8.0,<0.9.0"]
 vllm = [
     # Note: vllm < 0.10.0 has issues with transformers >= 4.54.0
-    "vllm @ git+https://github.com/vllm-project/vllm.git@v0.10.0 ; sys_platform == 'darwin'",
-    "vllm>=0.10.0,<0.10.1 ; sys_platform != 'darwin'",
+    "vllm @ git+https://github.com/vllm-project/vllm.git@v0.11.0 ; sys_platform == 'darwin'",
+    "vllm>=0.10.1,<0.11.1 ; sys_platform != 'darwin'",
 ]
 
 ## Dev Extra Sets ##

tests/generative_detectors/test_base.py

@@ -237,5 +237,5 @@ def test_content_analysis_errorresponse_verification(detection_base):
         result = asyncio.run(base_instance.content_analysis(content_request))
 
         assert isinstance(result, ErrorResponse)
-        assert result.type == "BadRequestError"
-        assert "does not have content" in result.message
+        assert result.error.type == "BadRequestError"
+        assert "does not have content" in result.error.message

tests/generative_detectors/test_granite_guardian.py

@@ -414,10 +414,10 @@ def test__make_tools_request_no_tool_calls(granite_guardian_detection):
     )
     processed_request = granite_guardian_detection_instance._make_tools_request(request)
     assert type(processed_request) == ErrorResponse
-    assert processed_request.code == HTTPStatus.BAD_REQUEST
+    assert processed_request.error.code == HTTPStatus.BAD_REQUEST
     assert (
         "no assistant message was provided with tool_calls for analysis"
-        in processed_request.message
+        in processed_request.error.message
     )
 
 
@@ -437,9 +437,10 @@ def test__make_tools_request_random_risk(granite_guardian_detection):
     )
     processed_request = granite_guardian_detection_instance._make_tools_request(request)
     assert type(processed_request) == ErrorResponse
-    assert processed_request.code == HTTPStatus.BAD_REQUEST
+    assert processed_request.error.code == HTTPStatus.BAD_REQUEST
     assert (
-        "tools analysis is not supported with given risk" in processed_request.message
+        "tools analysis is not supported with given risk"
+        in processed_request.error.message
     )
 
 
@@ -773,8 +774,10 @@ def test_request_to_chat_completion_request_empty_kwargs(granite_guardian_detect
         )
     )
     assert type(chat_request) == ErrorResponse
-    assert chat_request.code == HTTPStatus.BAD_REQUEST
-    assert "No risk_name or criteria_id for context analysis" in chat_request.message
+    assert chat_request.error.code == HTTPStatus.BAD_REQUEST
+    assert (
+        "No risk_name or criteria_id for context analysis" in chat_request.error.message
+    )
 
 
 def test_request_to_chat_completion_request_empty_guardian_config(
@@ -793,8 +796,10 @@ def test_request_to_chat_completion_request_empty_guardian_config(
         )
     )
     assert type(chat_request) == ErrorResponse
-    assert chat_request.code == HTTPStatus.BAD_REQUEST
-    assert "No risk_name or criteria_id for context analysis" in chat_request.message
+    assert chat_request.error.code == HTTPStatus.BAD_REQUEST
+    assert (
+        "No risk_name or criteria_id for context analysis" in chat_request.error.message
+    )
 
 
 def test_request_to_chat_completion_request_missing_risk_name_and_criteria_id(
@@ -816,8 +821,10 @@ def test_request_to_chat_completion_request_missing_risk_name_and_criteria_id(
         )
     )
     assert type(chat_request) == ErrorResponse
-    assert chat_request.code == HTTPStatus.BAD_REQUEST
-    assert "No risk_name or criteria_id for context analysis" in chat_request.message
+    assert chat_request.error.code == HTTPStatus.BAD_REQUEST
+    assert (
+        "No risk_name or criteria_id for context analysis" in chat_request.error.message
+    )
 
 
 def test_request_to_chat_completion_request_unsupported_risk_name(
@@ -839,10 +846,10 @@ def test_request_to_chat_completion_request_unsupported_risk_name(
         )
     )
     assert type(chat_request) == ErrorResponse
-    assert chat_request.code == HTTPStatus.BAD_REQUEST
+    assert chat_request.error.code == HTTPStatus.BAD_REQUEST
     assert (
         "risk_name or criteria_id foo is not compatible with context analysis"
-        in chat_request.message
+        in chat_request.error.message
     )
 
 
@@ -1085,10 +1092,10 @@ def test_context_analyze_unsupported_risk(
             granite_guardian_detection_instance.context_analyze(context_request)
         )
         assert type(detection_response) == ErrorResponse
-        assert detection_response.code == HTTPStatus.BAD_REQUEST
+        assert detection_response.error.code == HTTPStatus.BAD_REQUEST
         assert (
             "risk_name or criteria_id boo is not compatible with context analysis"
-            in detection_response.message
+            in detection_response.error.message
         )
 
 
@@ -1395,8 +1402,8 @@ def test_chat_detection_errors_on_stream(granite_guardian_detection):
         granite_guardian_detection_instance.chat(chat_request)
     )
     assert type(detection_response) == ErrorResponse
-    assert detection_response.code == HTTPStatus.BAD_REQUEST.value
-    assert "streaming is not supported" in detection_response.message
+    assert detection_response.error.code == HTTPStatus.BAD_REQUEST.value
+    assert "streaming is not supported" in detection_response.error.message
 
 
 def test_chat_detection_errors_on_jinja_template_error(granite_guardian_detection):
@@ -1414,8 +1421,8 @@ def test_chat_detection_errors_on_jinja_template_error(granite_guardian_detectio
             granite_guardian_detection_instance.chat(chat_request)
         )
         assert type(detection_response) == ErrorResponse
-        assert detection_response.code == HTTPStatus.BAD_REQUEST.value
-        assert "Template error" in detection_response.message
+        assert detection_response.error.code == HTTPStatus.BAD_REQUEST.value
+        assert "Template error" in detection_response.error.message
 
 
 def test_chat_detection_errors_on_undefined_jinja_error(granite_guardian_detection):
@@ -1433,8 +1440,8 @@ def test_chat_detection_errors_on_undefined_jinja_error(granite_guardian_detecti
             granite_guardian_detection_instance.chat(chat_request)
         )
         assert type(detection_response) == ErrorResponse
-        assert detection_response.code == HTTPStatus.BAD_REQUEST.value
-        assert "Template error" in detection_response.message
+        assert detection_response.error.code == HTTPStatus.BAD_REQUEST.value
+        assert "Template error" in detection_response.error.message
 
 
 def test_risk_bank_extraction(granite_guardian_detection):

tests/generative_detectors/test_llama_guard.py

@@ -375,7 +375,7 @@ def test_chat_detection_with_tools(llama_guard_detection):
     )
     response = asyncio.run(llama_guard_detection_instance.chat(chat_request))
     assert type(response) == ErrorResponse
-    assert response.code == HTTPStatus.NOT_IMPLEMENTED
+    assert response.error.code == HTTPStatus.NOT_IMPLEMENTED
 
 
 def test_context_analyze(llama_guard_detection):
@@ -392,7 +392,7 @@ def test_context_analyze(llama_guard_detection):
         llama_guard_detection_instance.context_analyze(context_request)
     )
     assert type(response) == ErrorResponse
-    assert response.code == HTTPStatus.NOT_IMPLEMENTED
+    assert response.error.code == HTTPStatus.NOT_IMPLEMENTED
 
 
 def test_generation_analyze(llama_guard_detection, llama_guard_completion_response):

tests/test_protocol.py

@@ -251,9 +251,9 @@ def test_response_from_single_content_detection_missing_content():
     assert type(detection_response) == ErrorResponse
     assert (
         "Choice 0 from chat completion does not have content"
-        in detection_response.message
+        in detection_response.error.message
     )
-    assert detection_response.code == HTTPStatus.BAD_REQUEST.value
+    assert detection_response.error.code == HTTPStatus.BAD_REQUEST.value
 
 
 #### General detection response tests
@@ -355,9 +355,9 @@ def test_response_from_completion_response_missing_content():
     assert type(detection_response) == ErrorResponse
     assert (
         "Choice 1 from chat completion does not have content"
-        in detection_response.message
+        in detection_response.error.message
     )
-    assert detection_response.code == HTTPStatus.BAD_REQUEST.value
+    assert detection_response.error.code == HTTPStatus.BAD_REQUEST.value
 
 
 def test_response_from_empty_string_content_detection():
@@ -388,6 +388,6 @@ def test_response_from_empty_string_content_detection():
     assert type(detection_response) == ErrorResponse
     assert (
         "Choice 0 from chat completion does not have content"
-        in detection_response.message
+        in detection_response.error.message
     )
-    assert detection_response.code == HTTPStatus.BAD_REQUEST.value
+    assert detection_response.error.code == HTTPStatus.BAD_REQUEST.value

vllm_detector_adapter/api_server.py

@@ -1,10 +1,12 @@
 # Standard
 from argparse import Namespace
+from http import HTTPStatus
 import inspect
 import signal
 
 # Third Party
 from fastapi import Request
+from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
 from starlette.datastructures import State
 from vllm.config import ModelConfig
@@ -14,7 +16,7 @@
 from vllm.entrypoints.logger import RequestLogger
 from vllm.entrypoints.openai import api_server
 from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
-from vllm.entrypoints.openai.protocol import ErrorResponse
+from vllm.entrypoints.openai.protocol import ErrorInfo, ErrorResponse
 from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
 from vllm.entrypoints.openai.tool_parsers import ToolParserManager
 from vllm.utils import FlexibleArgumentParser, is_valid_ipv6_address, set_ulimit
@@ -41,6 +43,7 @@
     # Third Party
     from vllm.reasoning import ReasoningParserManager
 
+
 TIMEOUT_KEEP_ALIVE = 5  # seconds
 
 # Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
@@ -162,6 +165,37 @@ def signal_handler(*_) -> None:
         # Use vllm build_app which adds middleware
         app = api_server.build_app(args)
 
+        # Override exception handler to flatten errors for detectors API
+        @app.exception_handler(RequestValidationError)
+        async def validation_exception_handler(
+            request: Request, exc: RequestValidationError
+        ):
+            exc_str = str(exc)
+            errors_str = str(exc.errors())
+            message = None
+            if exc.errors() and errors_str and errors_str != exc_str:
+                message = f"{exc_str} {errors_str}"
+            else:
+                message = exc_str
+
+            error_info = ErrorInfo(
+                message=message,
+                type=HTTPStatus.BAD_REQUEST.phrase,
+                code=HTTPStatus.BAD_REQUEST,
+            )
+
+            if request.url.path.startswith("/api/v1/text"):
+                # Flatten detectors API request validation errors
+                return JSONResponse(
+                    content=error_info.model_dump(), status_code=HTTPStatus.BAD_REQUEST
+                )
+            else:
+                # vLLM general request validation error handling
+                err = ErrorResponse(error=error_info)
+                return JSONResponse(
+                    content=err.model_dump(), status_code=HTTPStatus.BAD_REQUEST
+                )
+
         # api_server.init_app_state takes vllm_config
         # ref. https://github.com/vllm-project/vllm/pull/16572
         if hasattr(engine_client, "get_vllm_config"):
@@ -213,9 +247,9 @@ async def create_chat_detection(request: ChatDetectionRequest, raw_request: Requ
     detector_response = await chat_detection(raw_request).chat(request, raw_request)
 
     if isinstance(detector_response, ErrorResponse):
-        # ErrorResponse includes code and message, corresponding to errors for the detectorAPI
         return JSONResponse(
-            content=detector_response.model_dump(), status_code=detector_response.code
+            content=detector_response.error.model_dump(),
+            status_code=detector_response.error.code,
         )
 
     elif isinstance(detector_response, DetectionResponse):
@@ -235,9 +269,9 @@ async def create_context_doc_detection(
     )
 
     if isinstance(detector_response, ErrorResponse):
-        # ErrorResponse includes code and message, corresponding to errors for the detectorAPI
         return JSONResponse(
-            content=detector_response.model_dump(), status_code=detector_response.code
+            content=detector_response.error.model_dump(),
+            status_code=detector_response.error.code,
         )
 
     elif isinstance(detector_response, DetectionResponse):
@@ -256,9 +290,9 @@ async def create_contents_detection(
         request, raw_request
     )
     if isinstance(detector_response, ErrorResponse):
-        # ErrorResponse includes code and message, corresponding to errors for the detectorAPI
         return JSONResponse(
-            content=detector_response.model_dump(), status_code=detector_response.code
+            content=detector_response.error.model_dump(),
+            status_code=detector_response.error.code,
         )
 
     elif isinstance(detector_response, ContentsDetectionResponse):
@@ -277,9 +311,9 @@ async def create_generation_detection(
         request, raw_request
     )
     if isinstance(detector_response, ErrorResponse):
-        # ErrorResponse includes code and message, corresponding to errors for the detectorAPI
         return JSONResponse(
-            content=detector_response.model_dump(), status_code=detector_response.code
+            content=detector_response.error.model_dump(),
+            status_code=detector_response.error.code,
         )
 
     elif isinstance(detector_response, DetectionResponse):

vllm_detector_adapter/generative_detectors/base.py

@@ -12,6 +12,7 @@
 from vllm.entrypoints.openai.protocol import (
     ChatCompletionRequest,
     ChatCompletionResponse,
+    ErrorInfo,
     ErrorResponse,
 )
 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
@@ -156,9 +157,11 @@ def preprocess_request(  # noqa: F811
         # Tools detection is not generalized
         if request.tools:
             return ErrorResponse(
-                message="tools are not supported for the detector",
-                type="NotImplementedError",
-                code=HTTPStatus.NOT_IMPLEMENTED.value,
+                error=ErrorInfo(
+                    message="tools are not supported for the detector",
+                    type="NotImplementedError",
+                    code=HTTPStatus.NOT_IMPLEMENTED.value,
+                )
             )
         return request
 
@@ -241,9 +244,11 @@ async def process_chat_completion_with_scores(
         # object would look different, and content would have to be aggregated.
         if chat_completion_request.stream:
             return ErrorResponse(
-                message="streaming is not supported for the detector",
-                type="BadRequestError",
-                code=HTTPStatus.BAD_REQUEST.value,
+                error=ErrorInfo(
+                    message="streaming is not supported for the detector",
+                    type="BadRequestError",
+                    code=HTTPStatus.BAD_REQUEST.value,
+                )
             )
 
         # Manually set logprobs to True to calculate score later on
@@ -271,9 +276,11 @@ async def process_chat_completion_with_scores(
             # Users _may_ be able to correct some of these errors by changing the input
             # but the error message may not be directly user-comprehensible
             chat_response = ErrorResponse(
-                message=e.message or "Template error",
-                type="BadRequestError",
-                code=HTTPStatus.BAD_REQUEST.value,
+                error=ErrorInfo(
+                    message=e.message or "Template error",
+                    type="BadRequestError",
+                    code=HTTPStatus.BAD_REQUEST.value,
+                )
             )
 
         logger.debug("Raw chat completion response: %s", chat_response)
@@ -376,9 +383,11 @@ async def context_analyze(
         # Return "not implemented" here since context analysis may not
         # generally apply to all models at this time
         return ErrorResponse(
-            message="context analysis is not supported for the detector",
-            type="NotImplementedError",
-            code=HTTPStatus.NOT_IMPLEMENTED.value,
+            error=ErrorInfo(
+                message="context analysis is not supported for the detector",
+                type="NotImplementedError",
+                code=HTTPStatus.NOT_IMPLEMENTED.value,
+            )
         )
 
     async def content_analysis(