Increase the connection bufferLimit to avoid end_of_stream=false case

ResponseBody.EndOfStream sometimes is false even we use buffered mode. Current implementation does not take this into the consideration. Signed-off-by: Jiaxin Shan <[email protected]>
vllm-project · Feb 14, 2025 · 97dfb62 · 97dfb62
1 parent c4060bb
commit 97dfb62
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/config/gateway/gateway.yaml b/config/gateway/gateway.yaml
@@ -29,7 +29,7 @@ spec:
       kind: Gateway
       name: aibrix-eg
   connection:
-    bufferLimit: 262144
+    bufferLimit: 1048576
 ---
 apiVersion: gateway.envoyproxy.io/v1alpha1
 kind: EnvoyExtensionPolicy

diff --git a/test/e2e/routing_strategy_test.go b/test/e2e/routing_strategy_test.go
@@ -26,6 +26,7 @@ import (
 	"github.com/openai/openai-go"
 	"github.com/openai/openai-go/option"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestPrefixCacheModelInference(t *testing.T) {
@@ -63,7 +64,7 @@ func getTargetPodFromChatCompletion(t *testing.T, message string) string {
 		}),
 		Model: openai.F(modelName),
 	})
-	assert.NoError(t, err, "chat completitions failed")
+	require.NoError(t, err, "chat completitions failed %v", err)
 	assert.Equal(t, modelName, chatCompletion.Model)
 
 	return dst.Header.Get("target-pod")