Read model name from request body

config/gateway/gateway-plugin.yaml

@@ -50,3 +50,24 @@ spec:
                 fieldRef:
                   fieldPath: metadata.namespace
       serviceAccountName: aibrix-gateway-plugin
+---
+# this is a dummy route for incoming request and,
+# then request is routed to httproute using model name OR
+# request is routed based on the target for that model service
+# TODO (varun): check if this dummy route can be removed in future
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: reserved-router
+  namespace: aibrix-system
+spec:
+  parentRefs:
+    - name: aibrix-eg
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - name: aibrix-gateway-plugins
+          port: 50052

config/gateway/gateway.yaml

@@ -60,15 +60,10 @@ spec:
         match:
           prefix: "/"
           headers:
-            - name: "routing-strategy"
-              string_match:
-                exact: "random"
-            - name: "least-request"
-              string_match:
-                exact: "random"
-            - name: "routing-strategy"
-              string_match:
-                exact: "throughput"
+          - name: "routing-strategy"
+            string_match:
+              safe_regex:
+                regex: .*
         route:  
           cluster: original_destination_cluster
           timeout: 1000s  # Increase route timeout

docs/development/app/README.md

@@ -67,32 +67,42 @@ kubectl -n envoy-gateway-system port-forward service/envoy-aibrix-system-aibrix-
 
 # Add rpm/tpm config 
 ```shell
+# note: not mandatory to create user to access gateway API
+
 kubectl -n aibrix-system port-forward svc/aibrix-gateway-users 8090:8090 &
 
 curl http://localhost:8090/CreateUser \
   -H "Content-Type: application/json" \
   -d '{"name": "your-user-name","rpm": 100,"tpm": 1000}'
 ```
 
-Test request (ensure header model name matches with deployment's model name for routing)
+Test request
 ```shell
 curl -v http://localhost:8888/v1/chat/completions \
-  -H "user: your-user-name" \
-  -H "model: llama2-70b" \
   -H "Content-Type: application/json" \
   -H "Authorization: Bearer any_key" \
   -d '{
      "model": "llama2-70b",
      "messages": [{"role": "user", "content": "Say this is a test!"}],
      "temperature": 0.7
-   }' &
+   }'
+
+curl -v http://localhost:8888/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer any_key" \
+  -H "routing-strategy: random" \
+  -d '{
+     "model": "llama2-70b",
+     "messages": [{"role": "user", "content": "Say this is a test!"}],
+     "temperature": 0.7
+   }'
+
 
 # least-request based
 for i in {1..10}; do
   curl -v http://localhost:8888/v1/chat/completions \
   -H "user: your-user-name" \
   -H "routing-strategy: least-request" \
-  -H "model: llama2-70b" \
   -H "Content-Type: application/json" \
   -H "Authorization: Bearer any_key" \
   -d '{
@@ -106,8 +116,7 @@ done
 for i in {1..10}; do
   curl -v http://localhost:8888/v1/chat/completions \
   -H "user: your-user-name" \
-  -H "routing-strategy: throughput" \
-  -H "model: llama2-70b" \
+  -H "routing-strategy: random" \
   -H "Content-Type: application/json" \
   -H "Authorization: Bearer any_key" \
   -d '{

pkg/plugins/gateway/algorithms/least_request.go

@@ -77,5 +77,5 @@ func (r leastRequestRouter) Route(ctx context.Context, pods map[string]*v1.Pod)
 		}
 	}
 
-	return targetPodIP, nil
+	return targetPodIP + ":" + podPort, nil
 }

pkg/plugins/gateway/algorithms/random.go

@@ -46,5 +46,5 @@ func (r randomRouter) Route(ctx context.Context, pods map[string]*v1.Pod) (strin
 		k--
 	}
 
-	return selectedPod.Status.PodIP, nil
+	return selectedPod.Status.PodIP + ":" + podPort, nil
 }

pkg/plugins/gateway/algorithms/router.go

@@ -30,7 +30,7 @@ const (
 	throughput_generation = "avg_generation_throughput_toks_per_s"
 	latency               = "e2e_request_latency_seconds_sum"
 
-	podPort = 8000
+	podPort = "8000"
 )
 
 type Router interface {

pkg/plugins/gateway/algorithms/throughput.go

@@ -74,5 +74,5 @@ func (r throughputRouter) Route(ctx context.Context, pods map[string]*v1.Pod) (s
 		}
 	}
 
-	return targetPodIP, nil
+	return targetPodIP + ":" + podPort, nil
 }