Add host override to query specific pod (#86)

* Add rpm/tpm extension proc plugin * add license header * add extproc auto creation with httproute * add envoy patch policy * update go.mod * address comments * test * routing to specific pod * ext proc code refactoring * nit * bug fix for rpm/tpm check * remove unwanted file * add random router * for now rename ratelimiter dir -> gateway for monolithic plugin --------- Co-authored-by: varungupta <[email protected]>
vllm-project · Aug 21, 2024 · 851bd69 · 851bd69
1 parent 197ded0
commit 851bd69
Show file tree

Hide file tree

Showing 22 changed files with 1,060 additions and 494 deletions.
diff --git a/docs/development/app/README.md b/docs/development/app/README.md
@@ -47,7 +47,7 @@ helm get all eg -n envoy-gateway-system
 
 Port forward to the Envoy service:
 ```
-kubectl -n envoy-gateway-system port-forward service/${ENVOY_SERVICE} 8888:80 &
+kubectl -n envoy-gateway-system port-forward service/envoy-default-eg-e41e7b31 8888:80 &
 ```
 
 Start model router controller

diff --git a/docs/development/app/deployment.yaml b/docs/development/app/deployment.yaml
@@ -8,7 +8,7 @@ metadata:
     model.aibrix.ai: "llama2-70b"
     model.aibrix.ai/port: "8000"
 spec:
-  replicas: 1
+  replicas: 3
   selector:
     matchLabels:
       aibricks.com/lora-enabled: "true"
@@ -33,7 +33,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-
+            - name: MY_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
 ---
 # Debug only: Make sure pod can be visited from controller that deployed in mac.
 apiVersion: v1
@@ -50,7 +53,6 @@ spec:
       targetPort: 8000
       nodePort: 30080
   type: NodePort
-
 # ---
 # for test-purpose, if need to create HTTPRoute object manually
 # apiVersion: gateway.networking.k8s.io/v1
@@ -69,3 +71,27 @@ spec:
 #       backendRefs:
 #         - name: llama2-70b
 #           port: 8000
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: pod-reader
+  namespace: default
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: read-pods
+  namespace: default
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: default
+roleRef:
+  kind: Role
+  name: pod-reader
+  apiGroup: rbac.authorization.k8s.io
diff --git a/docs/development/app/gateway.yaml b/docs/development/app/gateway.yaml
@@ -14,4 +14,68 @@ spec:
   listeners:
     - name: http
       protocol: HTTP
-      port: 80
+      port: 80
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: envoy-gateway-config
+  namespace: envoy-gateway-system
+data:
+  envoy-gateway.yaml: |
+    apiVersion: gateway.envoyproxy.io/v1alpha1
+    kind: EnvoyGateway
+    provider:
+      type: Kubernetes
+    gateway:
+      controllerName: gateway.envoyproxy.io/gatewayclass-controller
+    extensionApis:
+      enableEnvoyPatchPolicy: true   
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyPatchPolicy
+metadata:
+  name: epp
+spec:
+  type: "JSONPatch"
+  targetRef:
+    group: gateway.networking.k8s.io
+    kind: Gateway
+    name: eg
+  jsonPatches:
+  - type: type.googleapis.com/envoy.config.route.v3.RouteConfiguration
+    name: "default/eg/http"
+    operation:
+      op: add
+      path: "/virtual_hosts/0/routes/0"
+      value:
+        name: orginal_route
+        match:
+          prefix: "/"
+          # headers:
+          #   # update ip address as needed and in production this config is not needed as backend will derive the pod ip
+          #   - name: "target-pod"
+          #     string_match:
+          #       exact: "10.244.1.3:8000"
+        route:  
+          cluster: original_destination_cluster
+          timeout: 1000s  # Increase route timeout
+        typed_per_filter_config:
+          "envoy.filters.http.ext_proc/envoyextensionpolicy/default/ext-proc-example/extproc/0":
+            "@type": "type.googleapis.com/envoy.config.route.v3.FilterConfig"
+            "config": {}
+  - type: "type.googleapis.com/envoy.config.cluster.v3.Cluster"
+    name: "envoy-patch-policy-override2"
+    operation:
+      op: add
+      path: ""
+      value:
+        name: original_destination_cluster
+        type: ORIGINAL_DST  
+        original_dst_lb_config:
+          use_http_header: true
+          http_header_name: "target-pod"
+        connect_timeout: 6s
+        lb_policy: CLUSTER_PROVIDED
+        dns_lookup_family: V4_ONLY
+
diff --git a/go.mod b/go.mod
@@ -1,6 +1,8 @@
 module github.com/aibrix/aibrix
 
-go 1.21
+go 1.22.5
+
+toolchain go1.22.6
 
 require (
 	github.com/onsi/ginkgo/v2 v2.14.0
@@ -18,53 +20,52 @@ require (
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/evanphx/json-patch/v5 v5.8.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.12.1 // indirect
+	github.com/evanphx/json-patch v5.9.0+incompatible // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
-	github.com/go-openapi/jsonpointer v0.20.0 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
-	github.com/google/uuid v1.3.1 // indirect
-	github.com/imdario/mergo v0.3.16 // indirect
+	github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/imdario/mergo v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.18.0 // indirect
-	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.45.0 // indirect
-	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/prometheus/client_golang v1.19.1 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/oauth2 v0.13.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/term v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.16.1 // indirect
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
@@ -75,3 +76,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace github.com/imdario/mergo v1.0.0 => dario.cat/mergo v0.3.16