Add GPU Optimizer deployment and update configurations

config/default/kustomization.yaml

@@ -23,6 +23,7 @@ resources:
 - ../rbac
 - ../manager
 - ../gateway
+- ../gpu-optimizer
 - ../dependency/kuberay-operator
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml

config/gpu-optimizer/deployment.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gpu-optimizer
+  namespace: aibrix-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: gpu-optimizer
+  template:
+    metadata:
+      labels:
+        app: gpu-optimizer
+    spec:
+      serviceAccountName: gpu-optimizer
+      automountServiceAccountToken: true
+      containers:
+      - name: gpu-optimizer
+        image: aibrix/runtime:nightly
+        command: ["python", "-m", "aibrix.gpu_optimizer.app"]
+        ports:
+        - containerPort: 8080
+        env:
+          - name: NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: REDIS_HOST
+            value: aibrix-redis-master.aibrix-system.svc.cluster.local

config/gpu-optimizer/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+- deployment.yaml
+- service.yaml
+- rbac.yaml

config/gpu-optimizer/rbac.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gpu-optimizer
+  namespace: aibrix-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: aibrix-system
+  name: gpu-optimizer
+rules:
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gpu-optimizer
+  namespace: aibrix-system
+subjects:
+  - kind: ServiceAccount
+    name: gpu-optimizer
+    namespace: aibrix-system
+roleRef:
+  kind: Role
+  name: gpu-optimizer
+  apiGroup: rbac.authorization.k8s.io

config/gpu-optimizer/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: gpu-optimizer
+  namespace: aibrix-system
+spec:
+  selector:
+    app: gpu-optimizer
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+  type: ClusterIP

config/overlays/vke/default/kustomization.yaml

@@ -7,6 +7,7 @@ resources:
 - ../../../rbac
 - manager
 - gateway
+- ../../../gpu-optimizer
 - ../../../dependency/kuberay-operator
 
 

development/simulator/deployment-a100.yaml

@@ -150,11 +150,11 @@ spec:
     apiVersion: apps/v1
     kind: Deployment
     name: simulator-llama2-7b-a100
-  minReplicas: 0
+  minReplicas: 1
   maxReplicas: 10
   targetMetric: "avg_prompt_throughput_toks_per_s" # Ignore if metricsSources is configured
   metricsSources: 
-    - endpoint: gpu-optimizer.aibrix-system.svc.cluster.local:8080
+    - endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080
       path: /metrics/aibrix-system/simulator-llama2-7b-a100
       metric: "vllm:deployment_replicas"
   targetValue: "1"

development/simulator/deployment-a40.yaml

@@ -149,11 +149,11 @@ spec:
     apiVersion: apps/v1
     kind: Deployment
     name: simulator-llama2-7b-a40
-  minReplicas: 0
+  minReplicas: 1
   maxReplicas: 10
   targetMetric: "avg_prompt_throughput_toks_per_s" # Ignore if metricsSources is configured
   metricsSources: 
-    - endpoint: gpu-optimizer.aibrix-system.svc.cluster.local:8080
+    - endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080
       path: /metrics/aibrix-system/simulator-llama2-7b-a40
       metric: "vllm:deployment_replicas"
   targetValue: "1"

python/aibrix/aibrix/gpu_optimizer/README.md

@@ -28,15 +28,15 @@ kubectl -n aibrix-system port-forward svc/aibrix-redis-master 6379:6379 1>/dev/n
 # Or use make
 make debug-init
 
-python optimizer/profiling/gen-profile.py simulator-llama2-7b-a100 -o "redis://localhost:6379/?model=llama2-7b"
+python optimizer/profiling/gen_profile.py simulator-llama2-7b-a100 -o "redis://localhost:6379/?model=llama2-7b"
 # Or use make
 make DP=simulator-llama2-7b-a100 gen-profile
 ```
 
 5. Deploy GPU Optimizer
 ```shell
 kubectl apply -f deployment.yaml
-kubectl -n aibrix-system port-forward svc/gpu-optimizer 8080:8080 1>/dev/null 2>&1 &
+kubectl -n aibrix-system port-forward svc/aibrix-gpu-optimizer 8080:8080 1>/dev/null 2>&1 &
 
 # Or use make
 make deploy
@@ -47,7 +47,7 @@ make deploy
 5. Start workload and see how model scale. Benchmark toolkit can be used to generate workload as:
 ```shell
 # Make sure gateway's local access, see docs/development/simulator/README.md for details.
-python optimizer/profiling/gpu-benchmark.py --backend=vllm --port 8888 --request-rate=10 --num-prompts=100 --input_len 2000 --output_len 128 --model=llama2-7b
+python optimizer/profiling/gpu_benchmark.py --backend=vllm --port 8888 --request-rate=10 --num-prompts=100 --input_len 2000 --output_len 128 --model=llama2-7b
 ```
 
 6. Observability: visit http://localhost:8080/dash/llama2-7b for workload pattern visualization. A independent visualization demo can access by: