Refactor the gateway-plugin and metadata service manifests (#531)

* Refactor the gateway-plugin and metadata service manifests Signed-off-by: Jiaxin Shan <[email protected]> * Update the CI workflow Signed-off-by: Jiaxin Shan <[email protected]> --------- Signed-off-by: Jiaxin Shan <[email protected]>
vllm-project · Dec 16, 2024 · 1b3619f · 1b3619f
1 parent 2b2f9ee
commit 1b3619f
Show file tree

Hide file tree

Showing 20 changed files with 67 additions and 66 deletions.
diff --git a/.github/workflows/installation-tests.yml b/.github/workflows/installation-tests.yml
@@ -31,7 +31,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        image: [controller-manager, plugins, runtime, users]
+        image: [controller-manager, gateway-plugins, runtime, metadata-service]
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -84,7 +84,7 @@ jobs:
 
       - name: Load image into Kind
         run: |
-          for image in controller-manager plugins runtime users; do
+          for image in controller-manager gateway-plugins runtime metadata-service; do
             docker load < ${image}-image/${image}.tar
           
             # Retag the image
@@ -99,9 +99,10 @@ jobs:
       - name: Deploy controller with the built image
         run: |
           kubectl create -k config/dependency
-          cd config/manager && kustomize edit set image controller=aibrix/controller-manager:${{ github.sha }}
-          cd ${{ github.workspace }}
-          cd config/gateway && kustomize edit set image plugins=aibrix/plugins:${{ github.sha }} && kustomize edit set image users=aibrix/users:${{ github.sha }}
+          cd config/default
+          kustomize edit set image controller=aibrix/controller-manager:${{ github.sha }}
+          kustomize edit set image gateway-plugins=aibrix/gateway-plugins:${{ github.sha }}
+          kustomize edit set image metadata-service=aibrix/metadata-service:${{ github.sha }}
           cd ${{ github.workspace }}
           kubectl create -k config/default
 

diff --git a/Makefile b/Makefile
@@ -4,7 +4,7 @@ GIT_COMMIT_HASH ?= $(shell git rev-parse HEAD)
 # Image URL to use all building/pushing image targets
 AIBRIX_CONTAINER_REGISTRY_NAMESPACE ?= aibrix
 DOCKERFILE_PATH ?= build/container
-IMAGES := controller-manager plugins runtime users
+IMAGES := controller-manager gateway-plugins runtime metadata-service
 
 # note: this is not being used, only for tracking some commands we have not updated yet.
 IMG ?= ${AIBRIX_CONTAINER_REGISTRY_NAMESPACE}/controller-manager:${GIT_COMMIT_HASH}
@@ -137,42 +137,42 @@ define push_image
 endef
 
 .PHONY: docker-build-all
-docker-build-all: docker-build-controller-manager docker-build-plugins docker-build-runtime docker-build-users ## Build all docker images
+docker-build-all: docker-build-controller-manager docker-build-gateway-plugins docker-build-runtime docker-build-metadata-service ## Build all docker images
 
 .PHONY: docker-build-controller-manager
 docker-build-controller-manager: ## Build docker image with the manager.
 	$(call build_and_tag,controller-manager,Dockerfile)
 
-.PHONY: docker-build-plugins
-docker-build-plugins: ## Build docker image with the plugins.
-	$(call build_and_tag,plugins,Dockerfile.gateway)
+.PHONY: docker-build-gateway-plugins
+docker-build-gateway-plugins: ## Build docker image with the gateway plugins.
+	$(call build_and_tag,gateway-plugins,Dockerfile.gateway)
 
 .PHONY: docker-build-runtime
 docker-build-runtime: ## Build docker image with the AI Runtime.
 	$(call build_and_tag,runtime,Dockerfile.runtime)
 
-.PHONY: docker-build-users
-docker-build-users: ## Build docker image with the users.
-	$(call build_and_tag,users,Dockerfile.users)
+.PHONY: docker-build-metadata-service
+docker-build-metadata-service: ## Build docker image with the metadata-service.
+	$(call build_and_tag,metadata-service,Dockerfile.metadata)
 
 .PHONY: docker-push-all
-docker-push-all: docker-push-controller-manager docker-push-plugins docker-push-runtime docker-push-users ## Push all docker images
+docker-push-all: docker-push-controller-manager docker-push-gateway-plugins docker-push-runtime docker-push-metadata-service ## Push all docker images
 
 .PHONY: docker-push-controller-manager
 docker-push-controller-manager: ## Push docker image with the manager.
 	$(call push_image,controller-manager)
 
-.PHONY: docker-push-plugins
-docker-push-plugins: ## Push docker image with the plugins.
-	$(call push_image,plugins)
+.PHONY: docker-push-gateway-plugins
+docker-push-gateway-plugins: ## Push docker image with the gateway plugins.
+	$(call push_image,gateway-plugins)
 
 .PHONY: docker-push-runtime
 docker-push-runtime: ## Push docker image with the AI Runtime.
 	$(call push_image,runtime)
 
-.PHONY: docker-push-users
-docker-push-users: ## Push docker image with the users.
-	$(call push_image,users)
+.PHONY: docker-push-metadata-service
+docker-push-metadata-service: ## Push docker image with the metadata-service.
+	$(call push_image,metadata-service)
 
 # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
 # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:

diff --git a/build/container/Dockerfile.gateway b/build/container/Dockerfile.gateway
@@ -21,13 +21,13 @@ COPY pkg/ pkg/
 # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
 # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o plugins cmd/plugins/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o gateway-plugins cmd/plugins/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
-COPY --from=builder /workspace/plugins .
+COPY --from=builder /workspace/gateway-plugins .
 USER 65532:65532
 
-ENTRYPOINT ["/plugins"]
+ENTRYPOINT ["/gateway-plugins"]
diff --git a/build/container/Dockerfile.users → build/container/Dockerfile.metadata b/build/container/Dockerfile.users → build/container/Dockerfile.metadata
@@ -14,23 +14,20 @@ RUN go mod download
 # Copy the go source
 COPY cmd/ cmd/
 COPY api/ api/
-COPY pkg/users/ pkg/users/
-COPY pkg/utils/ pkg/utils/
-COPY pkg/cache/ pkg/cache/
-COPY pkg/client/ pkg/client/
+COPY pkg/ pkg/
 
 # Build
 # the GOARCH has not a default value to allow the binary be built according to the host where the command
 # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
 # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o users cmd/users/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o metadata-service cmd/metadata/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
-COPY --from=builder /workspace/users .
+COPY --from=builder /workspace/metadata-service .
 USER 65532:65532
 
-ENTRYPOINT ["/users"]
+ENTRYPOINT ["/metadata-service"]
diff --git a/cmd/users/main.go → cmd/metadata/main.go b/cmd/users/main.go → cmd/metadata/main.go
@@ -17,7 +17,7 @@ limitations under the License.
 package main
 
 import (
-	"github.com/aibrix/aibrix/pkg/users"
+	"github.com/aibrix/aibrix/pkg/metadata"
 	"github.com/aibrix/aibrix/pkg/utils"
 	"k8s.io/klog/v2"
 )
@@ -26,6 +26,6 @@ func main() {
 	redisClient := utils.GetRedisClient()
 
 	klog.Info("Starting listening on port 8090")
-	srv := users.NewHTTPServer(":8090", redisClient)
+	srv := metadata.NewHTTPServer(":8090", redisClient)
 	klog.Fatal(srv.ListenAndServe())
 }
diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -23,6 +23,7 @@ resources:
 - ../rbac
 - ../manager
 - ../gateway
+- ../metadata
 - ../gpu-optimizer
 - ../dependency/kuberay-operator
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
@@ -53,3 +54,14 @@ patches:
       - kind: ServiceAccount
         name: aibrix-kuberay-operator-leader-election
         namespace: aibrix-system
+
+images:
+  - name: controller
+    newName: aibrix/controller-manager
+    newTag: nightly
+  - name: gateway-plugins
+    newName: aibrix/gateway-plugins
+    newTag: nightly
+  - name: metadata-service
+    newName: aibrix/metadata-service
+    newTag: nightly
diff --git a/config/gateway/gateway-plugin.yaml b/config/gateway/gateway-plugin.yaml
@@ -32,7 +32,7 @@ spec:
           command: ['sh', '-c', 'until echo "ping" | nc aibrix-redis-master 6379 -w 1  | grep -c PONG; do echo waiting for service aibrix-redis-master; sleep 2; done']
       containers:
         - name: gateway-plugin
-          image: plugins:latest
+          image: gateway-plugins:latest
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 50052
@@ -51,7 +51,7 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-      serviceAccountName: aibrix-gateway-plugin
+      serviceAccountName: aibrix-gateway-plugins
 ---
 # this is a dummy route for incoming request and,
 # then request is routed to httproute using model name OR

diff --git a/config/gateway/kustomization.yaml b/config/gateway/kustomization.yaml
@@ -1,15 +1,6 @@
 resources:
 - gateway.yaml
-- redis.yaml
 - gateway-plugin.yaml
-- users.yaml
 
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-images:
-- name: plugins
-  newName: aibrix/plugins
-  newTag: nightly
-- name: users
-  newName: aibrix/users
-  newTag: nightly
diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
@@ -2,7 +2,4 @@ resources:
 - manager.yaml
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-images:
-- name: controller
-  newName: aibrix/controller-manager
-  newTag: nightly
+
diff --git a/config/metadata/kustomization.yaml b/config/metadata/kustomization.yaml
@@ -0,0 +1,3 @@
+resources:
+- metadata.yaml
+- redis.yaml
diff --git a/config/gateway/users.yaml → config/metadata/metadata.yaml b/config/gateway/users.yaml → config/metadata/metadata.yaml
@@ -1,11 +1,11 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: gateway-users
+  name: metadata-service
   namespace: aibrix-system
 spec:
   selector:
-    app: gateway-users
+    app: metadata-service
   ports:
     - protocol: TCP
       port: 8090
@@ -14,25 +14,25 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: gateway-users
+  name: metadata-service
   namespace: aibrix-system
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: gateway-users
+      app: metadata-service
   template:
     metadata:
       labels:
-        app: gateway-users
+        app: metadata-service
     spec:
       initContainers:
         - name: init-c
           image: busybox:stable
           command: ['sh', '-c', 'until echo "ping" | nc aibrix-redis-master 6379 -w 1  | grep -c PONG; do echo waiting for service aibrix-redis-master; sleep 2; done']
       containers:
-        - name: gateway-users
-          image: users:latest
+        - name: metadata-service
+          image: metadata-service:latest
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8090
@@ -49,4 +49,5 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-      serviceAccountName: aibrix-gateway-plugin
+      # TODO: separate this part
+      serviceAccountName: aibrix-gateway-plugins
diff --git a/config/gateway/redis.yaml → config/metadata/redis.yaml b/config/gateway/redis.yaml → config/metadata/redis.yaml
diff --git a/config/rbac/gateway/role_binding_gateway.yaml b/config/rbac/gateway/role_binding_gateway.yaml
@@ -4,12 +4,12 @@ metadata:
   labels:
     app.kubernetes.io/name: aibrix
     app.kubernetes.io/managed-by: kustomize
-  name: gateway-plugin-rolebinding
+  name: gateway-plugins-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: gateway-plugin-role
+  name: gateway-plugins-role
 subjects:
 - kind: ServiceAccount
-  name: gateway-plugin
+  name: gateway-plugins
   namespace: system
diff --git a/config/rbac/gateway/role_gateway.yaml b/config/rbac/gateway/role_gateway.yaml
@@ -2,7 +2,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: gateway-plugin-role
+  name: gateway-plugins-role
 rules:
 - apiGroups:
   - ""

diff --git a/config/rbac/gateway/service_account_gateway.yaml b/config/rbac/gateway/service_account_gateway.yaml
@@ -4,5 +4,5 @@ metadata:
   labels:
     app.kubernetes.io/name: aibrix
     app.kubernetes.io/managed-by: kustomize
-  name: gateway-plugin
+  name: gateway-plugins
   namespace: system
diff --git a/development/app/README.md b/development/app/README.md
@@ -102,7 +102,7 @@ curl http://localhost:8000/v1/chat/completions \
 ```shell
 # note: not mandatory to create user to access gateway API
 
-kubectl -n aibrix-system port-forward svc/aibrix-gateway-users 8090:8090 &
+kubectl -n aibrix-system port-forward svc/aibrix-metadata-service 8090:8090 &
 
 curl http://localhost:8090/CreateUser \
   -H "Content-Type: application/json" \

diff --git a/docs/source/getting_started/quickstart.rst b/docs/source/getting_started/quickstart.rst
@@ -159,7 +159,6 @@ Save yaml as `service.yaml` and run `kubectl apply -f service.yaml`.
    2. The `--served-model-name` argument value in the `Deployment` command is also consistent with the `Service` name and `model.aibrix.ai/name` label.
 
 
-
 Invoke the model endpoint using gateway api
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

diff --git a/pkg/users/README.md → pkg/metadata/README.md b/pkg/users/README.md → pkg/metadata/README.md
@@ -1,7 +1,7 @@
 
 ```shell
-kubectl apply -f docs/development/app/users.yaml
-kubectl -n aibrix-system port-forward svc/aibrix-gateway-users 8090:8090 &
+kubectl apply -f docs/development/app/metadata.yaml
+kubectl -n aibrix-system port-forward svc/aibrix-metadata-service 8090:8090 &
 ```
 
 # Create user

diff --git a/pkg/users/helpers.go → pkg/metadata/helpers.go b/pkg/users/helpers.go → pkg/metadata/helpers.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package users
+package metadata
 
 import (
 	"encoding/json"

diff --git a/pkg/users/users.go → pkg/metadata/users.go b/pkg/users/users.go → pkg/metadata/users.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package users
+package metadata
 
 import (
 	"errors"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -159,7 +159,6 @@ Save yaml as `service.yaml` and run `kubectl apply -f service.yaml`.
		2. The `--served-model-name` argument value in the `Deployment` command is also consistent with the `Service` name and `model.aibrix.ai/name` label.



		Invoke the model endpoint using gateway api
		^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Expand Down