From 00091a7d4962b8adc654704cb644ea43298dc5bc Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Wed, 25 Jun 2025 22:02:49 -0400 Subject: [PATCH 1/8] Integrated aqua with model group. --- ads/aqua/model/constants.py | 1 + ads/aqua/model/entities.py | 15 +++ ads/aqua/model/model.py | 107 +++++++++++-------- ads/aqua/modeldeployment/deployment.py | 56 +++++----- ads/model/model_metadata.py | 2 +- tests/unitary/with_extras/aqua/test_model.py | 42 +++++--- 6 files changed, 132 insertions(+), 91 deletions(-) diff --git a/ads/aqua/model/constants.py b/ads/aqua/model/constants.py index 194245fe4..ce3e3f51d 100644 --- a/ads/aqua/model/constants.py +++ b/ads/aqua/model/constants.py @@ -20,6 +20,7 @@ class ModelCustomMetadataFields(ExtendedEnum): DEPLOYMENT_CONTAINER_URI = "deployment-container-uri" MULTIMODEL_GROUP_COUNT = "model_group_count" MULTIMODEL_METADATA = "multi_model_metadata" + MODEL_GROUP_CONFIG = "OCI_MODEL_GROUP_CUSTOM_METADATA" class ModelTask(ExtendedEnum): diff --git a/ads/aqua/model/entities.py b/ads/aqua/model/entities.py index 0bbcdfb0b..72c374f74 100644 --- a/ads/aqua/model/entities.py +++ b/ads/aqua/model/entities.py @@ -383,3 +383,18 @@ class ModelFileDescription(Serializable): class Config: alias_generator = to_camel extra = "allow" + + +class MemberModel(Serializable): + """Describes the member model of a model group. + + Attributes: + model_id (str): The id of member model. + inference_key (str): The inference key of member model. + """ + + model_id: str = Field(..., description="The id of member model.") + inference_key: str = Field(None, description="The inference key of member model.") + + class Config: + extra = "allow" diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index 2b5d7108f..925535139 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -79,7 +79,7 @@ AquaModelReadme, AquaModelSummary, ImportModelDetails, - ModelFileDescription, + MemberModel, ModelValidationResult, ) from ads.aqua.model.enums import MultiModelSupportedTaskType @@ -102,6 +102,7 @@ ) from ads.model import DataScienceModel from ads.model.common.utils import MetadataArtifactPathType +from ads.model.datascience_model_group import DataScienceModelGroup from ads.model.model_metadata import ( MetadataCustomCategory, ModelCustomMetadata, @@ -235,13 +236,15 @@ def create( def create_multi( self, models: List[AquaMultiModelRef], + create_deployment_details, + model_config_summary, project_id: Optional[str] = None, compartment_id: Optional[str] = None, freeform_tags: Optional[Dict] = None, defined_tags: Optional[Dict] = None, source_models: Optional[Dict[str, DataScienceModel]] = None, **kwargs, # noqa: ARG002 - ) -> DataScienceModel: + ) -> DataScienceModelGroup: """ Creates a multi-model grouping using the provided model list. @@ -249,6 +252,11 @@ def create_multi( ---------- models : List[AquaMultiModelRef] List of AquaMultiModelRef instances for creating a multi-model group. + create_deployment_details : CreateModelDeploymentDetails + An instance of CreateModelDeploymentDetails containing all required and optional + fields for creating a model deployment via Aqua. + model_config_summary : ModelConfigSummary + Summary Model Deployment configuration for the group of models. project_id : Optional[str] The project ID for the multi-model group. compartment_id : Optional[str] @@ -264,8 +272,8 @@ def create_multi( Returns ------- - DataScienceModel - Instance of DataScienceModel object. + DataScienceModelGroup + Instance of DataScienceModelGroup object. """ if not models: @@ -274,7 +282,6 @@ def create_multi( ) display_name_list = [] - model_file_description_list: List[ModelFileDescription] = [] model_custom_metadata = ModelCustomMetadata() service_inference_containers = ( @@ -337,11 +344,6 @@ def create_multi( "Please register the model with a file description." ) - # Track model file description in a validated structure - model_file_description_list.append( - ModelFileDescription(**model_file_description) - ) - # Ensure base model has a valid artifact if not source_model.artifact: logger.error( @@ -396,11 +398,6 @@ def create_multi( "Please register the model with a file description." ) - # Track model file description in a validated structure - model_file_description_list.append( - ModelFileDescription(**ft_model_file_description) - ) - # Extract fine-tuned model path _, fine_tune_path = extract_fine_tune_artifacts_path( fine_tune_source_model @@ -481,6 +478,22 @@ def create_multi( description="Number of models in the group.", category="Other", ) + model_custom_metadata.add( + key=ModelCustomMetadataFields.MODEL_GROUP_CONFIG, + value=self._build_model_group_config( + create_deployment_details=create_deployment_details, + model_config_summary=model_config_summary, + deployment_container=deployment_container, + ), + description="Configs required to deploy multi models.", + category="Other", + ) + model_custom_metadata.add( + key=ModelCustomMetadataFields.MULTIMODEL_METADATA, + value=json.dumps([model.model_dump() for model in models]), + description="Metadata to store user's multi model input.", + category="Other", + ) # Combine tags. The `Tags.AQUA_TAG` has been excluded, because we don't want to show # the models created for multi-model purpose in the AQUA models list. @@ -491,8 +504,8 @@ def create_multi( } # Create multi-model group - custom_model = ( - DataScienceModel() + custom_model_group = ( + DataScienceModelGroup() .with_compartment_id(compartment_id) .with_project_id(project_id) .with_display_name(model_group_display_name) @@ -500,37 +513,15 @@ def create_multi( .with_freeform_tags(**tags) .with_defined_tags(**(defined_tags or {})) .with_custom_metadata_list(model_custom_metadata) + .with_member_models( + [MemberModel(model_id=model.model_id).model_dump() for model in models] + ) ) - # Update multi model file description to attach artifacts - custom_model.with_model_file_description( - json_dict=ModelFileDescription( - models=[ - models - for model_file_description in model_file_description_list - for models in model_file_description.models - ] - ).model_dump(by_alias=True) - ) - - # Finalize creation - custom_model.create(model_by_reference=True) + custom_model_group.create() logger.info( - f"Aqua Model '{custom_model.id}' created with models: {', '.join(display_name_list)}." - ) - - # Create custom metadata for multi model metadata - custom_model.create_custom_metadata_artifact( - metadata_key_name=ModelCustomMetadataFields.MULTIMODEL_METADATA, - artifact_path_or_content=json.dumps( - [model.model_dump() for model in models] - ).encode(), - path_type=MetadataArtifactPathType.CONTENT, - ) - - logger.debug( - f"Multi model metadata uploaded for Aqua model: {custom_model.id}." + f"Aqua Model Group'{custom_model_group.id}' created with models: {', '.join(display_name_list)}." ) # Track telemetry event @@ -540,7 +531,33 @@ def create_multi( detail=combined_models, ) - return custom_model + return custom_model_group + + def _build_model_group_config( + self, + create_deployment_details, + model_config_summary, + deployment_container: str, + ) -> str: + """Builds model group config required to deploy multi models.""" + container_type_key = ( + create_deployment_details.container_family or deployment_container + ) + container_config = self.get_container_config_item(container_type_key) + container_spec = container_config.spec if container_config else UNKNOWN + + container_params = container_spec.cli_param if container_spec else UNKNOWN + + from ads.aqua.modeldeployment.model_group_config import ModelGroupConfig + + multi_model_config = ModelGroupConfig.from_create_model_deployment_details( + create_deployment_details, + model_config_summary, + container_type_key, + container_params, + ) + + return multi_model_config.model_dump_json() @telemetry(entry_point="plugin=model&action=get", name="aqua") def get(self, model_id: str) -> "AquaModel": diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index 82402af4b..ee4b721a6 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -7,7 +7,7 @@ import shlex import threading from datetime import datetime, timedelta -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union from cachetools import TTLCache, cached from oci.data_science.models import ModelDeploymentShapeSummary @@ -40,7 +40,6 @@ AQUA_MODEL_TYPE_CUSTOM, AQUA_MODEL_TYPE_MULTI, AQUA_MODEL_TYPE_SERVICE, - AQUA_MULTI_MODEL_CONFIG, MODEL_BY_REFERENCE_OSS_PATH_KEY, MODEL_NAME_DELIMITER, UNKNOWN_DICT, @@ -65,7 +64,6 @@ ConfigValidationError, CreateModelDeploymentDetails, ) -from ads.aqua.modeldeployment.model_group_config import ModelGroupConfig from ads.common.object_storage_details import ObjectStorageDetails from ads.common.utils import UNKNOWN, get_log_links from ads.common.work_request import DataScienceWorkRequest @@ -79,6 +77,7 @@ PROJECT_OCID, ) from ads.model.datascience_model import DataScienceModel +from ads.model.datascience_model_group import DataScienceModelGroup from ads.model.deployment import ( ModelDeployment, ModelDeploymentContainerRuntime, @@ -325,8 +324,10 @@ def create( f"Multi models ({source_model_ids}) provided. Delegating to multi model creation method." ) - aqua_model = model_app.create_multi( + aqua_model_group = model_app.create_multi( models=create_deployment_details.models, + create_deployment_details=create_deployment_details, + model_config_summary=model_config_summary, compartment_id=compartment_id, project_id=project_id, freeform_tags=freeform_tags, @@ -334,8 +335,7 @@ def create( source_models=source_models, ) return self._create_multi( - aqua_model=aqua_model, - model_config_summary=model_config_summary, + aqua_model_group=aqua_model_group, create_deployment_details=create_deployment_details, container_config=container_config, ) @@ -562,8 +562,7 @@ def _create( def _create_multi( self, - aqua_model: DataScienceModel, - model_config_summary: ModelDeploymentConfigSummary, + aqua_model_group: DataScienceModelGroup, create_deployment_details: CreateModelDeploymentDetails, container_config: AquaContainerConfig, ) -> AquaDeployment: @@ -571,15 +570,14 @@ def _create_multi( Parameters ---------- - model_config_summary : model_config_summary - Summary Model Deployment configuration for the group of models. - aqua_model : DataScienceModel - An instance of Aqua data science model. + aqua_model_group : DataScienceModelGroup + An instance of Aqua data science model group. create_deployment_details : CreateModelDeploymentDetails An instance of CreateModelDeploymentDetails containing all required and optional fields for creating a model deployment via Aqua. container_config: Dict Container config dictionary. + Returns ------- AquaDeployment @@ -589,23 +587,12 @@ def _create_multi( env_var = {**(create_deployment_details.env_var or UNKNOWN_DICT)} container_type_key = self._get_container_type_key( - model=aqua_model, + model=aqua_model_group, container_family=create_deployment_details.container_family, ) container_config = self.get_container_config_item(container_type_key) container_spec = container_config.spec if container_config else UNKNOWN - container_params = container_spec.cli_param if container_spec else UNKNOWN - - multi_model_config = ModelGroupConfig.from_create_model_deployment_details( - create_deployment_details, - model_config_summary, - container_type_key, - container_params, - ) - - env_var.update({AQUA_MULTI_MODEL_CONFIG: multi_model_config.model_dump_json()}) - env_vars = container_spec.env_vars if container_spec else [] for env in env_vars: if isinstance(env, dict): @@ -614,7 +601,7 @@ def _create_multi( if key not in env_var: env_var.update(env) - logger.info(f"Env vars used for deploying {aqua_model.id} : {env_var}.") + logger.info(f"Env vars used for deploying {aqua_model_group.id} : {env_var}.") container_image_uri = ( create_deployment_details.container_image_uri @@ -627,7 +614,7 @@ def _create_multi( container_spec.health_check_port if container_spec else None ) tags = { - Tags.AQUA_MODEL_ID_TAG: aqua_model.id, + Tags.AQUA_MODEL_ID_TAG: aqua_model_group.id, Tags.MULTIMODEL_TYPE_TAG: "true", Tags.AQUA_TAG: "active", **(create_deployment_details.freeform_tags or UNKNOWN_DICT), @@ -637,7 +624,7 @@ def _create_multi( aqua_deployment = self._create_deployment( create_deployment_details=create_deployment_details, - aqua_model_id=aqua_model.id, + aqua_model_id=aqua_model_group.id, model_name=model_name, model_type=AQUA_MODEL_TYPE_MULTI, container_image_uri=container_image_uri, @@ -794,7 +781,9 @@ def _create_deployment( ) @staticmethod - def _get_container_type_key(model: DataScienceModel, container_family: str) -> str: + def _get_container_type_key( + model: Union[DataScienceModel, DataScienceModelGroup], container_family: str + ) -> str: container_type_key = UNKNOWN if container_family: container_type_key = container_family @@ -970,7 +959,12 @@ def get(self, model_deployment_id: str, **kwargs) -> "AquaDeploymentDetail": f"Invalid multi model deployment {model_deployment_id}." f"Make sure the {Tags.AQUA_MODEL_ID_TAG} tag is added to the deployment." ) - aqua_model = DataScienceModel.from_id(aqua_model_id) + + if "datasciencemodelgroup" in aqua_model_id: + aqua_model = DataScienceModelGroup.from_id(aqua_model_id) + else: + aqua_model = DataScienceModel.from_id(aqua_model_id) + custom_metadata_list = aqua_model.custom_metadata_list multi_model_metadata_value = custom_metadata_list.get( ModelCustomMetadataFields.MULTIMODEL_METADATA, @@ -984,7 +978,9 @@ def get(self, model_deployment_id: str, **kwargs) -> "AquaDeploymentDetail": f"Ensure that the required custom metadata `{ModelCustomMetadataFields.MULTIMODEL_METADATA}` is added to the AQUA multi-model `{aqua_model.display_name}` ({aqua_model.id})." ) multi_model_metadata = json.loads( - aqua_model.dsc_model.get_custom_metadata_artifact( + multi_model_metadata_value + if isinstance(aqua_model, DataScienceModelGroup) + else aqua_model.dsc_model.get_custom_metadata_artifact( metadata_key_name=ModelCustomMetadataFields.MULTIMODEL_METADATA ).decode("utf-8") ) diff --git a/ads/model/model_metadata.py b/ads/model/model_metadata.py index 6b73b17f5..f0428ec9c 100644 --- a/ads/model/model_metadata.py +++ b/ads/model/model_metadata.py @@ -37,7 +37,7 @@ logger = logging.getLogger("ADS") METADATA_SIZE_LIMIT = 32000 -METADATA_VALUE_LENGTH_LIMIT = 255 +METADATA_VALUE_LENGTH_LIMIT = 16000 METADATA_DESCRIPTION_LENGTH_LIMIT = 255 _METADATA_EMPTY_VALUE = "NA" CURRENT_WORKING_DIR = "." diff --git a/tests/unitary/with_extras/aqua/test_model.py b/tests/unitary/with_extras/aqua/test_model.py index 61d9b849d..5f96cb9a4 100644 --- a/tests/unitary/with_extras/aqua/test_model.py +++ b/tests/unitary/with_extras/aqua/test_model.py @@ -42,6 +42,7 @@ from ads.aqua.model.enums import MultiModelSupportedTaskType from ads.common.object_storage_details import ObjectStorageDetails from ads.model.datascience_model import DataScienceModel +from ads.model.datascience_model_group import DataScienceModelGroup from ads.model.model_metadata import ( ModelCustomMetadata, ModelProvenanceMetadata, @@ -457,14 +458,12 @@ def test_create_model(self, mock_from_id, mock_validate, mock_create): ) assert model.provenance_metadata.training_id == "test_training_id" - @patch.object(DataScienceModel, "create_custom_metadata_artifact") - @patch.object(DataScienceModel, "create") + @patch.object(DataScienceModelGroup, "create") @patch.object(AquaApp, "get_container_config") def test_create_multimodel( self, mock_get_container_config, - mock_create, - mock_create_custom_metadata_artifact, + mock_create_group, ): mock_get_container_config.return_value = get_container_config() mock_model = MagicMock() @@ -482,6 +481,8 @@ def test_create_multimodel( ) mock_model.custom_metadata_list = custom_metadata_list + mock_create_deployment_details = MagicMock() + mock_model_config_summary = MagicMock() model_info_1 = AquaMultiModelRef( model_id="test_model_id_1", @@ -503,8 +504,10 @@ def test_create_multimodel( } with pytest.raises(AquaValueError): - model = self.app.create_multi( + model_group = self.app.create_multi( models=[model_info_1, model_info_2], + create_deployment_details=mock_create_deployment_details, + model_config_summary=mock_model_config_summary, project_id="test_project_id", compartment_id="test_compartment_id", source_models=model_details, @@ -513,8 +516,10 @@ def test_create_multimodel( mock_model.freeform_tags["aqua_service_model"] = TestDataset.SERVICE_MODEL_ID with pytest.raises(AquaValueError): - model = self.app.create_multi( + model_group = self.app.create_multi( models=[model_info_1, model_info_2], + create_deployment_details=mock_create_deployment_details, + model_config_summary=mock_model_config_summary, project_id="test_project_id", compartment_id="test_compartment_id", source_models=model_details, @@ -523,8 +528,10 @@ def test_create_multimodel( mock_model.freeform_tags["task"] = "text-generation" with pytest.raises(AquaValueError): - model = self.app.create_multi( + model_group = self.app.create_multi( models=[model_info_1, model_info_2], + create_deployment_details=mock_create_deployment_details, + model_config_summary=mock_model_config_summary, project_id="test_project_id", compartment_id="test_compartment_id", source_models=model_details, @@ -541,8 +548,10 @@ def test_create_multimodel( model_info_1.model_task = "invalid_task" with pytest.raises(AquaValueError): - model = self.app.create_multi( + model_group = self.app.create_multi( models=[model_info_1, model_info_2], + create_deployment_details=mock_create_deployment_details, + model_config_summary=mock_model_config_summary, project_id="test_project_id", compartment_id="test_compartment_id", source_models=model_details, @@ -552,8 +561,10 @@ def test_create_multimodel( model_info_1.model_task = None mock_model.freeform_tags["task"] = "unsupported_task" with pytest.raises(AquaValueError): - model = self.app.create_multi( + model_group = self.app.create_multi( models=[model_info_1, model_info_2], + create_deployment_details=mock_create_deployment_details, + model_config_summary=mock_model_config_summary, project_id="test_project_id", compartment_id="test_compartment_id", source_models=model_details, @@ -580,22 +591,23 @@ def test_create_multimodel( model_details[model_info_3.model_id] = mock_model # will create a multi-model group - model = self.app.create_multi( + model_group = self.app.create_multi( models=[model_info_1, model_info_2, model_info_3], + create_deployment_details=mock_create_deployment_details, + model_config_summary=mock_model_config_summary, project_id="test_project_id", compartment_id="test_compartment_id", source_models=model_details, ) - mock_create.assert_called_with(model_by_reference=True) + mock_create_group.assert_called() mock_model.compartment_id = TestDataset.SERVICE_COMPARTMENT_ID - mock_create.return_value = mock_model - assert model.freeform_tags == {"aqua_multimodel": "true"} - assert model.custom_metadata_list.get("model_group_count").value == "3" + assert model_group.freeform_tags == {"aqua_multimodel": "true"} + assert model_group.custom_metadata_list.get("model_group_count").value == "3" assert ( - model.custom_metadata_list.get("deployment-container").value + model_group.custom_metadata_list.get("deployment-container").value == "odsc-vllm-serving" ) From 46d23f0cc02bedcf91674205a58e55e9b96d6437 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Fri, 27 Jun 2025 18:08:07 -0400 Subject: [PATCH 2/8] Updated pr. --- ads/aqua/modeldeployment/deployment.py | 5 +- ads/aqua/modeldeployment/entities.py | 21 +- ads/model/deployment/model_deployment.py | 124 ++++--- .../deployment/model_deployment_runtime.py | 40 ++- .../test_model_deployment_v2.py | 339 +----------------- 5 files changed, 152 insertions(+), 377 deletions(-) diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index ee4b721a6..3e3f06788 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -719,11 +719,14 @@ def _create_deployment( .with_health_check_port(health_check_port) .with_env(env_var) .with_deployment_mode(ModelDeploymentMode.HTTPS) - .with_model_uri(aqua_model_id) .with_region(self.region) .with_overwrite_existing_artifact(True) .with_remove_existing_artifact(True) ) + if "datasciencemodelgroup" in aqua_model_id: + container_runtime.with_model_group_id(aqua_model_id) + else: + container_runtime.with_model_uri(aqua_model_id) if cmd_var: container_runtime.with_cmd(cmd_var) diff --git a/ads/aqua/modeldeployment/entities.py b/ads/aqua/modeldeployment/entities.py index ebce26dc8..fde05e666 100644 --- a/ads/aqua/modeldeployment/entities.py +++ b/ads/aqua/modeldeployment/entities.py @@ -147,13 +147,25 @@ def from_oci_model_deployment( AquaDeployment: The instance of the Aqua model deployment. """ - instance_configuration = oci_model_deployment.model_deployment_configuration_details.model_configuration_details.instance_configuration + model_deployment_configuration_details = ( + oci_model_deployment.model_deployment_configuration_details + ) + if model_deployment_configuration_details.deployment_type == "SINGLE_MODEL": + instance_configuration = model_deployment_configuration_details.model_configuration_details.instance_configuration + instance_count = model_deployment_configuration_details.model_configuration_details.scaling_policy.instance_count + model_id = model_deployment_configuration_details.model_configuration_details.model_id + else: + instance_configuration = model_deployment_configuration_details.infrastructure_configuration_details.instance_configuration + instance_count = model_deployment_configuration_details.infrastructure_configuration_details.scaling_policy.instance_count + model_id = model_deployment_configuration_details.model_group_configuration_details.model_group_id + instance_shape_config_details = ( instance_configuration.model_deployment_instance_shape_config_details ) - instance_count = oci_model_deployment.model_deployment_configuration_details.model_configuration_details.scaling_policy.instance_count - environment_variables = oci_model_deployment.model_deployment_configuration_details.environment_configuration_details.environment_variables - cmd = oci_model_deployment.model_deployment_configuration_details.environment_configuration_details.cmd + environment_variables = model_deployment_configuration_details.environment_configuration_details.environment_variables + cmd = ( + model_deployment_configuration_details.environment_configuration_details.cmd + ) shape_info = ShapeInfo( instance_shape=instance_configuration.instance_shape_name, instance_count=instance_count, @@ -168,7 +180,6 @@ def from_oci_model_deployment( else None ), ) - model_id = oci_model_deployment._model_deployment_configuration_details.model_configuration_details.model_id tags = {} tags.update(oci_model_deployment.freeform_tags or UNKNOWN_DICT) tags.update(oci_model_deployment.defined_tags or UNKNOWN_DICT) diff --git a/ads/model/deployment/model_deployment.py b/ads/model/deployment/model_deployment.py index 56a70c112..21e083499 100644 --- a/ads/model/deployment/model_deployment.py +++ b/ads/model/deployment/model_deployment.py @@ -1,22 +1,27 @@ #!/usr/bin/env python -# -*- coding: utf-8; -*- -# Copyright (c) 2021, 2023 Oracle and/or its affiliates. +# Copyright (c) 2021, 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ import collections import copy import datetime -import oci -import warnings import time -from typing import Dict, List, Union, Any +import warnings +from typing import Any, Dict, List, Union +import oci import oci.loggingsearch -from ads.common import auth as authutil import pandas as pd -from ads.model.serde.model_input import JsonModelInputSERDE +from oci.data_science.models import ( + CreateModelDeploymentDetails, + LogDetails, + UpdateModelDeploymentDetails, +) + +from ads.common import auth as authutil +from ads.common import utils as ads_utils from ads.common.oci_logging import ( LOG_INTERVAL, LOG_RECORDS_LIMIT, @@ -30,10 +35,10 @@ from ads.model.deployment.common.utils import send_request from ads.model.deployment.model_deployment_infrastructure import ( DEFAULT_BANDWIDTH_MBPS, + DEFAULT_MEMORY_IN_GBS, + DEFAULT_OCPUS, DEFAULT_REPLICA, DEFAULT_SHAPE_NAME, - DEFAULT_OCPUS, - DEFAULT_MEMORY_IN_GBS, MODEL_DEPLOYMENT_INFRASTRUCTURE_TYPE, ModelDeploymentInfrastructure, ) @@ -45,18 +50,14 @@ ModelDeploymentRuntimeType, OCIModelDeploymentRuntimeType, ) +from ads.model.serde.model_input import JsonModelInputSERDE from ads.model.service.oci_datascience_model_deployment import ( OCIDataScienceModelDeployment, ) -from ads.common import utils as ads_utils + from .common import utils from .common.utils import State from .model_deployment_properties import ModelDeploymentProperties -from oci.data_science.models import ( - LogDetails, - CreateModelDeploymentDetails, - UpdateModelDeploymentDetails, -) DEFAULT_WAIT_TIME = 1200 DEFAULT_POLL_INTERVAL = 10 @@ -964,7 +965,9 @@ def predict( except oci.exceptions.ServiceError as ex: # When bandwidth exceeds the allocated value, TooManyRequests error (429) will be raised by oci backend. if ex.status == 429: - bandwidth_mbps = self.infrastructure.bandwidth_mbps or DEFAULT_BANDWIDTH_MBPS + bandwidth_mbps = ( + self.infrastructure.bandwidth_mbps or DEFAULT_BANDWIDTH_MBPS + ) utils.get_logger().warning( f"Load balancer bandwidth exceeds the allocated {bandwidth_mbps} Mbps." "To estimate the actual bandwidth, use formula: (payload size in KB) * (estimated requests per second) * 8 / 1024." @@ -1518,9 +1521,9 @@ def _build_model_deployment_details(self) -> CreateModelDeploymentDetails: self.infrastructure.CONST_CATEGORY_LOG_DETAILS: self._build_category_log_details(), } - return OCIDataScienceModelDeployment( - **create_model_deployment_details - ).to_oci_model(CreateModelDeploymentDetails) + return CreateModelDeploymentDetails( + **ads_utils.batch_convert_case(create_model_deployment_details, "snake") + ) def _update_model_deployment_details( self, **kwargs @@ -1545,9 +1548,10 @@ def _update_model_deployment_details( self.infrastructure.CONST_MODEL_DEPLOYMENT_CONFIG_DETAILS: self._build_model_deployment_configuration_details(), self.infrastructure.CONST_CATEGORY_LOG_DETAILS: self._build_category_log_details(), } - return OCIDataScienceModelDeployment( - **update_model_deployment_details - ).to_oci_model(UpdateModelDeploymentDetails) + + return UpdateModelDeploymentDetails( + **ads_utils.batch_convert_case(update_model_deployment_details, "snake") + ) def _update_spec(self, **kwargs) -> "ModelDeployment": """Updates model deployment specs from kwargs. @@ -1644,22 +1648,22 @@ def _build_model_deployment_configuration_details(self) -> Dict: } if infrastructure.subnet_id: - instance_configuration[ - infrastructure.CONST_SUBNET_ID - ] = infrastructure.subnet_id + instance_configuration[infrastructure.CONST_SUBNET_ID] = ( + infrastructure.subnet_id + ) if infrastructure.private_endpoint_id: if not hasattr( oci.data_science.models.InstanceConfiguration, "private_endpoint_id" ): # TODO: add oci version with private endpoint support. - raise EnvironmentError( + raise OSError( "Private endpoint is not supported in the current OCI SDK installed." ) - instance_configuration[ - infrastructure.CONST_PRIVATE_ENDPOINT_ID - ] = infrastructure.private_endpoint_id + instance_configuration[infrastructure.CONST_PRIVATE_ENDPOINT_ID] = ( + infrastructure.private_endpoint_id + ) scaling_policy = { infrastructure.CONST_POLICY_TYPE: "FIXED_SIZE", @@ -1667,13 +1671,13 @@ def _build_model_deployment_configuration_details(self) -> Dict: or DEFAULT_REPLICA, } - if not runtime.model_uri: + if not (runtime.model_uri or runtime.model_group_id): raise ValueError( - "Missing parameter model uri. Try reruning it after model uri is configured." + "Missing parameter model uri and model group id. Try reruning it after model or model group is configured." ) model_id = runtime.model_uri - if not model_id.startswith("ocid"): + if model_id and not model_id.startswith("ocid"): from ads.model.datascience_model import DataScienceModel dsc_model = DataScienceModel( @@ -1704,7 +1708,7 @@ def _build_model_deployment_configuration_details(self) -> Dict: oci.data_science.models, "ModelDeploymentEnvironmentConfigurationDetails", ): - raise EnvironmentError( + raise OSError( "Environment variable hasn't been supported in the current OCI SDK installed." ) @@ -1720,9 +1724,9 @@ def _build_model_deployment_configuration_details(self) -> Dict: and runtime.inference_server.upper() == MODEL_DEPLOYMENT_INFERENCE_SERVER_TRITON ): - environment_variables[ - "CONTAINER_TYPE" - ] = MODEL_DEPLOYMENT_INFERENCE_SERVER_TRITON + environment_variables["CONTAINER_TYPE"] = ( + MODEL_DEPLOYMENT_INFERENCE_SERVER_TRITON + ) runtime.set_spec(runtime.CONST_ENV, environment_variables) environment_configuration_details = { runtime.CONST_ENVIRONMENT_CONFIG_TYPE: runtime.environment_config_type, @@ -1734,7 +1738,7 @@ def _build_model_deployment_configuration_details(self) -> Dict: oci.data_science.models, "OcirModelDeploymentEnvironmentConfigurationDetails", ): - raise EnvironmentError( + raise OSError( "Container runtime hasn't been supported in the current OCI SDK installed." ) environment_configuration_details["image"] = runtime.image @@ -1742,9 +1746,9 @@ def _build_model_deployment_configuration_details(self) -> Dict: environment_configuration_details["cmd"] = runtime.cmd environment_configuration_details["entrypoint"] = runtime.entrypoint environment_configuration_details["serverPort"] = runtime.server_port - environment_configuration_details[ - "healthCheckPort" - ] = runtime.health_check_port + environment_configuration_details["healthCheckPort"] = ( + runtime.health_check_port + ) model_deployment_configuration_details = { infrastructure.CONST_DEPLOYMENT_TYPE: "SINGLE_MODEL", @@ -1752,9 +1756,27 @@ def _build_model_deployment_configuration_details(self) -> Dict: runtime.CONST_ENVIRONMENT_CONFIG_DETAILS: environment_configuration_details, } + if runtime.model_group_id: + model_deployment_configuration_details[ + infrastructure.CONST_DEPLOYMENT_TYPE + ] = "MODEL_GROUP" + model_deployment_configuration_details["modelGroupConfigurationDetails"] = { + runtime.CONST_MODEL_GROUP_ID: runtime.model_group_id + } + model_deployment_configuration_details[ + "infrastructureConfigurationDetails" + ] = { + "infrastructureType": "INSTANCE_POOL", + infrastructure.CONST_BANDWIDTH_MBPS: infrastructure.bandwidth_mbps + or DEFAULT_BANDWIDTH_MBPS, + infrastructure.CONST_INSTANCE_CONFIG: instance_configuration, + infrastructure.CONST_SCALING_POLICY: scaling_policy, + } + model_configuration_details.pop(runtime.CONST_MODEL_ID) + if runtime.deployment_mode == ModelDeploymentMode.STREAM: if not hasattr(oci.data_science.models, "StreamConfigurationDetails"): - raise EnvironmentError( + raise OSError( "Model deployment mode hasn't been supported in the current OCI SDK installed." ) model_deployment_configuration_details[ @@ -1786,9 +1808,13 @@ def _build_category_log_details(self) -> Dict: logs = {} if ( - self.infrastructure.access_log and - self.infrastructure.access_log.get(self.infrastructure.CONST_LOG_GROUP_ID, None) - and self.infrastructure.access_log.get(self.infrastructure.CONST_LOG_ID, None) + self.infrastructure.access_log + and self.infrastructure.access_log.get( + self.infrastructure.CONST_LOG_GROUP_ID, None + ) + and self.infrastructure.access_log.get( + self.infrastructure.CONST_LOG_ID, None + ) ): logs[self.infrastructure.CONST_ACCESS] = { self.infrastructure.CONST_LOG_GROUP_ID: self.infrastructure.access_log.get( @@ -1799,9 +1825,13 @@ def _build_category_log_details(self) -> Dict: ), } if ( - self.infrastructure.predict_log and - self.infrastructure.predict_log.get(self.infrastructure.CONST_LOG_GROUP_ID, None) - and self.infrastructure.predict_log.get(self.infrastructure.CONST_LOG_ID, None) + self.infrastructure.predict_log + and self.infrastructure.predict_log.get( + self.infrastructure.CONST_LOG_GROUP_ID, None + ) + and self.infrastructure.predict_log.get( + self.infrastructure.CONST_LOG_ID, None + ) ): logs[self.infrastructure.CONST_PREDICT] = { self.infrastructure.CONST_LOG_GROUP_ID: self.infrastructure.predict_log.get( diff --git a/ads/model/deployment/model_deployment_runtime.py b/ads/model/deployment/model_deployment_runtime.py index 26e31f9cd..adfa48d1d 100644 --- a/ads/model/deployment/model_deployment_runtime.py +++ b/ads/model/deployment/model_deployment_runtime.py @@ -1,11 +1,11 @@ #!/usr/bin/env python -# -*- coding: utf-8; -*- -# Copyright (c) 2023 Oracle and/or its affiliates. +# Copyright (c) 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/from typing import Dict from typing import Dict, List + from ads.jobs.builders.base import Builder MODEL_DEPLOYMENT_RUNTIME_KIND = "runtime" @@ -41,6 +41,8 @@ class ModelDeploymentRuntime(Builder): The output stream ids of model deployment. model_uri: str The model uri of model deployment. + model_group_id: str + The model group id of model deployment. bucket_uri: str The OCI Object Storage URI where large size model artifacts will be copied to. auth: Dict @@ -66,6 +68,8 @@ class ModelDeploymentRuntime(Builder): Sets the output stream ids of model deployment with_model_uri(model_uri) Sets the model uri of model deployment + with_model_group_id(model_group_id) + Sets the model group id of model deployment with_bucket_uri(bucket_uri) Sets the bucket uri when uploading large size model. with_auth(auth) @@ -82,6 +86,7 @@ class ModelDeploymentRuntime(Builder): CONST_MODEL_ID = "modelId" CONST_MODEL_URI = "modelUri" + CONST_MODEL_GROUP_ID = "modelGroupId" CONST_ENV = "env" CONST_ENVIRONMENT_VARIABLES = "environmentVariables" CONST_ENVIRONMENT_CONFIG_TYPE = "environmentConfigurationType" @@ -103,6 +108,7 @@ class ModelDeploymentRuntime(Builder): CONST_OUTPUT_STREAM_IDS: "output_stream_ids", CONST_DEPLOYMENT_MODE: "deployment_mode", CONST_MODEL_URI: "model_uri", + CONST_MODEL_GROUP_ID: "model_group_id", CONST_BUCKET_URI: "bucket_uri", CONST_AUTH: "auth", CONST_REGION: "region", @@ -120,6 +126,9 @@ class ModelDeploymentRuntime(Builder): MODEL_CONFIG_DETAILS_PATH = ( "model_deployment_configuration_details.model_configuration_details" ) + MODEL_GROUP_CONFIG_DETAILS_PATH = ( + "model_deployment_configuration_details.model_group_configuration_details" + ) payload_attribute_map = { CONST_ENV: f"{ENVIRONMENT_CONFIG_DETAILS_PATH}.environment_variables", @@ -127,6 +136,7 @@ class ModelDeploymentRuntime(Builder): CONST_OUTPUT_STREAM_IDS: f"{STREAM_CONFIG_DETAILS_PATH}.output_stream_ids", CONST_DEPLOYMENT_MODE: "deployment_mode", CONST_MODEL_URI: f"{MODEL_CONFIG_DETAILS_PATH}.model_id", + CONST_MODEL_GROUP_ID: f"{MODEL_GROUP_CONFIG_DETAILS_PATH}.model_group_id", } def __init__(self, spec: Dict = None, **kwargs) -> None: @@ -278,6 +288,32 @@ def with_model_uri(self, model_uri: str) -> "ModelDeploymentRuntime": """ return self.set_spec(self.CONST_MODEL_URI, model_uri) + @property + def model_group_id(self) -> str: + """The model group id of model deployment. + + Returns + ------- + str + The model group id of model deployment. + """ + return self.get_spec(self.CONST_MODEL_GROUP_ID, None) + + def with_model_group_id(self, model_group_id: str) -> "ModelDeploymentRuntime": + """Sets the model group id of model deployment. + + Parameters + ---------- + model_group_id: str + The model group id of model deployment. + + Returns + ------- + ModelDeploymentRuntime + The ModelDeploymentRuntime instance (self). + """ + return self.set_spec(self.CONST_MODEL_GROUP_ID, model_group_id) + @property def bucket_uri(self) -> str: """The bucket uri of model. diff --git a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py index 589c58d70..f86f1816a 100644 --- a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py +++ b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*-- -# Copyright (c) 2023 Oracle and/or its affiliates. +# Copyright (c) 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ import copy @@ -589,151 +589,6 @@ def test_build_category_log_details(self): }, } - @patch.object(DataScienceModel, "create") - def test_build_model_deployment_details(self, mock_create): - dsc_model = MagicMock() - dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" - mock_create.return_value = dsc_model - model_deployment = self.initialize_model_deployment() - create_model_deployment_details = ( - model_deployment._build_model_deployment_details() - ) - - mock_create.assert_called() - - assert isinstance( - create_model_deployment_details, - CreateModelDeploymentDetails, - ) - assert ( - create_model_deployment_details.display_name - == model_deployment.display_name - ) - assert ( - create_model_deployment_details.description == model_deployment.description - ) - assert ( - create_model_deployment_details.freeform_tags - == model_deployment.freeform_tags - ) - assert ( - create_model_deployment_details.defined_tags - == model_deployment.defined_tags - ) - - category_log_details = create_model_deployment_details.category_log_details - assert isinstance(category_log_details, CategoryLogDetails) - assert ( - category_log_details.access.log_id - == model_deployment.infrastructure.access_log["logId"] - ) - assert ( - category_log_details.access.log_group_id - == model_deployment.infrastructure.access_log["logGroupId"] - ) - assert ( - category_log_details.predict.log_id - == model_deployment.infrastructure.predict_log["logId"] - ) - assert ( - category_log_details.predict.log_group_id - == model_deployment.infrastructure.predict_log["logGroupId"] - ) - - model_deployment_configuration_details = ( - create_model_deployment_details.model_deployment_configuration_details - ) - assert isinstance( - model_deployment_configuration_details, - SingleModelDeploymentConfigurationDetails, - ) - assert model_deployment_configuration_details.deployment_type == "SINGLE_MODEL" - - environment_configuration_details = ( - model_deployment_configuration_details.environment_configuration_details - ) - assert isinstance( - environment_configuration_details, - OcirModelDeploymentEnvironmentConfigurationDetails, - ) - assert ( - environment_configuration_details.environment_configuration_type - == "OCIR_CONTAINER" - ) - assert ( - environment_configuration_details.environment_variables - == model_deployment.runtime.env - ) - assert environment_configuration_details.cmd == model_deployment.runtime.cmd - assert environment_configuration_details.image == model_deployment.runtime.image - assert ( - environment_configuration_details.image_digest - == model_deployment.runtime.image_digest - ) - assert ( - environment_configuration_details.entrypoint - == model_deployment.runtime.entrypoint - ) - assert ( - environment_configuration_details.server_port - == model_deployment.runtime.server_port - ) - assert ( - environment_configuration_details.health_check_port - == model_deployment.runtime.health_check_port - ) - - model_configuration_details = ( - model_deployment_configuration_details.model_configuration_details - ) - assert isinstance( - model_configuration_details, - ModelConfigurationDetails, - ) - assert ( - model_configuration_details.bandwidth_mbps - == model_deployment.infrastructure.bandwidth_mbps - ) - assert ( - model_configuration_details.model_id == model_deployment.runtime.model_uri - ) - - instance_configuration = model_configuration_details.instance_configuration - assert isinstance(instance_configuration, InstanceConfiguration) - assert ( - instance_configuration.instance_shape_name - == model_deployment.infrastructure.shape_name - ) - assert ( - instance_configuration.model_deployment_instance_shape_config_details.ocpus - == model_deployment.infrastructure.shape_config_details["ocpus"] - ) - assert ( - instance_configuration.model_deployment_instance_shape_config_details.memory_in_gbs - == model_deployment.infrastructure.shape_config_details["memoryInGBs"] - ) - - scaling_policy = model_configuration_details.scaling_policy - assert isinstance(scaling_policy, FixedSizeScalingPolicy) - assert scaling_policy.policy_type == "FIXED_SIZE" - assert scaling_policy.instance_count == model_deployment.infrastructure.replica - - # stream_configuration_details = ( - # model_deployment_configuration_details.stream_configuration_details - # ) - # assert isinstance( - # stream_configuration_details, - # StreamConfigurationDetails, - # ) - # assert ( - # stream_configuration_details.input_stream_ids - # == model_deployment.runtime.input_stream_ids - # ) - # assert ( - # stream_configuration_details.output_stream_ids - # == model_deployment.runtime.output_stream_ids - # ) - def test_update_from_oci_model(self): model_deployment = self.initialize_model_deployment() model_deployment_from_oci = model_deployment._update_from_oci_model( @@ -882,151 +737,6 @@ def test_model_deployment_from_dict(self): assert new_model_deployment.to_dict() == model_deployment.to_dict() - @patch.object(DataScienceModel, "create") - def test_update_model_deployment_details(self, mock_create): - dsc_model = MagicMock() - dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" - mock_create.return_value = dsc_model - model_deployment = self.initialize_model_deployment() - update_model_deployment_details = ( - model_deployment._update_model_deployment_details() - ) - - mock_create.assert_called() - - assert isinstance( - update_model_deployment_details, - UpdateModelDeploymentDetails, - ) - assert ( - update_model_deployment_details.display_name - == model_deployment.display_name - ) - assert ( - update_model_deployment_details.description == model_deployment.description - ) - assert ( - update_model_deployment_details.freeform_tags - == model_deployment.freeform_tags - ) - assert ( - update_model_deployment_details.defined_tags - == model_deployment.defined_tags - ) - - category_log_details = update_model_deployment_details.category_log_details - assert isinstance(category_log_details, UpdateCategoryLogDetails) - assert ( - category_log_details.access.log_id - == model_deployment.infrastructure.access_log["logId"] - ) - assert ( - category_log_details.access.log_group_id - == model_deployment.infrastructure.access_log["logGroupId"] - ) - assert ( - category_log_details.predict.log_id - == model_deployment.infrastructure.predict_log["logId"] - ) - assert ( - category_log_details.predict.log_group_id - == model_deployment.infrastructure.predict_log["logGroupId"] - ) - - model_deployment_configuration_details = ( - update_model_deployment_details.model_deployment_configuration_details - ) - assert isinstance( - model_deployment_configuration_details, - UpdateSingleModelDeploymentConfigurationDetails, - ) - assert model_deployment_configuration_details.deployment_type == "SINGLE_MODEL" - - environment_configuration_details = ( - model_deployment_configuration_details.environment_configuration_details - ) - assert isinstance( - environment_configuration_details, - UpdateOcirModelDeploymentEnvironmentConfigurationDetails, - ) - assert ( - environment_configuration_details.environment_configuration_type - == "OCIR_CONTAINER" - ) - assert ( - environment_configuration_details.environment_variables - == model_deployment.runtime.env - ) - assert environment_configuration_details.cmd == model_deployment.runtime.cmd - assert environment_configuration_details.image == model_deployment.runtime.image - assert ( - environment_configuration_details.image_digest - == model_deployment.runtime.image_digest - ) - assert ( - environment_configuration_details.entrypoint - == model_deployment.runtime.entrypoint - ) - assert ( - environment_configuration_details.server_port - == model_deployment.runtime.server_port - ) - assert ( - environment_configuration_details.health_check_port - == model_deployment.runtime.health_check_port - ) - - model_configuration_details = ( - model_deployment_configuration_details.model_configuration_details - ) - assert isinstance( - model_configuration_details, - UpdateModelConfigurationDetails, - ) - assert ( - model_configuration_details.bandwidth_mbps - == model_deployment.infrastructure.bandwidth_mbps - ) - assert ( - model_configuration_details.model_id == model_deployment.runtime.model_uri - ) - - instance_configuration = model_configuration_details.instance_configuration - assert isinstance(instance_configuration, InstanceConfiguration) - assert ( - instance_configuration.instance_shape_name - == model_deployment.infrastructure.shape_name - ) - assert ( - instance_configuration.model_deployment_instance_shape_config_details.ocpus - == model_deployment.infrastructure.shape_config_details["ocpus"] - ) - assert ( - instance_configuration.model_deployment_instance_shape_config_details.memory_in_gbs - == model_deployment.infrastructure.shape_config_details["memoryInGBs"] - ) - - scaling_policy = model_configuration_details.scaling_policy - assert isinstance(scaling_policy, FixedSizeScalingPolicy) - assert scaling_policy.policy_type == "FIXED_SIZE" - assert scaling_policy.instance_count == model_deployment.infrastructure.replica - - # stream_configuration_details = ( - # model_deployment_configuration_details.stream_configuration_details - # ) - # assert isinstance( - # stream_configuration_details, - # UpdateStreamConfigurationDetails, - # ) - # assert ( - # stream_configuration_details.input_stream_ids - # == model_deployment.runtime.input_stream_ids - # ) - # assert ( - # stream_configuration_details.output_stream_ids - # == model_deployment.runtime.output_stream_ids - # ) - @patch.object( ModelDeploymentInfrastructure, "_load_default_properties", return_value={} ) @@ -1127,9 +837,7 @@ def test_from_ocid(self, mock_from_ocid): "create_model_deployment", ) @patch.object(DataScienceModel, "create") - def test_deploy( - self, mock_create, mock_create_model_deployment, mock_sync - ): + def test_deploy(self, mock_create, mock_create_model_deployment, mock_sync): dsc_model = MagicMock() dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" mock_create.return_value = dsc_model @@ -1346,44 +1054,35 @@ def test_update_spec(self): model_deployment = self.initialize_model_deployment() model_deployment._update_spec( display_name="test_updated_name", - freeform_tags={"test_updated_key":"test_updated_value"}, - access_log={ - "log_id": "test_updated_access_log_id" - }, - predict_log={ - "log_group_id": "test_updated_predict_log_group_id" - }, - shape_config_details={ - "ocpus": 100, - "memoryInGBs": 200 - }, + freeform_tags={"test_updated_key": "test_updated_value"}, + access_log={"log_id": "test_updated_access_log_id"}, + predict_log={"log_group_id": "test_updated_predict_log_group_id"}, + shape_config_details={"ocpus": 100, "memoryInGBs": 200}, replica=20, image="test_updated_image", - env={ - "test_updated_env_key":"test_updated_env_value" - } + env={"test_updated_env_key": "test_updated_env_value"}, ) assert model_deployment.display_name == "test_updated_name" assert model_deployment.freeform_tags == { - "test_updated_key":"test_updated_value" + "test_updated_key": "test_updated_value" } assert model_deployment.infrastructure.access_log == { "logId": "test_updated_access_log_id", - "logGroupId": "fakeid.loggroup.oc1.iad.xxx" + "logGroupId": "fakeid.loggroup.oc1.iad.xxx", } assert model_deployment.infrastructure.predict_log == { "logId": "fakeid.log.oc1.iad.xxx", - "logGroupId": "test_updated_predict_log_group_id" + "logGroupId": "test_updated_predict_log_group_id", } assert model_deployment.infrastructure.shape_config_details == { "ocpus": 100, - "memoryInGBs": 200 + "memoryInGBs": 200, } assert model_deployment.infrastructure.replica == 20 assert model_deployment.runtime.image == "test_updated_image" assert model_deployment.runtime.env == { - "test_updated_env_key":"test_updated_env_value" + "test_updated_env_key": "test_updated_env_value" } @patch.object(OCIDataScienceMixin, "sync") @@ -1393,18 +1092,14 @@ def test_update_spec(self): ) @patch.object(DataScienceModel, "create") def test_model_deployment_with_large_size_artifact( - self, - mock_create, - mock_create_model_deployment, - mock_sync + self, mock_create, mock_create_model_deployment, mock_sync ): dsc_model = MagicMock() dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" mock_create.return_value = dsc_model model_deployment = self.initialize_model_deployment() ( - model_deployment.runtime - .with_auth({"test_key":"test_value"}) + model_deployment.runtime.with_auth({"test_key": "test_value"}) .with_region("test_region") .with_overwrite_existing_artifact(True) .with_remove_existing_artifact(True) @@ -1425,18 +1120,18 @@ def test_model_deployment_with_large_size_artifact( mock_create_model_deployment.return_value = response model_deployment = self.initialize_model_deployment() model_deployment.set_spec(model_deployment.CONST_ID, "test_model_deployment_id") - + create_model_deployment_details = ( model_deployment._build_model_deployment_details() ) model_deployment.deploy(wait_for_completion=False) mock_create.assert_called_with( bucket_uri="test_bucket_uri", - auth={"test_key":"test_value"}, + auth={"test_key": "test_value"}, region="test_region", overwrite_existing_artifact=True, remove_existing_artifact=True, - timeout=100 + timeout=100, ) mock_create_model_deployment.assert_called_with(create_model_deployment_details) mock_sync.assert_called() From 6c4da24fbe7e0689e20cbdcf6c33a957e444b5e3 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Tue, 1 Jul 2025 12:52:34 -0400 Subject: [PATCH 3/8] Updated pr. --- ads/aqua/model/constants.py | 1 - ads/aqua/model/model.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ads/aqua/model/constants.py b/ads/aqua/model/constants.py index ce3e3f51d..194245fe4 100644 --- a/ads/aqua/model/constants.py +++ b/ads/aqua/model/constants.py @@ -20,7 +20,6 @@ class ModelCustomMetadataFields(ExtendedEnum): DEPLOYMENT_CONTAINER_URI = "deployment-container-uri" MULTIMODEL_GROUP_COUNT = "model_group_count" MULTIMODEL_METADATA = "multi_model_metadata" - MODEL_GROUP_CONFIG = "OCI_MODEL_GROUP_CUSTOM_METADATA" class ModelTask(ExtendedEnum): diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index 925535139..959fcc8e2 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -52,6 +52,7 @@ AQUA_MODEL_ARTIFACT_FILE, AQUA_MODEL_TOKENIZER_CONFIG, AQUA_MODEL_TYPE_CUSTOM, + AQUA_MULTI_MODEL_CONFIG, HF_METADATA_FOLDER, LICENSE, MODEL_BY_REFERENCE_OSS_PATH_KEY, @@ -479,7 +480,7 @@ def create_multi( category="Other", ) model_custom_metadata.add( - key=ModelCustomMetadataFields.MODEL_GROUP_CONFIG, + key=AQUA_MULTI_MODEL_CONFIG, value=self._build_model_group_config( create_deployment_details=create_deployment_details, model_config_summary=model_config_summary, From 9dede3880783709e671b2e4b000490dd7e31cda4 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Thu, 3 Jul 2025 15:28:57 -0400 Subject: [PATCH 4/8] Updated pr. --- ads/aqua/model/model.py | 127 +---------- ads/aqua/modeldeployment/deployment.py | 198 +++++++++++++++++- .../modeldeployment/model_group_config.py | 27 +-- .../with_extras/aqua/test_deployment.py | 29 ++- tests/unitary/with_extras/aqua/test_model.py | 102 ++------- 5 files changed, 241 insertions(+), 242 deletions(-) diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index 959fcc8e2..9aba63e15 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # Copyright (c) 2024, 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ -import json import os import pathlib import re @@ -39,12 +38,11 @@ generate_tei_cmd_var, get_artifact_path, get_hf_model_info, - get_preferred_compatible_family, list_os_files_with_extension, load_config, upload_folder, ) -from ads.aqua.config.container_config import AquaContainerConfig, Usage +from ads.aqua.config.container_config import AquaContainerConfig from ads.aqua.constants import ( AQUA_MODEL_ARTIFACT_CONFIG, AQUA_MODEL_ARTIFACT_CONFIG_MODEL_NAME, @@ -52,7 +50,6 @@ AQUA_MODEL_ARTIFACT_FILE, AQUA_MODEL_TOKENIZER_CONFIG, AQUA_MODEL_TYPE_CUSTOM, - AQUA_MULTI_MODEL_CONFIG, HF_METADATA_FOLDER, LICENSE, MODEL_BY_REFERENCE_OSS_PATH_KEY, @@ -237,8 +234,7 @@ def create( def create_multi( self, models: List[AquaMultiModelRef], - create_deployment_details, - model_config_summary, + model_custom_metadata: ModelCustomMetadata, project_id: Optional[str] = None, compartment_id: Optional[str] = None, freeform_tags: Optional[Dict] = None, @@ -253,11 +249,8 @@ def create_multi( ---------- models : List[AquaMultiModelRef] List of AquaMultiModelRef instances for creating a multi-model group. - create_deployment_details : CreateModelDeploymentDetails - An instance of CreateModelDeploymentDetails containing all required and optional - fields for creating a model deployment via Aqua. - model_config_summary : ModelConfigSummary - Summary Model Deployment configuration for the group of models. + model_custom_metadata : ModelCustomMetadata + Custom metadata for creating model group. project_id : Optional[str] The project ID for the multi-model group. compartment_id : Optional[str] @@ -276,46 +269,7 @@ def create_multi( DataScienceModelGroup Instance of DataScienceModelGroup object. """ - - if not models: - raise AquaValueError( - "Model list cannot be empty. Please provide at least one model for deployment." - ) - display_name_list = [] - model_custom_metadata = ModelCustomMetadata() - - service_inference_containers = ( - self.get_container_config().to_dict().get("inference") - ) - - supported_container_families = [ - container_config_item.family - for container_config_item in service_inference_containers - if any( - usage.upper() in container_config_item.usages - for usage in [Usage.MULTI_MODEL, Usage.OTHER] - ) - ] - - if not supported_container_families: - raise AquaValueError( - "Currently, there are no containers that support multi-model deployment." - ) - - selected_models_deployment_containers = set() - - if not source_models: - # Collect all unique model IDs (including fine-tuned models) - source_model_ids = list( - {model_id for model in models for model_id in model.all_model_ids()} - ) - logger.debug( - "Fetching source model metadata for model IDs: %s", source_model_ids - ) - - # Fetch source model metadata - source_models = self.get_multi_source(source_model_ids) or {} # Process each model in the input list for model in models: @@ -417,85 +371,12 @@ def create_multi( display_name_list.append(ft_model.model_name) - # Validate deployment container consistency - deployment_container = source_model.custom_metadata_list.get( - ModelCustomMetadataFields.DEPLOYMENT_CONTAINER, - ModelCustomMetadataItem( - key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER - ), - ).value - - if deployment_container not in supported_container_families: - logger.error( - "Unsupported deployment container '%s' for model '%s'. Supported: %s", - deployment_container, - source_model.id, - supported_container_families, - ) - raise AquaValueError( - f"Unsupported deployment container '{deployment_container}' for model '{source_model.id}'. " - f"Only {supported_container_families} are supported for multi-model deployments." - ) - - selected_models_deployment_containers.add(deployment_container) - - if not selected_models_deployment_containers: - raise AquaValueError( - "None of the selected models are associated with a recognized container family. " - "Please review the selected models, or select a different group of models." - ) - - # Check if the all models in the group shares same container family - if len(selected_models_deployment_containers) > 1: - deployment_container = get_preferred_compatible_family( - selected_families=selected_models_deployment_containers - ) - if not deployment_container: - raise AquaValueError( - "The selected models are associated with different container families: " - f"{list(selected_models_deployment_containers)}." - "For multi-model deployment, all models in the group must belong to the same container " - "family or to compatible container families." - ) - else: - deployment_container = selected_models_deployment_containers.pop() - # Generate model group details timestamp = datetime.now().strftime("%Y%m%d") model_group_display_name = f"model_group_{timestamp}" combined_models = ", ".join(display_name_list) model_group_description = f"Multi-model grouping using {combined_models}." - # Add global metadata - model_custom_metadata.add( - key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER, - value=deployment_container, - description=f"Inference container mapping for {model_group_display_name}", - category="Other", - ) - model_custom_metadata.add( - key=ModelCustomMetadataFields.MULTIMODEL_GROUP_COUNT, - value=str(len(models)), - description="Number of models in the group.", - category="Other", - ) - model_custom_metadata.add( - key=AQUA_MULTI_MODEL_CONFIG, - value=self._build_model_group_config( - create_deployment_details=create_deployment_details, - model_config_summary=model_config_summary, - deployment_container=deployment_container, - ), - description="Configs required to deploy multi models.", - category="Other", - ) - model_custom_metadata.add( - key=ModelCustomMetadataFields.MULTIMODEL_METADATA, - value=json.dumps([model.model_dump() for model in models]), - description="Metadata to store user's multi model input.", - category="Other", - ) - # Combine tags. The `Tags.AQUA_TAG` has been excluded, because we don't want to show # the models created for multi-model purpose in the AQUA models list. tags = { diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index 3e3f06788..eadcbbc5b 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -29,6 +29,7 @@ get_container_params_type, get_ocid_substring, get_params_list, + get_preferred_compatible_family, get_resource_name, get_restricted_params_by_container, load_gpu_shapes_index, @@ -40,6 +41,7 @@ AQUA_MODEL_TYPE_CUSTOM, AQUA_MODEL_TYPE_MULTI, AQUA_MODEL_TYPE_SERVICE, + AQUA_MULTI_MODEL_CONFIG, MODEL_BY_REFERENCE_OSS_PATH_KEY, MODEL_NAME_DELIMITER, UNKNOWN_DICT, @@ -64,6 +66,7 @@ ConfigValidationError, CreateModelDeploymentDetails, ) +from ads.aqua.modeldeployment.model_group_config import ModelGroupConfig from ads.common.object_storage_details import ObjectStorageDetails from ads.common.utils import UNKNOWN, get_log_links from ads.common.work_request import DataScienceWorkRequest @@ -84,7 +87,7 @@ ModelDeploymentInfrastructure, ModelDeploymentMode, ) -from ads.model.model_metadata import ModelCustomMetadataItem +from ads.model.model_metadata import ModelCustomMetadata, ModelCustomMetadataItem from ads.telemetry import telemetry @@ -324,10 +327,16 @@ def create( f"Multi models ({source_model_ids}) provided. Delegating to multi model creation method." ) - aqua_model_group = model_app.create_multi( + model_group_custom_metadata = self._create_model_group_custom_metadata( models=create_deployment_details.models, create_deployment_details=create_deployment_details, model_config_summary=model_config_summary, + source_models=source_models, + ) + + aqua_model_group = model_app.create_multi( + models=create_deployment_details.models, + model_custom_metadata=model_group_custom_metadata, compartment_id=compartment_id, project_id=project_id, freeform_tags=freeform_tags, @@ -340,6 +349,191 @@ def create( container_config=container_config, ) + @telemetry(entry_point="plugin=model&action=create", name="aqua") + def _create_model_group_custom_metadata( + self, + models: List[AquaMultiModelRef], + create_deployment_details: CreateModelDeploymentDetails, + model_config_summary: ModelDeploymentConfigSummary, + source_models: Optional[Dict[str, DataScienceModel]] = None, + ) -> ModelCustomMetadata: + """ + Creates multi-model group metadata list. + + Parameters + ---------- + models : List[AquaMultiModelRef] + List of AquaMultiModelRef instances for creating a multi-model group. + create_deployment_details : CreateModelDeploymentDetails + An instance of CreateModelDeploymentDetails containing all required and optional + fields for creating a model deployment via Aqua. + model_config_summary : ModelConfigSummary + Summary Model Deployment configuration for the group of models. + source_models: Optional[Dict[str, DataScienceModel]] + A mapping of model OCIDs to their corresponding `DataScienceModel` objects. + This dictionary contains metadata for all models involved in the multi-model deployment, + including both base models and fine-tuned weights. + + Returns + ------- + ModelCustomMetadata + Instance of ModelCustomMetadata object. + """ + + if not models: + raise AquaValueError( + "Model list cannot be empty. Please provide at least one model for deployment." + ) + + model_custom_metadata = ModelCustomMetadata() + + service_inference_containers = ( + self.get_container_config().to_dict().get("inference") + ) + + supported_container_families = [ + container_config_item.family + for container_config_item in service_inference_containers + if any( + usage.upper() in container_config_item.usages + for usage in [Usage.MULTI_MODEL, Usage.OTHER] + ) + ] + + if not supported_container_families: + raise AquaValueError( + "Currently, there are no containers that support multi-model deployment." + ) + + selected_models_deployment_containers = set() + + if not source_models: + # Collect all unique model IDs (including fine-tuned models) + source_model_ids = list( + {model_id for model in models for model_id in model.all_model_ids()} + ) + logger.debug( + "Fetching source model metadata for model IDs: %s", source_model_ids + ) + + # Fetch source model metadata + source_models = self.get_multi_source(source_model_ids) or {} + + # Process each model in the input list + for model in models: + # Retrieve base model metadata + source_model: DataScienceModel = source_models.get(model.model_id) + if not source_model: + logger.error( + "Failed to fetch metadata for base model ID: %s", model.model_id + ) + raise AquaValueError( + f"Unable to retrieve metadata for base model ID: {model.model_id}." + ) + + # Validate deployment container consistency + deployment_container = source_model.custom_metadata_list.get( + ModelCustomMetadataFields.DEPLOYMENT_CONTAINER, + ModelCustomMetadataItem( + key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER + ), + ).value + + if deployment_container not in supported_container_families: + logger.error( + "Unsupported deployment container '%s' for model '%s'. Supported: %s", + deployment_container, + source_model.id, + supported_container_families, + ) + raise AquaValueError( + f"Unsupported deployment container '{deployment_container}' for model '{source_model.id}'. " + f"Only {supported_container_families} are supported for multi-model deployments." + ) + + selected_models_deployment_containers.add(deployment_container) + + if not selected_models_deployment_containers: + raise AquaValueError( + "None of the selected models are associated with a recognized container family. " + "Please review the selected models, or select a different group of models." + ) + + # Check if the all models in the group shares same container family + if len(selected_models_deployment_containers) > 1: + deployment_container = get_preferred_compatible_family( + selected_families=selected_models_deployment_containers + ) + if not deployment_container: + raise AquaValueError( + "The selected models are associated with different container families: " + f"{list(selected_models_deployment_containers)}." + "For multi-model deployment, all models in the group must belong to the same container " + "family or to compatible container families." + ) + else: + deployment_container = selected_models_deployment_containers.pop() + + # Generate model group details + timestamp = datetime.now().strftime("%Y%m%d") + model_group_display_name = f"model_group_{timestamp}" + + # Add global metadata + model_custom_metadata.add( + key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER, + value=deployment_container, + description=f"Inference container mapping for {model_group_display_name}", + category="Other", + ) + model_custom_metadata.add( + key=ModelCustomMetadataFields.MULTIMODEL_GROUP_COUNT, + value=str(len(models)), + description="Number of models in the group.", + category="Other", + ) + model_custom_metadata.add( + key=AQUA_MULTI_MODEL_CONFIG, + value=self._build_model_group_config( + create_deployment_details=create_deployment_details, + model_config_summary=model_config_summary, + deployment_container=deployment_container, + ), + description="Configs required to deploy multi models.", + category="Other", + ) + model_custom_metadata.add( + key=ModelCustomMetadataFields.MULTIMODEL_METADATA, + value=json.dumps([model.model_dump() for model in models]), + description="Metadata to store user's multi model input.", + category="Other", + ) + + return model_custom_metadata + + def _build_model_group_config( + self, + create_deployment_details, + model_config_summary, + deployment_container: str, + ) -> str: + """Builds model group config required to deploy multi models.""" + container_type_key = ( + create_deployment_details.container_family or deployment_container + ) + container_config = self.get_container_config_item(container_type_key) + container_spec = container_config.spec if container_config else UNKNOWN + + container_params = container_spec.cli_param if container_spec else UNKNOWN + + multi_model_config = ModelGroupConfig.from_create_model_deployment_details( + create_deployment_details, + model_config_summary, + container_type_key, + container_params, + ) + + return multi_model_config.model_dump_json() + def _create( self, aqua_model: DataScienceModel, diff --git a/ads/aqua/modeldeployment/model_group_config.py b/ads/aqua/modeldeployment/model_group_config.py index e452ec7f5..f24078109 100644 --- a/ads/aqua/modeldeployment/model_group_config.py +++ b/ads/aqua/modeldeployment/model_group_config.py @@ -4,7 +4,7 @@ from typing import List, Optional, Tuple, Union -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field from typing_extensions import Self from ads.aqua import logger @@ -61,18 +61,19 @@ class BaseModelSpec(BaseModel): description="Optional list of fine-tuned model variants associated with this base model.", ) - @field_validator("model_path") @classmethod - def clean_model_path(cls, artifact_path_prefix: str) -> str: - """Validates and cleans the file path for model_path parameter.""" - if ObjectStorageDetails.is_oci_path(artifact_path_prefix): - os_path = ObjectStorageDetails.from_path(artifact_path_prefix) - artifact_path_prefix = os_path.filepath.rstrip("/") - return artifact_path_prefix - - raise AquaValueError( - "The base model path is not available in the model artifact." - ) + def build_model_path(cls, model_id: str, artifact_path_prefix: str) -> str: + """Cleans and builds the file path for model_path parameter + to format: / + """ + if not ObjectStorageDetails.is_oci_path(artifact_path_prefix): + raise AquaValueError( + "The base model path is not available in the model artifact." + ) + + os_path = ObjectStorageDetails.from_path(artifact_path_prefix) + artifact_path_prefix = os_path.filepath.rstrip("/") + return model_id + "/" + artifact_path_prefix @classmethod def dedup_lora_modules(cls, fine_tune_weights: List[LoraModuleSpec]): @@ -99,7 +100,7 @@ def from_aqua_multi_model_ref( return cls( model_id=model.model_id, - model_path=model.artifact_location, + model_path=cls.build_model_path(model.model_id, model.artifact_location), params=model_params, model_task=model.model_task, fine_tune_weights=cls.dedup_lora_modules(model.fine_tune_weights), diff --git a/tests/unitary/with_extras/aqua/test_deployment.py b/tests/unitary/with_extras/aqua/test_deployment.py index 2e6329fa8..ec573ffdc 100644 --- a/tests/unitary/with_extras/aqua/test_deployment.py +++ b/tests/unitary/with_extras/aqua/test_deployment.py @@ -23,7 +23,6 @@ ModelDeployWorkloadConfigurationDetails, ) from parameterized import parameterized -from pydantic import ValidationError import ads.aqua.modeldeployment.deployment import ads.config @@ -1014,14 +1013,14 @@ class TestDataset: { "model_id": "model_a", "fine_tune_weights": [], - "model_path": "", + "model_path": "model_a/", "model_task": "text_embedding", "params": "--example-container-params test --served-model-name test_model_1 --tensor-parallel-size 1 --trust-remote-code --max-model-len 60000", }, { "model_id": "model_b", "fine_tune_weights": [], - "model_path": "", + "model_path": "model_b/", "model_task": "image_text_to_text", "params": "--example-container-params test --served-model-name test_model_2 --tensor-parallel-size 2 --trust-remote-code --max-model-len 32000", }, @@ -1034,7 +1033,7 @@ class TestDataset: "model_path": "oci://test_bucket@test_namespace/models/ft-models/meta-llama-3b/ocid1.datasciencejob.oc1.iad.", }, ], - "model_path": "", + "model_path": "model_c/", "model_task": "code_synthesis", "params": "--example-container-params test --served-model-name test_model_3 --tensor-parallel-size 4", }, @@ -1046,14 +1045,14 @@ class TestDataset: { "model_id": "model_a", "fine_tune_weights": [], - "model_path": "", + "model_path": "model_a/", "model_task": "text_embedding", "params": "--example-container-params test --served-model-name test_model_1 --tensor-parallel-size 1 --trust-remote-code --max-model-len 60000", }, { "model_id": "model_b", "fine_tune_weights": [], - "model_path": "", + "model_path": "model_b/", "model_task": "image_text_to_text", "params": "--example-container-params test --served-model-name test_model_2 --tensor-parallel-size 2 --trust-remote-code --max-model-len 32000", }, @@ -1794,6 +1793,9 @@ def test_create_deployment_for_tei_byoc_embedding_model( @patch.object(AquaApp, "get_container_image") @patch("ads.model.deployment.model_deployment.ModelDeployment.deploy") @patch("ads.aqua.modeldeployment.AquaDeploymentApp.get_deployment_config") + @patch( + "ads.aqua.modeldeployment.AquaDeploymentApp._create_model_group_custom_metadata" + ) @patch( "ads.aqua.modeldeployment.entities.CreateModelDeploymentDetails.validate_multimodel_deployment_feasibility" ) @@ -1806,6 +1808,7 @@ def test_create_deployment_for_multi_model( mock_get_multi_source, mock_validate_input_models, mock_validate_multimodel_deployment_feasibility, + mock_create_model_group_custom_metadata, mock_get_deployment_config, mock_deploy, mock_get_container_image, @@ -1813,6 +1816,7 @@ def test_create_deployment_for_multi_model( mock_get_container_config, ): """Test to create a deployment for multi models.""" + mock_create_model_group_custom_metadata.return_value = MagicMock() mock_get_container_config.return_value = ( AquaContainerConfig.from_service_config( service_containers=TestDataset.CONTAINER_LIST @@ -2434,16 +2438,11 @@ def test_invalid_from_aqua_multi_model_ref( model_params = "--dummy-param" if expect_error: - with pytest.raises(ValidationError) as excinfo: + with pytest.raises( + AquaValueError, + match="The base model path is not available in the model artifact.", + ): BaseModelSpec.from_aqua_multi_model_ref(model_ref, model_params) - errs = excinfo.value.errors() - if not model_path.startswith("oci://"): - model_path_errors = [e for e in errs if e["loc"] == ("model_path",)] - assert model_path_errors, f"expected a model_path error, got: {errs!r}" - assert ( - "the base model path is not available in the model artifact." - in model_path_errors[0]["msg"].lower() - ) else: BaseModelSpec.from_aqua_multi_model_ref(model_ref, model_params) diff --git a/tests/unitary/with_extras/aqua/test_model.py b/tests/unitary/with_extras/aqua/test_model.py index 5f96cb9a4..42d9e8da0 100644 --- a/tests/unitary/with_extras/aqua/test_model.py +++ b/tests/unitary/with_extras/aqua/test_model.py @@ -475,14 +475,8 @@ def test_create_multimodel( } mock_model.id = "mock_model_id" mock_model.artifact = "mock_artifact_path" - custom_metadata_list = ModelCustomMetadata() - custom_metadata_list.add( - **{"key": "deployment-container", "value": "odsc-tgi-serving"} - ) - mock_model.custom_metadata_list = custom_metadata_list - mock_create_deployment_details = MagicMock() - mock_model_config_summary = MagicMock() + model_custom_metadata = MagicMock() model_info_1 = AquaMultiModelRef( model_id="test_model_id_1", @@ -498,45 +492,24 @@ def test_create_multimodel( env_var={"params": "--trust-remote-code --max-model-len 32000"}, ) + # testing fine tuned model in model group + model_info_3 = AquaMultiModelRef( + model_id="test_model_id_3", + gpu_count=2, + model_task="image_text_to_text", + env_var={"params": "--trust-remote-code --max-model-len 32000"}, + artifact_location="oci://test_bucket@test_namespace/models/meta-llama/Llama-3.2-3B-Instruct", + fine_tune_artifact="oci://test_bucket@test_namespace/models/ft-models/meta-llama-3b/ocid1.datasciencejob.oc1.iad.", + ) + model_details = { model_info_1.model_id: mock_model, model_info_2.model_id: mock_model, + model_info_3.model_id: mock_model, } - with pytest.raises(AquaValueError): - model_group = self.app.create_multi( - models=[model_info_1, model_info_2], - create_deployment_details=mock_create_deployment_details, - model_config_summary=mock_model_config_summary, - project_id="test_project_id", - compartment_id="test_compartment_id", - source_models=model_details, - ) - mock_model.freeform_tags["aqua_service_model"] = TestDataset.SERVICE_MODEL_ID - - with pytest.raises(AquaValueError): - model_group = self.app.create_multi( - models=[model_info_1, model_info_2], - create_deployment_details=mock_create_deployment_details, - model_config_summary=mock_model_config_summary, - project_id="test_project_id", - compartment_id="test_compartment_id", - source_models=model_details, - ) - mock_model.freeform_tags["task"] = "text-generation" - - with pytest.raises(AquaValueError): - model_group = self.app.create_multi( - models=[model_info_1, model_info_2], - create_deployment_details=mock_create_deployment_details, - model_config_summary=mock_model_config_summary, - project_id="test_project_id", - compartment_id="test_compartment_id", - source_models=model_details, - ) - custom_metadata_list = ModelCustomMetadata() custom_metadata_list.add( **{"key": "deployment-container", "value": "odsc-vllm-serving"} @@ -544,57 +517,15 @@ def test_create_multimodel( mock_model.custom_metadata_list = custom_metadata_list - # testing _extract_model_task when a user passes an invalid task to AquaMultiModelRef - model_info_1.model_task = "invalid_task" - - with pytest.raises(AquaValueError): - model_group = self.app.create_multi( - models=[model_info_1, model_info_2], - create_deployment_details=mock_create_deployment_details, - model_config_summary=mock_model_config_summary, - project_id="test_project_id", - compartment_id="test_compartment_id", - source_models=model_details, - ) - - # testing if a user tries to invoke a model with a task mode that is not yet supported - model_info_1.model_task = None - mock_model.freeform_tags["task"] = "unsupported_task" - with pytest.raises(AquaValueError): - model_group = self.app.create_multi( - models=[model_info_1, model_info_2], - create_deployment_details=mock_create_deployment_details, - model_config_summary=mock_model_config_summary, - project_id="test_project_id", - compartment_id="test_compartment_id", - source_models=model_details, - ) - - mock_model.freeform_tags["task"] = "text-generation" - model_info_1.model_task = "text_embedding" - # testing requesting metadata from fine tuned model to add to model group mock_model.model_file_description = ( TestDataset.fine_tuned_model_file_description ) - # testing fine tuned model in model group - model_info_3 = AquaMultiModelRef( - model_id="test_model_id_3", - gpu_count=2, - model_task="image_text_to_text", - env_var={"params": "--trust-remote-code --max-model-len 32000"}, - artifact_location="oci://test_bucket@test_namespace/models/meta-llama/Llama-3.2-3B-Instruct", - fine_tune_artifact="oci://test_bucket@test_namespace/models/ft-models/meta-llama-3b/ocid1.datasciencejob.oc1.iad.", - ) - - model_details[model_info_3.model_id] = mock_model - # will create a multi-model group model_group = self.app.create_multi( models=[model_info_1, model_info_2, model_info_3], - create_deployment_details=mock_create_deployment_details, - model_config_summary=mock_model_config_summary, + model_custom_metadata=model_custom_metadata, project_id="test_project_id", compartment_id="test_compartment_id", source_models=model_details, @@ -602,14 +533,7 @@ def test_create_multimodel( mock_create_group.assert_called() - mock_model.compartment_id = TestDataset.SERVICE_COMPARTMENT_ID - assert model_group.freeform_tags == {"aqua_multimodel": "true"} - assert model_group.custom_metadata_list.get("model_group_count").value == "3" - assert ( - model_group.custom_metadata_list.get("deployment-container").value - == "odsc-vllm-serving" - ) @pytest.mark.parametrize( "foundation_model_type", From c3f7ff2ffc215f3a21c715c4364590a79735d4fd Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Thu, 3 Jul 2025 15:38:17 -0400 Subject: [PATCH 5/8] Updated pr. --- ads/aqua/model/model.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index 9aba63e15..5acd050db 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -415,32 +415,6 @@ def create_multi( return custom_model_group - def _build_model_group_config( - self, - create_deployment_details, - model_config_summary, - deployment_container: str, - ) -> str: - """Builds model group config required to deploy multi models.""" - container_type_key = ( - create_deployment_details.container_family or deployment_container - ) - container_config = self.get_container_config_item(container_type_key) - container_spec = container_config.spec if container_config else UNKNOWN - - container_params = container_spec.cli_param if container_spec else UNKNOWN - - from ads.aqua.modeldeployment.model_group_config import ModelGroupConfig - - multi_model_config = ModelGroupConfig.from_create_model_deployment_details( - create_deployment_details, - model_config_summary, - container_type_key, - container_params, - ) - - return multi_model_config.model_dump_json() - @telemetry(entry_point="plugin=model&action=get", name="aqua") def get(self, model_id: str) -> "AquaModel": """Gets the information of an Aqua model. From 3265fe713e374b754e8b536159b6438f0a6c3cb2 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Mon, 7 Jul 2025 13:13:55 -0400 Subject: [PATCH 6/8] Updated pr. --- ads/aqua/model/model.py | 163 ++---------------- ads/aqua/modeldeployment/deployment.py | 162 +++++++++++++++-- .../modeldeployment/model_group_config.py | 2 +- ads/model/deployment/model_deployment.py | 13 +- .../with_extras/aqua/test_deployment.py | 14 +- tests/unitary/with_extras/aqua/test_model.py | 4 + 6 files changed, 185 insertions(+), 173 deletions(-) diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index 5acd050db..729f43622 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -3,7 +3,6 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ import os import pathlib -import re from datetime import datetime, timedelta from threading import Lock from typing import Any, Dict, List, Optional, Set, Union @@ -80,8 +79,6 @@ MemberModel, ModelValidationResult, ) -from ads.aqua.model.enums import MultiModelSupportedTaskType -from ads.aqua.model.utils import extract_fine_tune_artifacts_path from ads.common.auth import default_signer from ads.common.oci_resource import SEARCH_TYPE, OCIResource from ads.common.utils import UNKNOWN, get_console_link, is_path_exists, read_file @@ -235,11 +232,13 @@ def create_multi( self, models: List[AquaMultiModelRef], model_custom_metadata: ModelCustomMetadata, + model_group_display_name: str, + model_group_description: str, + tags: Dict, + combined_models: str, project_id: Optional[str] = None, compartment_id: Optional[str] = None, - freeform_tags: Optional[Dict] = None, defined_tags: Optional[Dict] = None, - source_models: Optional[Dict[str, DataScienceModel]] = None, **kwargs, # noqa: ARG002 ) -> DataScienceModelGroup: """ @@ -251,141 +250,26 @@ def create_multi( List of AquaMultiModelRef instances for creating a multi-model group. model_custom_metadata : ModelCustomMetadata Custom metadata for creating model group. + model_group_display_name: str + The model group display name. + model_group_description: str + The model group description. + tags: Dict + The tags of model group. + combined_models: str + The name of models to be grouped and deployed. project_id : Optional[str] The project ID for the multi-model group. compartment_id : Optional[str] The compartment ID for the multi-model group. - freeform_tags : Optional[Dict] - Freeform tags for the model. defined_tags : Optional[Dict] Defined tags for the model. - source_models: Optional[Dict[str, DataScienceModel]] - A mapping of model OCIDs to their corresponding `DataScienceModel` objects. - This dictionary contains metadata for all models involved in the multi-model deployment, - including both base models and fine-tuned weights. Returns ------- DataScienceModelGroup Instance of DataScienceModelGroup object. """ - display_name_list = [] - - # Process each model in the input list - for model in models: - # Retrieve base model metadata - source_model: DataScienceModel = source_models.get(model.model_id) - if not source_model: - logger.error( - "Failed to fetch metadata for base model ID: %s", model.model_id - ) - raise AquaValueError( - f"Unable to retrieve metadata for base model ID: {model.model_id}." - ) - - # Use display name as fallback if model name not provided - model.model_name = model.model_name or source_model.display_name - - # Validate model file description - model_file_description = source_model.model_file_description - if not model_file_description: - logger.error( - "Model '%s' (%s) has no file description.", - source_model.display_name, - model.model_id, - ) - raise AquaValueError( - f"Model '{source_model.display_name}' (ID: {model.model_id}) has no file description. " - "Please register the model with a file description." - ) - - # Ensure base model has a valid artifact - if not source_model.artifact: - logger.error( - "Base model '%s' (%s) has no artifact.", - model.model_name, - model.model_id, - ) - raise AquaValueError( - f"Model '{model.model_name}' (ID: {model.model_id}) has no registered artifacts. " - "Please register the model before deployment." - ) - - # Set base model artifact path - model.artifact_location = source_model.artifact - logger.debug( - "Model '%s' artifact path set to: %s", - model.model_name, - model.artifact_location, - ) - - display_name_list.append(model.model_name) - - # Extract model task metadata from source model - self._extract_model_task(model, source_model) - - # Process fine-tuned weights if provided - for ft_model in model.fine_tune_weights or []: - fine_tune_source_model: DataScienceModel = source_models.get( - ft_model.model_id - ) - if not fine_tune_source_model: - logger.error( - "Failed to fetch metadata for fine-tuned model ID: %s", - ft_model.model_id, - ) - raise AquaValueError( - f"Unable to retrieve metadata for fine-tuned model ID: {ft_model.model_id}." - ) - - # Validate model file description - ft_model_file_description = ( - fine_tune_source_model.model_file_description - ) - if not ft_model_file_description: - logger.error( - "Model '%s' (%s) has no file description.", - fine_tune_source_model.display_name, - ft_model.model_id, - ) - raise AquaValueError( - f"Model '{fine_tune_source_model.display_name}' (ID: {ft_model.model_id}) has no file description. " - "Please register the model with a file description." - ) - - # Extract fine-tuned model path - _, fine_tune_path = extract_fine_tune_artifacts_path( - fine_tune_source_model - ) - logger.debug( - "Resolved fine-tuned model path for '%s': %s", - ft_model.model_id, - fine_tune_path, - ) - ft_model.model_path = fine_tune_path - - # Use fallback name if needed - ft_model.model_name = ( - ft_model.model_name or fine_tune_source_model.display_name - ) - - display_name_list.append(ft_model.model_name) - - # Generate model group details - timestamp = datetime.now().strftime("%Y%m%d") - model_group_display_name = f"model_group_{timestamp}" - combined_models = ", ".join(display_name_list) - model_group_description = f"Multi-model grouping using {combined_models}." - - # Combine tags. The `Tags.AQUA_TAG` has been excluded, because we don't want to show - # the models created for multi-model purpose in the AQUA models list. - tags = { - # Tags.AQUA_TAG: "active", - Tags.MULTIMODEL_TYPE_TAG: "true", - **(freeform_tags or {}), - } - - # Create multi-model group custom_model_group = ( DataScienceModelGroup() .with_compartment_id(compartment_id) @@ -399,11 +283,10 @@ def create_multi( [MemberModel(model_id=model.model_id).model_dump() for model in models] ) ) - custom_model_group.create() logger.info( - f"Aqua Model Group'{custom_model_group.id}' created with models: {', '.join(display_name_list)}." + f"Aqua Model Group'{custom_model_group.id}' created with models: {combined_models}." ) # Track telemetry event @@ -679,26 +562,6 @@ def edit_registered_model( else: raise AquaRuntimeError("Only registered unverified models can be edited.") - def _extract_model_task( - self, - model: AquaMultiModelRef, - source_model: DataScienceModel, - ) -> None: - """In a Multi Model Deployment, will set model_task parameter in AquaMultiModelRef from freeform tags or user""" - # user does not supply model task, we extract from model metadata - if not model.model_task: - model.model_task = source_model.freeform_tags.get(Tags.TASK, UNKNOWN) - - task_tag = re.sub(r"-", "_", model.model_task).lower() - # re-visit logic when more model task types are supported - if task_tag in MultiModelSupportedTaskType: - model.model_task = task_tag - else: - raise AquaValueError( - f"Invalid or missing {task_tag} tag for selected model {source_model.display_name}. " - f"Currently only `{MultiModelSupportedTaskType.values()}` models are supported for multi model deployment." - ) - def _fetch_metric_from_metadata( self, custom_metadata_list: ModelCustomMetadata, diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index eadcbbc5b..8d0a12871 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -4,6 +4,7 @@ import json +import re import shlex import threading from datetime import datetime, timedelta @@ -49,6 +50,7 @@ from ads.aqua.data import AquaResourceIdentifier from ads.aqua.model import AquaModelApp from ads.aqua.model.constants import AquaModelMetadataKeys, ModelCustomMetadataFields +from ads.aqua.model.enums import MultiModelSupportedTaskType from ads.aqua.model.utils import ( extract_base_model_from_ft, extract_fine_tune_artifacts_path, @@ -327,21 +329,30 @@ def create( f"Multi models ({source_model_ids}) provided. Delegating to multi model creation method." ) - model_group_custom_metadata = self._create_model_group_custom_metadata( + ( + model_group_display_name, + model_group_description, + tags, + model_custom_metadata, + combined_models, + ) = self._build_model_group_configs( models=create_deployment_details.models, create_deployment_details=create_deployment_details, model_config_summary=model_config_summary, + freeform_tags=freeform_tags, source_models=source_models, ) aqua_model_group = model_app.create_multi( models=create_deployment_details.models, - model_custom_metadata=model_group_custom_metadata, + model_custom_metadata=model_custom_metadata, + model_group_display_name=model_group_display_name, + model_group_description=model_group_description, + tags=tags, + combined_models=combined_models, compartment_id=compartment_id, project_id=project_id, - freeform_tags=freeform_tags, defined_tags=defined_tags, - source_models=source_models, ) return self._create_multi( aqua_model_group=aqua_model_group, @@ -350,15 +361,17 @@ def create( ) @telemetry(entry_point="plugin=model&action=create", name="aqua") - def _create_model_group_custom_metadata( + def _build_model_group_configs( self, models: List[AquaMultiModelRef], create_deployment_details: CreateModelDeploymentDetails, model_config_summary: ModelDeploymentConfigSummary, + freeform_tags: Optional[Dict] = None, source_models: Optional[Dict[str, DataScienceModel]] = None, - ) -> ModelCustomMetadata: + **kwargs, # noqa: ARG002 + ) -> tuple: """ - Creates multi-model group metadata list. + Builds configs for a multi-model grouping using the provided model list. Parameters ---------- @@ -369,6 +382,8 @@ def _create_model_group_custom_metadata( fields for creating a model deployment via Aqua. model_config_summary : ModelConfigSummary Summary Model Deployment configuration for the group of models. + freeform_tags : Optional[Dict] + Freeform tags for the model. source_models: Optional[Dict[str, DataScienceModel]] A mapping of model OCIDs to their corresponding `DataScienceModel` objects. This dictionary contains metadata for all models involved in the multi-model deployment, @@ -376,8 +391,8 @@ def _create_model_group_custom_metadata( Returns ------- - ModelCustomMetadata - Instance of ModelCustomMetadata object. + tuple + A tuple of required metadata and strings to create model group. """ if not models: @@ -385,6 +400,7 @@ def _create_model_group_custom_metadata( "Model list cannot be empty. Please provide at least one model for deployment." ) + display_name_list = [] model_custom_metadata = ModelCustomMetadata() service_inference_containers = ( @@ -431,6 +447,96 @@ def _create_model_group_custom_metadata( f"Unable to retrieve metadata for base model ID: {model.model_id}." ) + # Use display name as fallback if model name not provided + model.model_name = model.model_name or source_model.display_name + + # Validate model file description + model_file_description = source_model.model_file_description + if not model_file_description: + logger.error( + "Model '%s' (%s) has no file description.", + source_model.display_name, + model.model_id, + ) + raise AquaValueError( + f"Model '{source_model.display_name}' (ID: {model.model_id}) has no file description. " + "Please register the model with a file description." + ) + + # Ensure base model has a valid artifact + if not source_model.artifact: + logger.error( + "Base model '%s' (%s) has no artifact.", + model.model_name, + model.model_id, + ) + raise AquaValueError( + f"Model '{model.model_name}' (ID: {model.model_id}) has no registered artifacts. " + "Please register the model before deployment." + ) + + # Set base model artifact path + model.artifact_location = source_model.artifact + logger.debug( + "Model '%s' artifact path set to: %s", + model.model_name, + model.artifact_location, + ) + + display_name_list.append(model.model_name) + + # Extract model task metadata from source model + self._extract_model_task(model, source_model) + + # Process fine-tuned weights if provided + for ft_model in model.fine_tune_weights or []: + fine_tune_source_model: DataScienceModel = source_models.get( + ft_model.model_id + ) + if not fine_tune_source_model: + logger.error( + "Failed to fetch metadata for fine-tuned model ID: %s", + ft_model.model_id, + ) + raise AquaValueError( + f"Unable to retrieve metadata for fine-tuned model ID: {ft_model.model_id}." + ) + + # Validate model file description + ft_model_file_description = ( + fine_tune_source_model.model_file_description + ) + if not ft_model_file_description: + logger.error( + "Model '%s' (%s) has no file description.", + fine_tune_source_model.display_name, + ft_model.model_id, + ) + raise AquaValueError( + f"Model '{fine_tune_source_model.display_name}' (ID: {ft_model.model_id}) has no file description. " + "Please register the model with a file description." + ) + + # Extract fine-tuned model path + _, fine_tune_path = extract_fine_tune_artifacts_path( + fine_tune_source_model + ) + logger.debug( + "Resolved fine-tuned model path for '%s': %s", + ft_model.model_id, + fine_tune_path, + ) + ft_model.model_path = ( + ft_model.model_id + "/" + fine_tune_path.lstrip("/") + ) + + # Use fallback name if needed + ft_model.model_name = ( + ft_model.model_name or fine_tune_source_model.display_name + ) + + display_name_list.append(ft_model.model_name) + # Validate deployment container consistency deployment_container = source_model.custom_metadata_list.get( ModelCustomMetadataFields.DEPLOYMENT_CONTAINER, @@ -477,6 +583,8 @@ def _create_model_group_custom_metadata( # Generate model group details timestamp = datetime.now().strftime("%Y%m%d") model_group_display_name = f"model_group_{timestamp}" + combined_models = ", ".join(display_name_list) + model_group_description = f"Multi-model grouping using {combined_models}." # Add global metadata model_custom_metadata.add( @@ -508,7 +616,41 @@ def _create_model_group_custom_metadata( category="Other", ) - return model_custom_metadata + # Combine tags. The `Tags.AQUA_TAG` has been excluded, because we don't want to show + # the models created for multi-model purpose in the AQUA models list. + tags = { + # Tags.AQUA_TAG: "active", + Tags.MULTIMODEL_TYPE_TAG: "true", + **(freeform_tags or {}), + } + + return ( + model_group_display_name, + model_group_description, + tags, + model_custom_metadata, + combined_models, + ) + + def _extract_model_task( + self, + model: AquaMultiModelRef, + source_model: DataScienceModel, + ) -> None: + """In a Multi Model Deployment, will set model_task parameter in AquaMultiModelRef from freeform tags or user""" + # user does not supply model task, we extract from model metadata + if not model.model_task: + model.model_task = source_model.freeform_tags.get(Tags.TASK, UNKNOWN) + + task_tag = re.sub(r"-", "_", model.model_task).lower() + # re-visit logic when more model task types are supported + if task_tag in MultiModelSupportedTaskType: + model.model_task = task_tag + else: + raise AquaValueError( + f"Invalid or missing {task_tag} tag for selected model {source_model.display_name}. " + f"Currently only `{MultiModelSupportedTaskType.values()}` models are supported for multi model deployment." + ) def _build_model_group_config( self, diff --git a/ads/aqua/modeldeployment/model_group_config.py b/ads/aqua/modeldeployment/model_group_config.py index f24078109..99f7c2fee 100644 --- a/ads/aqua/modeldeployment/model_group_config.py +++ b/ads/aqua/modeldeployment/model_group_config.py @@ -73,7 +73,7 @@ def build_model_path(cls, model_id: str, artifact_path_prefix: str) -> str: os_path = ObjectStorageDetails.from_path(artifact_path_prefix) artifact_path_prefix = os_path.filepath.rstrip("/") - return model_id + "/" + artifact_path_prefix + return model_id + "/" + artifact_path_prefix.lstrip("/") @classmethod def dedup_lora_modules(cls, fine_tune_weights: List[LoraModuleSpec]): diff --git a/ads/model/deployment/model_deployment.py b/ads/model/deployment/model_deployment.py index 21e083499..4bbd4b7ab 100644 --- a/ads/model/deployment/model_deployment.py +++ b/ads/model/deployment/model_deployment.py @@ -1521,9 +1521,9 @@ def _build_model_deployment_details(self) -> CreateModelDeploymentDetails: self.infrastructure.CONST_CATEGORY_LOG_DETAILS: self._build_category_log_details(), } - return CreateModelDeploymentDetails( - **ads_utils.batch_convert_case(create_model_deployment_details, "snake") - ) + return OCIDataScienceModelDeployment( + **create_model_deployment_details + ).to_oci_model(CreateModelDeploymentDetails) def _update_model_deployment_details( self, **kwargs @@ -1548,10 +1548,9 @@ def _update_model_deployment_details( self.infrastructure.CONST_MODEL_DEPLOYMENT_CONFIG_DETAILS: self._build_model_deployment_configuration_details(), self.infrastructure.CONST_CATEGORY_LOG_DETAILS: self._build_category_log_details(), } - - return UpdateModelDeploymentDetails( - **ads_utils.batch_convert_case(update_model_deployment_details, "snake") - ) + return OCIDataScienceModelDeployment( + **update_model_deployment_details + ).to_oci_model(UpdateModelDeploymentDetails) def _update_spec(self, **kwargs) -> "ModelDeployment": """Updates model deployment specs from kwargs. diff --git a/tests/unitary/with_extras/aqua/test_deployment.py b/tests/unitary/with_extras/aqua/test_deployment.py index ec573ffdc..5d71cbc27 100644 --- a/tests/unitary/with_extras/aqua/test_deployment.py +++ b/tests/unitary/with_extras/aqua/test_deployment.py @@ -1793,9 +1793,7 @@ def test_create_deployment_for_tei_byoc_embedding_model( @patch.object(AquaApp, "get_container_image") @patch("ads.model.deployment.model_deployment.ModelDeployment.deploy") @patch("ads.aqua.modeldeployment.AquaDeploymentApp.get_deployment_config") - @patch( - "ads.aqua.modeldeployment.AquaDeploymentApp._create_model_group_custom_metadata" - ) + @patch("ads.aqua.modeldeployment.AquaDeploymentApp._build_model_group_configs") @patch( "ads.aqua.modeldeployment.entities.CreateModelDeploymentDetails.validate_multimodel_deployment_feasibility" ) @@ -1808,7 +1806,7 @@ def test_create_deployment_for_multi_model( mock_get_multi_source, mock_validate_input_models, mock_validate_multimodel_deployment_feasibility, - mock_create_model_group_custom_metadata, + mock_build_model_group_configs, mock_get_deployment_config, mock_deploy, mock_get_container_image, @@ -1816,7 +1814,13 @@ def test_create_deployment_for_multi_model( mock_get_container_config, ): """Test to create a deployment for multi models.""" - mock_create_model_group_custom_metadata.return_value = MagicMock() + mock_build_model_group_configs.return_value = ( + "mock_group_name", + "mock_group_description", + {}, + MagicMock(), + "mock_combined_models", + ) mock_get_container_config.return_value = ( AquaContainerConfig.from_service_config( service_containers=TestDataset.CONTAINER_LIST diff --git a/tests/unitary/with_extras/aqua/test_model.py b/tests/unitary/with_extras/aqua/test_model.py index 42d9e8da0..5835529b4 100644 --- a/tests/unitary/with_extras/aqua/test_model.py +++ b/tests/unitary/with_extras/aqua/test_model.py @@ -526,6 +526,10 @@ def test_create_multimodel( model_group = self.app.create_multi( models=[model_info_1, model_info_2, model_info_3], model_custom_metadata=model_custom_metadata, + model_group_display_name="test_model_group_name", + model_group_description="test_model_group_description", + tags={"aqua_multimodel": "true"}, + combined_models="test_combined_models", project_id="test_project_id", compartment_id="test_compartment_id", source_models=model_details, From 22baaad730c32576cae79ed5e7bc82525e738de7 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Mon, 7 Jul 2025 15:35:20 -0400 Subject: [PATCH 7/8] Updated pr. --- ads/aqua/model/entities.py | 15 ----------- ads/aqua/model/model.py | 16 +++++------ ads/aqua/modeldeployment/deployment.py | 28 +++++++++++--------- ads/aqua/modeldeployment/entities.py | 20 ++++++++++++-- ads/model/deployment/model_deployment.py | 9 +++++-- tests/unitary/with_extras/aqua/test_model.py | 2 +- 6 files changed, 50 insertions(+), 40 deletions(-) diff --git a/ads/aqua/model/entities.py b/ads/aqua/model/entities.py index 72c374f74..0bbcdfb0b 100644 --- a/ads/aqua/model/entities.py +++ b/ads/aqua/model/entities.py @@ -383,18 +383,3 @@ class ModelFileDescription(Serializable): class Config: alias_generator = to_camel extra = "allow" - - -class MemberModel(Serializable): - """Describes the member model of a model group. - - Attributes: - model_id (str): The id of member model. - inference_key (str): The inference key of member model. - """ - - model_id: str = Field(..., description="The id of member model.") - inference_key: str = Field(None, description="The inference key of member model.") - - class Config: - extra = "allow" diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index 729f43622..f7204fd72 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -76,7 +76,6 @@ AquaModelReadme, AquaModelSummary, ImportModelDetails, - MemberModel, ModelValidationResult, ) from ads.common.auth import default_signer @@ -235,7 +234,7 @@ def create_multi( model_group_display_name: str, model_group_description: str, tags: Dict, - combined_models: str, + combined_model_names: str, project_id: Optional[str] = None, compartment_id: Optional[str] = None, defined_tags: Optional[Dict] = None, @@ -250,13 +249,15 @@ def create_multi( List of AquaMultiModelRef instances for creating a multi-model group. model_custom_metadata : ModelCustomMetadata Custom metadata for creating model group. + All model group custom metadata, including 'multi_model_metadata' and 'MULTI_MODEL_CONFIG' will be translated as a + list of dict and placed under environment variable 'OCI_MODEL_GROUP_CUSTOM_METADATA' in model deployment. model_group_display_name: str The model group display name. model_group_description: str The model group description. tags: Dict The tags of model group. - combined_models: str + combined_model_names: str The name of models to be grouped and deployed. project_id : Optional[str] The project ID for the multi-model group. @@ -279,21 +280,20 @@ def create_multi( .with_freeform_tags(**tags) .with_defined_tags(**(defined_tags or {})) .with_custom_metadata_list(model_custom_metadata) - .with_member_models( - [MemberModel(model_id=model.model_id).model_dump() for model in models] - ) + # TODO: add member model inference key + .with_member_models([{"model_id": model.model_id for model in models}]) ) custom_model_group.create() logger.info( - f"Aqua Model Group'{custom_model_group.id}' created with models: {combined_models}." + f"Aqua Model Group'{custom_model_group.id}' created with models: {combined_model_names}." ) # Track telemetry event self.telemetry.record_event_async( category="aqua/multimodel", action="create", - detail=combined_models, + detail=combined_model_names, ) return custom_model_group diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index 8d0a12871..fbe069b52 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -334,7 +334,7 @@ def create( model_group_description, tags, model_custom_metadata, - combined_models, + combined_model_names, ) = self._build_model_group_configs( models=create_deployment_details.models, create_deployment_details=create_deployment_details, @@ -349,7 +349,7 @@ def create( model_group_display_name=model_group_display_name, model_group_description=model_group_description, tags=tags, - combined_models=combined_models, + combined_model_names=combined_model_names, compartment_id=compartment_id, project_id=project_id, defined_tags=defined_tags, @@ -360,7 +360,6 @@ def create( container_config=container_config, ) - @telemetry(entry_point="plugin=model&action=create", name="aqua") def _build_model_group_configs( self, models: List[AquaMultiModelRef], @@ -392,7 +391,7 @@ def _build_model_group_configs( Returns ------- tuple - A tuple of required metadata and strings to create model group. + A tuple of required metadata ('multi_model_metadata' and 'MULTI_MODEL_CONFIG') and strings to create model group. """ if not models: @@ -583,8 +582,8 @@ def _build_model_group_configs( # Generate model group details timestamp = datetime.now().strftime("%Y%m%d") model_group_display_name = f"model_group_{timestamp}" - combined_models = ", ".join(display_name_list) - model_group_description = f"Multi-model grouping using {combined_models}." + combined_model_names = ", ".join(display_name_list) + model_group_description = f"Multi-model grouping using {combined_model_names}." # Add global metadata model_custom_metadata.add( @@ -605,7 +604,7 @@ def _build_model_group_configs( create_deployment_details=create_deployment_details, model_config_summary=model_config_summary, deployment_container=deployment_container, - ), + ).model_dump_json(), description="Configs required to deploy multi models.", category="Other", ) @@ -629,7 +628,7 @@ def _build_model_group_configs( model_group_description, tags, model_custom_metadata, - combined_models, + combined_model_names, ) def _extract_model_task( @@ -657,7 +656,7 @@ def _build_model_group_config( create_deployment_details, model_config_summary, deployment_container: str, - ) -> str: + ) -> ModelGroupConfig: """Builds model group config required to deploy multi models.""" container_type_key = ( create_deployment_details.container_family or deployment_container @@ -674,7 +673,7 @@ def _build_model_group_config( container_params, ) - return multi_model_config.model_dump_json() + return multi_model_config def _create( self, @@ -1059,7 +1058,7 @@ def _create_deployment( .with_overwrite_existing_artifact(True) .with_remove_existing_artifact(True) ) - if "datasciencemodelgroup" in aqua_model_id: + if self._if_model_group(aqua_model_id): container_runtime.with_model_group_id(aqua_model_id) else: container_runtime.with_model_uri(aqua_model_id) @@ -1299,7 +1298,7 @@ def get(self, model_deployment_id: str, **kwargs) -> "AquaDeploymentDetail": f"Make sure the {Tags.AQUA_MODEL_ID_TAG} tag is added to the deployment." ) - if "datasciencemodelgroup" in aqua_model_id: + if self._if_model_group(aqua_model_id): aqua_model = DataScienceModelGroup.from_id(aqua_model_id) else: aqua_model = DataScienceModel.from_id(aqua_model_id) @@ -1334,6 +1333,11 @@ def get(self, model_deployment_id: str, **kwargs) -> "AquaDeploymentDetail": log=AquaResourceIdentifier(log_id, log_name, log_url), ) + @staticmethod + def _if_model_group(model_id: str) -> bool: + """Checks if it's model group id or not.""" + return "datasciencemodelgroup" in model_id.lower() + @telemetry( entry_point="plugin=deployment&action=get_deployment_config", name="aqua" ) diff --git a/ads/aqua/modeldeployment/entities.py b/ads/aqua/modeldeployment/entities.py index fde05e666..0b65bc213 100644 --- a/ads/aqua/modeldeployment/entities.py +++ b/ads/aqua/modeldeployment/entities.py @@ -10,6 +10,7 @@ from ads.aqua import logger from ads.aqua.common.entities import AquaMultiModelRef from ads.aqua.common.enums import Tags +from ads.aqua.common.errors import AquaValueError from ads.aqua.config.utils.serializer import Serializable from ads.aqua.constants import UNKNOWN_DICT from ads.aqua.data import AquaResourceIdentifier @@ -21,6 +22,7 @@ from ads.common.serializer import DataClassSerializable from ads.common.utils import UNKNOWN, get_console_link from ads.model.datascience_model import DataScienceModel +from ads.model.deployment.model_deployment import ModelDeploymentType from ads.model.model_metadata import ModelCustomMetadataItem @@ -150,14 +152,28 @@ def from_oci_model_deployment( model_deployment_configuration_details = ( oci_model_deployment.model_deployment_configuration_details ) - if model_deployment_configuration_details.deployment_type == "SINGLE_MODEL": + if ( + model_deployment_configuration_details.deployment_type + == ModelDeploymentType.SINGLE_MODEL + ): instance_configuration = model_deployment_configuration_details.model_configuration_details.instance_configuration instance_count = model_deployment_configuration_details.model_configuration_details.scaling_policy.instance_count model_id = model_deployment_configuration_details.model_configuration_details.model_id - else: + elif ( + model_deployment_configuration_details.deployment_type + == ModelDeploymentType.MODEL_GROUP + ): instance_configuration = model_deployment_configuration_details.infrastructure_configuration_details.instance_configuration instance_count = model_deployment_configuration_details.infrastructure_configuration_details.scaling_policy.instance_count model_id = model_deployment_configuration_details.model_group_configuration_details.model_group_id + else: + allowed_deployment_types = ", ".join( + [key for key in dir(ModelDeploymentType) if not key.startswith("__")] + ) + raise AquaValueError( + f"Invalid AQUA deployment with type {model_deployment_configuration_details.deployment_type}." + f"Only {allowed_deployment_types} are supported at this moment. Specify a different AQUA model deployment." + ) instance_shape_config_details = ( instance_configuration.model_deployment_instance_shape_config_details diff --git a/ads/model/deployment/model_deployment.py b/ads/model/deployment/model_deployment.py index 4bbd4b7ab..57a4c683b 100644 --- a/ads/model/deployment/model_deployment.py +++ b/ads/model/deployment/model_deployment.py @@ -81,6 +81,11 @@ class ModelDeploymentLogType: ACCESS = "access" +class ModelDeploymentType: + SINGLE_MODEL = "SINGLE_MODEL" + MODEL_GROUP = "MODEL_GROUP" + + class LogNotConfiguredError(Exception): # pragma: no cover pass @@ -1750,7 +1755,7 @@ def _build_model_deployment_configuration_details(self) -> Dict: ) model_deployment_configuration_details = { - infrastructure.CONST_DEPLOYMENT_TYPE: "SINGLE_MODEL", + infrastructure.CONST_DEPLOYMENT_TYPE: ModelDeploymentType.SINGLE_MODEL, infrastructure.CONST_MODEL_CONFIG_DETAILS: model_configuration_details, runtime.CONST_ENVIRONMENT_CONFIG_DETAILS: environment_configuration_details, } @@ -1758,7 +1763,7 @@ def _build_model_deployment_configuration_details(self) -> Dict: if runtime.model_group_id: model_deployment_configuration_details[ infrastructure.CONST_DEPLOYMENT_TYPE - ] = "MODEL_GROUP" + ] = ModelDeploymentType.MODEL_GROUP model_deployment_configuration_details["modelGroupConfigurationDetails"] = { runtime.CONST_MODEL_GROUP_ID: runtime.model_group_id } diff --git a/tests/unitary/with_extras/aqua/test_model.py b/tests/unitary/with_extras/aqua/test_model.py index 5835529b4..878f75a9c 100644 --- a/tests/unitary/with_extras/aqua/test_model.py +++ b/tests/unitary/with_extras/aqua/test_model.py @@ -529,7 +529,7 @@ def test_create_multimodel( model_group_display_name="test_model_group_name", model_group_description="test_model_group_description", tags={"aqua_multimodel": "true"}, - combined_models="test_combined_models", + combined_model_names="test_combined_models", project_id="test_project_id", compartment_id="test_compartment_id", source_models=model_details, From 67ff0a51caaae83052529c12c8f44938738a5ae3 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Mon, 7 Jul 2025 16:16:41 -0400 Subject: [PATCH 8/8] Fixed unit test. --- .../default_setup/model_deployment/test_model_deployment_v2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py index f86f1816a..2a6b19c66 100644 --- a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py +++ b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py @@ -370,6 +370,7 @@ def test__load_default_properties(self, mock_from_ocid): ModelDeploymentInfrastructure.CONST_SHAPE_NAME: infrastructure.shape_name, ModelDeploymentInfrastructure.CONST_BANDWIDTH_MBPS: 10, ModelDeploymentInfrastructure.CONST_SHAPE_CONFIG_DETAILS: { + "cpu_baseline": None, "ocpus": 10.0, "memory_in_gbs": 36.0, },