Merge dev to test (#107) (#108)

* Create bounds for FineTune hyperparameters (#103)

* Test bound to hyperparameters

* Update finetune llm hyperparameters

* Remove option to use PEFT, always on use now

* Fixing pipeline general asset test (#106)

---------

Co-authored-by: Lucas Pavanelli <[email protected]>
Co-authored-by: Thiago Castro Ferreira <[email protected]>

Author: 3 people

aixplain/factories/finetune_factory/__init__.py

@@ -28,7 +28,6 @@
 from aixplain.modules.finetune import Finetune
 from aixplain.modules.finetune.cost import FinetuneCost
 from aixplain.modules.finetune.hyperparameters import Hyperparameters
-from aixplain.modules.finetune.peft import Peft
 from aixplain.modules.dataset import Dataset
 from aixplain.modules.model import Model
 from aixplain.utils import config
@@ -66,7 +65,6 @@ def create(
         model: Model,
         prompt_template: Optional[Text] = None,
         hyperparameters: Optional[Hyperparameters] = None,
-        peft: Optional[Peft] = None,
         train_percentage: Optional[float] = 100,
         dev_percentage: Optional[float] = 0,
     ) -> Finetune:
@@ -78,7 +76,6 @@ def create(
             model (Model): Model to be fine-tuned.
             prompt_template (Text, optional): Fine-tuning prompt_template. Should reference columns in the dataset using format <<COLUMN_NAME>>. Defaults to None.
             hyperparameters (Hyperparameters, optional): Hyperparameters for fine-tuning. Defaults to None.
-            peft (Peft, optional): PEFT (Parameter-Efficient Fine-Tuning) configuration. Defaults to None.
             train_percentage (float, optional): Percentage of training samples. Defaults to 100.
             dev_percentage (float, optional): Percentage of development samples. Defaults to 0.
         Returns:
@@ -106,8 +103,6 @@ def create(
                 parameters["prompt"] = prompt_template
             if hyperparameters is not None:
                 parameters["hyperparameters"] = hyperparameters.to_dict()
-            if peft is not None:
-                parameters["peft"] = peft.to_dict()
             payload["parameters"] = parameters
             logging.info(f"Start service for POST Create FineTune - {url} - {headers} - {json.dumps(payload)}")
             r = _request_with_retry("post", url, headers=headers, json=payload)
@@ -123,7 +118,6 @@ def create(
                 dev_percentage=dev_percentage,
                 prompt_template=prompt_template,
                 hyperparameters=hyperparameters,
-                peft=peft,
             )
         except Exception:
             error_message = f"Create FineTune: Error with payload {json.dumps(payload)}"

aixplain/modules/finetune/__init__.py

@@ -26,7 +26,6 @@
 from urllib.parse import urljoin
 from aixplain.modules.finetune.cost import FinetuneCost
 from aixplain.modules.finetune.hyperparameters import Hyperparameters
-from aixplain.modules.finetune.peft import Peft
 from aixplain.factories.model_factory import ModelFactory
 from aixplain.modules.asset import Asset
 from aixplain.modules.dataset import Dataset
@@ -52,7 +51,6 @@ class Finetune(Asset):
         dev_percentage (float): Percentage of development samples.
         prompt_template (Text): Fine-tuning prompt_template.
         hyperparameters (Hyperparameters): Hyperparameters for fine-tuning.
-        peft (Peft): PEFT (Parameter-Efficient Fine-Tuning) configuration.
         additional_info (dict): Additional information to be saved with the FineTune.
         backend_url (str): URL of the backend.
         api_key (str): The TEAM API key used for authentication.
@@ -72,7 +70,6 @@ def __init__(
         dev_percentage: Optional[float] = 0,
         prompt_template: Optional[Text] = None,
         hyperparameters: Optional[Hyperparameters] = None,
-        peft: Optional[Peft] = None,
         **additional_info,
     ) -> None:
         """Create a FineTune with the necessary information.
@@ -90,7 +87,6 @@ def __init__(
             dev_percentage (float, optional): Percentage of development samples. Defaults to 0.
             prompt_template (Text, optional): Fine-tuning prompt_template. Should reference columns in the dataset using format <<COLUMN_NAME>>. Defaults to None.
             hyperparameters (Hyperparameters, optional): Hyperparameters for fine-tuning. Defaults to None.
-            peft (Peft, optional): PEFT (Parameter-Efficient Fine-Tuning) configuration. Defaults to None.
             **additional_info: Additional information to be saved with the FineTune.
         """
         super().__init__(id, name, description, supplier, version)
@@ -101,7 +97,6 @@ def __init__(
         self.dev_percentage = dev_percentage
         self.prompt_template = prompt_template
         self.hyperparameters = hyperparameters
-        self.peft = peft
         self.additional_info = additional_info
         self.backend_url = config.BACKEND_URL
         self.api_key = config.TEAM_API_KEY
@@ -134,8 +129,6 @@ def start(self) -> Model:
                 parameters["prompt"] = self.prompt_template
             if self.hyperparameters is not None:
                 parameters["hyperparameters"] = self.hyperparameters.to_dict()
-            if self.peft is not None:
-                parameters["peft"] = self.peft.to_dict()
             payload["parameters"] = parameters
             logging.info(f"Start service for POST Start FineTune - {url} - {headers} - {json.dumps(payload)}")
             r = _request_with_retry("post", url, headers=headers, json=payload)

aixplain/modules/finetune/hyperparameters.py

@@ -1,8 +1,10 @@
 from dataclasses import dataclass
 from dataclasses_json import dataclass_json
+from enum import Enum
+from typing import Text
 
 
-class SchedulerType:
+class SchedulerType(Text, Enum):
     LINEAR = "linear"
     COSINE = "cosine"
     COSINE_WITH_RESTARTS = "cosine_with_restarts"
@@ -13,19 +15,49 @@ class SchedulerType:
     REDUCE_ON_PLATEAU = "reduce_lr_on_plateau"
 
 
+EPOCHS_MAX_VALUE = 4
+MAX_SEQ_LENGTH_MAX_VALUE = 4096
+GENERATION_MAX_LENGTH_MAX_VALUE = 225
+
+
 @dataclass_json
 @dataclass
 class Hyperparameters(object):
-    epochs: int = 4
-    train_batch_size: int = 4
-    eval_batch_size: int = 4
-    learning_rate: float = 2e-5
+    epochs: int = 1
+    learning_rate: float = 1e-5
     generation_max_length: int = 225
-    tokenizer_batch_size: int = 256
-    gradient_checkpointing: bool = False
-    gradient_accumulation_steps: int = 1
     max_seq_length: int = 4096
     warmup_ratio: float = 0.0
     warmup_steps: int = 0
-    early_stopping_patience: int = 1
     lr_scheduler_type: SchedulerType = SchedulerType.LINEAR
+
+    def __post_init__(self):
+        if not isinstance(self.epochs, int):
+            raise TypeError("epochs should be of type int")
+
+        if not isinstance(self.learning_rate, float):
+            raise TypeError("learning_rate should be of type float")
+
+        if not isinstance(self.generation_max_length, int):
+            raise TypeError("generation_max_length should be of type int")
+
+        if not isinstance(self.max_seq_length, int):
+            raise TypeError("max_seq_length should be of type int")
+
+        if not isinstance(self.warmup_ratio, float):
+            raise TypeError("warmup_ratio should be of type float")
+
+        if not isinstance(self.warmup_steps, int):
+            raise TypeError("warmup_steps should be of type int")
+
+        if not isinstance(self.lr_scheduler_type, SchedulerType):
+            raise TypeError("lr_scheduler_type should be of type SchedulerType")
+
+        if self.epochs > EPOCHS_MAX_VALUE:
+            raise ValueError(f"epochs must be one less than {EPOCHS_MAX_VALUE}")
+
+        if self.max_seq_length > MAX_SEQ_LENGTH_MAX_VALUE:
+            raise ValueError(f"max_seq_length must be less than {MAX_SEQ_LENGTH_MAX_VALUE}")
+
+        if self.generation_max_length > GENERATION_MAX_LENGTH_MAX_VALUE:
+            raise ValueError(f"generation_max_length must be less than {GENERATION_MAX_LENGTH_MAX_VALUE}")

aixplain/modules/finetune/peft.py

@@ -1,11 +1,5 @@
 [
     {"model_name": "gpt2", "model_id": "64e615671567f848804985e1", "dataset_name": "Test text generation dataset"},
-    {"model_name": "falcon 7b instruct", "model_id": "65519d57bf42e6037ab109d5", "dataset_name": "Test text generation dataset"},
-    {"model_name": "bloomz 7b", "model_id": "6551ab17bf42e6037ab109e0", "dataset_name": "Test text generation dataset"},
-    {"model_name": "MPT 7B", "model_id": "6551a72bbf42e6037ab109d9", "dataset_name": "Test text generation dataset"},
-    {"model_name": "falcon 7b", "model_id": "6551bff9bf42e6037ab109e1", "dataset_name": "Test text generation dataset"},
-    {"model_name": "mistral 7b", "model_id": "6551a9e7bf42e6037ab109de", "dataset_name": "Test text generation dataset"},
-    {"model_name": "MPT 7B Storywriter", "model_id": "6551a870bf42e6037ab109db", "dataset_name": "Test text generation dataset"},
     {"model_name": "llama 2 7b", "model_id": "6543cb991f695e72028e9428", "dataset_name": "Test text generation dataset"},
     {"model_name": "Llama 2 7B Chat", "model_id": "65519ee7bf42e6037ab109d8", "dataset_name": "Test text generation dataset"}
 ]

tests/functional/finetune/data/finetune_test_cost_estimation.json

@@ -1,7 +1,7 @@
 [
     {
-        "model_name": "gpt2",
-        "model_id": "64e615671567f848804985e1",
+        "model_name": "llama2 7b", 
+        "model_id": "6543cb991f695e72028e9428",
         "dataset_name": "Test text generation dataset",
         "inference_data": "Hello!",
         "required_dev": true

tests/functional/finetune/data/finetune_test_end2end.json

@@ -40,7 +40,10 @@ def test_list(asset_name):
 def test_run(inputs, asset_name):
     asset_details = inputs[asset_name]
     AssetFactory = __get_asset_factory(asset_name)
-    asset = AssetFactory.get(asset_details["id"])
+    if asset_name == "pipeline":
+        asset = AssetFactory.list(query=asset_details["name"])["results"][0]
+    else:
+        asset = AssetFactory.get(asset_details["id"])
     payload = asset_details["data"]
     if type(payload) is dict:
         output = asset.run(**payload)

tests/functional/general_assets/asset_functional_test.py

@@ -4,7 +4,7 @@
         "data": "This is a test sentence."
     },
     "pipeline": {
-        "id" : "64da138fa27cffd5e0c3c30d",
+        "name": "SingleNodePipeline",
         "data": "This is a test sentence."
     },
     "metric": {