feat(clusters): convert cluster groups and names to lower case (#2182)

Implementation note:
groups and cluster names are converted to lower case as soon
as possible when at the borders of the application (using LowerCaseStr):
in the API, in the properties classes read from files, in the command
data from database ...
This way, we don't have to bother about it inside the application
logic.

[ANT-2311]

antarest/core/model.py

@@ -13,6 +13,9 @@
 import enum
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
+import typing_extensions as te
+from pydantic import StringConstraints
+
 from antarest.core.serialization import AntaresBaseModel
 
 if TYPE_CHECKING:
@@ -22,6 +25,7 @@
 JSON = Dict[str, Any]
 ELEMENT = Union[str, int, float, bool, bytes]
 SUB_JSON = Union[ELEMENT, JSON, List[Any], None]
+LowerCaseStr = te.Annotated[str, StringConstraints(to_lower=True)]
 
 
 class PublicMode(enum.StrEnum):

antarest/launcher/extensions/adequacy_patch/extension.py

@@ -23,7 +23,7 @@
 from antarest.core.utils.utils import assert_this
 from antarest.launcher.extensions.interface import ILauncherExtension
 from antarest.study.service import StudyService
-from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 
 logger = logging.getLogger(__name__)

antarest/study/business/area_management.py

@@ -32,7 +32,8 @@
     ThermalAreasProperties,
     UIProperties,
 )
-from antarest.study.storage.rawstudy.model.filesystem.config.model import Area, DistrictSet, transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import Area, DistrictSet
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 from antarest.study.storage.storage_service import StudyStorageService
 from antarest.study.storage.variantstudy.model.command.create_area import CreateArea

antarest/study/business/areas/renewable_management.py

@@ -22,12 +22,13 @@
 from antarest.study.business.enum_ignore_case import EnumIgnoreCase
 from antarest.study.business.utils import execute_or_add_commands
 from antarest.study.model import Study
-from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.config.renewable import (
     RenewableConfig,
     RenewableConfigType,
     RenewableProperties,
     create_renewable_config,
+    create_renewable_properties,
 )
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 from antarest.study.storage.storage_service import StudyStorageService
@@ -274,7 +275,6 @@ def update_cluster(
         Raises:
             RenewableClusterNotFound: If the cluster to update is not found.
         """
-
         study_version = StudyVersion.parse(study.version)
         file_study = self._get_file_study(study)
         path = _CLUSTER_PATH.format(area_id=area_id, cluster_id=cluster_id)
@@ -284,19 +284,19 @@ def update_cluster(
         except KeyError:
             raise RenewableClusterNotFound(path, cluster_id) from None
         else:
-            old_config = create_renewable_config(study_version, **values)
+            old_properties = create_renewable_properties(study_version, **values)
 
         # use Python values to synchronize Config and Form values
         new_values = cluster_data.model_dump(by_alias=False, exclude_none=True)
-        new_config = old_config.copy(exclude={"id"}, update=new_values)
-        new_data = new_config.model_dump(mode="json", by_alias=True, exclude={"id"})
+        new_properties = old_properties.copy(exclude={"id"}, update=new_values)
 
         # create the dict containing the new values using aliases
         data: t.Dict[str, t.Any] = {}
-        for field_name, field in new_config.model_fields.items():
-            if field_name in new_values:
-                name = field.alias if field.alias else field_name
-                data[name] = new_data[name]
+        for updated_field, updated_value in new_values.items():
+            if updated_field in old_properties.model_fields:
+                field_info = old_properties.model_fields[updated_field]
+                field_name = field_info.alias if field_info.alias else updated_field
+                data[field_name] = updated_value
 
         # create the update config commands with the modified data
         command_context = self.storage_service.variant_study_service.command_factory.command_context
@@ -308,7 +308,7 @@ def update_cluster(
         ]
         execute_or_add_commands(study, file_study, commands, self.storage_service)
 
-        values = new_config.model_dump(by_alias=False)
+        values = new_properties.model_dump(by_alias=False)
         return RenewableClusterOutput(**values, id=cluster_id)
 
     def delete_clusters(self, study: Study, area_id: str, cluster_ids: t.Sequence[str]) -> None:
@@ -357,9 +357,8 @@ def duplicate_cluster(
         Raises:
             DuplicateRenewableCluster: If a cluster with the new name already exists in the area.
         """
-        new_id = transform_name_to_id(new_cluster_name, lower=False)
-        lower_new_id = new_id.lower()
-        if any(lower_new_id == cluster.id.lower() for cluster in self.get_clusters(study, area_id)):
+        new_id = transform_name_to_id(new_cluster_name)
+        if any(new_id == cluster.id for cluster in self.get_clusters(study, area_id)):
             raise DuplicateRenewableCluster(area_id, new_id)
 
         # Cluster duplication
@@ -371,9 +370,8 @@ def duplicate_cluster(
         create_cluster_cmd = self._make_create_cluster_cmd(area_id, new_config, study_version)
 
         # Matrix edition
-        lower_source_id = source_id.lower()
-        source_path = f"input/renewables/series/{area_id}/{lower_source_id}/series"
-        new_path = f"input/renewables/series/{area_id}/{lower_new_id}/series"
+        source_path = f"input/renewables/series/{area_id}/{source_id}/series"
+        new_path = f"input/renewables/series/{area_id}/{new_id}/series"
 
         # Prepare and execute commands
         storage_service = self.storage_service.get_storage(study)

antarest/study/business/areas/st_storage_management.py

@@ -33,7 +33,7 @@
 from antarest.study.business.all_optional_meta import all_optional_model, camel_case_model
 from antarest.study.business.utils import execute_or_add_commands
 from antarest.study.model import STUDY_VERSION_8_8, Study
-from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.config.st_storage import (
     STStorage880Config,
     STStorage880Properties,
@@ -305,7 +305,7 @@ def _make_create_cluster_cmd(
     ) -> CreateSTStorage:
         command = CreateSTStorage(
             area_id=area_id,
-            parameters=cluster,
+            parameters=cluster.model_dump(mode="json", by_alias=True, exclude={"id"}),
             command_context=self.storage_service.variant_study_service.command_factory.command_context,
             study_version=study_version,
         )
@@ -551,8 +551,7 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_clus
             ClusterAlreadyExists: If a cluster with the new name already exists in the area.
         """
         new_id = transform_name_to_id(new_cluster_name)
-        lower_new_id = new_id.lower()
-        if any(lower_new_id == storage.id.lower() for storage in self.get_storages(study, area_id)):
+        if any(new_id == storage.id for storage in self.get_storages(study, area_id)):
             raise DuplicateSTStorage(area_id, new_id)
 
         # Cluster duplication
@@ -571,16 +570,13 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_clus
         create_cluster_cmd = self._make_create_cluster_cmd(area_id, new_config, study_version)
 
         # Matrix edition
-        lower_source_id = source_id.lower()
         # noinspection SpellCheckingInspection
         ts_names = ["pmax_injection", "pmax_withdrawal", "lower_rule_curve", "upper_rule_curve", "inflows"]
         source_paths = [
-            _STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=lower_source_id, ts_name=ts_name)
-            for ts_name in ts_names
+            _STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=source_id, ts_name=ts_name) for ts_name in ts_names
         ]
         new_paths = [
-            _STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=lower_new_id, ts_name=ts_name)
-            for ts_name in ts_names
+            _STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=new_id, ts_name=ts_name) for ts_name in ts_names
         ]
 
         # Prepare and execute commands

antarest/study/business/areas/thermal_management.py

@@ -28,12 +28,13 @@
 from antarest.study.business.all_optional_meta import all_optional_model, camel_case_model
 from antarest.study.business.utils import execute_or_add_commands
 from antarest.study.model import STUDY_VERSION_8_7, Study
-from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.config.thermal import (
     Thermal870Config,
     Thermal870Properties,
     ThermalConfigType,
     create_thermal_config,
+    create_thermal_properties,
 )
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 from antarest.study.storage.storage_service import StudyStorageService
@@ -348,7 +349,6 @@ def update_cluster(
             ThermalClusterNotFound: If the provided `cluster_id` does not match the ID of the cluster
             in the provided cluster_data.
         """
-
         study_version = StudyVersion.parse(study.version)
         file_study = self._get_file_study(study)
         path = _CLUSTER_PATH.format(area_id=area_id, cluster_id=cluster_id)
@@ -357,19 +357,19 @@ def update_cluster(
         except KeyError:
             raise ThermalClusterNotFound(path, cluster_id) from None
         else:
-            old_config = create_thermal_config(study_version, **values)
+            old_properties = create_thermal_properties(study_version, **values)
 
         # Use Python values to synchronize Config and Form values
         new_values = cluster_data.model_dump(mode="json", by_alias=False, exclude_none=True)
-        new_config = old_config.copy(exclude={"id"}, update=new_values)
-        new_data = new_config.model_dump(mode="json", by_alias=True, exclude={"id"})
+        new_properties = old_properties.copy(exclude={"id"}, update=new_values)
 
         # create the dict containing the new values using aliases
         data: t.Dict[str, t.Any] = {}
-        for field_name, field in new_config.model_fields.items():
-            if field_name in new_values:
-                name = field.alias if field.alias else field_name
-                data[name] = new_data[name]
+        for updated_field, updated_value in new_values.items():
+            if updated_field in old_properties.model_fields:
+                field_info = old_properties.model_fields[updated_field]
+                field_name = field_info.alias if field_info.alias else updated_field
+                data[field_name] = updated_value
 
         # create the update config commands with the modified data
         command_context = self.storage_service.variant_study_service.command_factory.command_context
@@ -381,7 +381,7 @@ def update_cluster(
         ]
         execute_or_add_commands(study, file_study, commands, self.storage_service)
 
-        values = {**new_config.model_dump(mode="json", by_alias=False), "id": cluster_id}
+        values = {**new_properties.model_dump(mode="json", by_alias=False), "id": cluster_id}
         return ThermalClusterOutput.model_validate(values)
 
     def delete_clusters(self, study: Study, area_id: str, cluster_ids: t.Sequence[str]) -> None:
@@ -431,9 +431,8 @@ def duplicate_cluster(
         Raises:
             ClusterAlreadyExists: If a cluster with the new name already exists in the area.
         """
-        new_id = transform_name_to_id(new_cluster_name, lower=False)
-        lower_new_id = new_id.lower()
-        if any(lower_new_id == cluster.id.lower() for cluster in self.get_clusters(study, area_id)):
+        new_id = transform_name_to_id(new_cluster_name)
+        if any(new_id == cluster.id for cluster in self.get_clusters(study, area_id)):
             raise DuplicateThermalCluster(area_id, new_id)
 
         # Cluster duplication
@@ -445,23 +444,22 @@ def duplicate_cluster(
         create_cluster_cmd = self._make_create_cluster_cmd(area_id, new_config, study_version)
 
         # Matrix edition
-        lower_source_id = source_id.lower()
         source_paths = [
-            f"input/thermal/series/{area_id}/{lower_source_id}/series",
-            f"input/thermal/prepro/{area_id}/{lower_source_id}/modulation",
-            f"input/thermal/prepro/{area_id}/{lower_source_id}/data",
+            f"input/thermal/series/{area_id}/{source_id}/series",
+            f"input/thermal/prepro/{area_id}/{source_id}/modulation",
+            f"input/thermal/prepro/{area_id}/{source_id}/data",
         ]
         new_paths = [
-            f"input/thermal/series/{area_id}/{lower_new_id}/series",
-            f"input/thermal/prepro/{area_id}/{lower_new_id}/modulation",
-            f"input/thermal/prepro/{area_id}/{lower_new_id}/data",
+            f"input/thermal/series/{area_id}/{new_id}/series",
+            f"input/thermal/prepro/{area_id}/{new_id}/modulation",
+            f"input/thermal/prepro/{area_id}/{new_id}/data",
         ]
         study_version = StudyVersion.parse(study.version)
         if study_version >= STUDY_VERSION_8_7:
-            source_paths.append(f"input/thermal/series/{area_id}/{lower_source_id}/CO2Cost")
-            source_paths.append(f"input/thermal/series/{area_id}/{lower_source_id}/fuelCost")
-            new_paths.append(f"input/thermal/series/{area_id}/{lower_new_id}/CO2Cost")
-            new_paths.append(f"input/thermal/series/{area_id}/{lower_new_id}/fuelCost")
+            source_paths.append(f"input/thermal/series/{area_id}/{source_id}/CO2Cost")
+            source_paths.append(f"input/thermal/series/{area_id}/{source_id}/fuelCost")
+            new_paths.append(f"input/thermal/series/{area_id}/{new_id}/CO2Cost")
+            new_paths.append(f"input/thermal/series/{area_id}/{new_id}/fuelCost")
 
         # Prepare and execute commands
         commands: t.List[t.Union[CreateCluster, ReplaceMatrix]] = [create_cluster_cmd]
@@ -479,8 +477,7 @@ def duplicate_cluster(
         return ThermalClusterOutput(**new_config.model_dump(mode="json", by_alias=False))
 
     def validate_series(self, study: Study, area_id: str, cluster_id: str) -> bool:
-        lower_cluster_id = cluster_id.lower()
-        thermal_cluster_path = Path(f"input/thermal/series/{area_id}/{lower_cluster_id}")
+        thermal_cluster_path = Path(f"input/thermal/series/{area_id}/{cluster_id.lower()}")
         series_path = [thermal_cluster_path / "series"]
         if StudyVersion.parse(study.version) >= STUDY_VERSION_8_7:
             series_path.append(thermal_cluster_path / "CO2Cost")

antarest/study/business/binding_constraint_management.py

@@ -30,7 +30,7 @@
     MatrixWidthMismatchError,
     WrongMatrixHeightError,
 )
-from antarest.core.model import JSON
+from antarest.core.model import JSON, LowerCaseStr
 from antarest.core.requests import CaseInsensitiveDict
 from antarest.core.serialization import AntaresBaseModel
 from antarest.core.utils.string import to_camel_case
@@ -44,7 +44,7 @@
     BindingConstraintFrequency,
     BindingConstraintOperator,
 )
-from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 from antarest.study.storage.storage_service import StudyStorageService
 from antarest.study.storage.variantstudy.business.matrix_constants.binding_constraint.series_after_v87 import (
@@ -340,7 +340,7 @@ class ConstraintOutput830(ConstraintOutputBase):
 
 
 class ConstraintOutput870(ConstraintOutput830):
-    group: str = DEFAULT_GROUP
+    group: LowerCaseStr = DEFAULT_GROUP
 
 
 # WARNING: Do not change the order of the following line, it is used to determine

antarest/study/business/district_manager.py

@@ -16,7 +16,7 @@
 from antarest.core.serialization import AntaresBaseModel
 from antarest.study.business.utils import execute_or_add_commands
 from antarest.study.model import Study
-from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
 from antarest.study.storage.storage_service import StudyStorageService
 from antarest.study.storage.variantstudy.model.command.create_district import CreateDistrict, DistrictBaseFilter
 from antarest.study.storage.variantstudy.model.command.remove_district import RemoveDistrict

antarest/study/business/table_mode_management.py

@@ -217,7 +217,9 @@ def update_table_data(
             thermals_by_areas = collections.defaultdict(dict)
             for key, values in data.items():
                 area_id, cluster_id = key.split(" / ")
-                thermals_by_areas[area_id][cluster_id] = ThermalClusterInput(**values)
+                # Thermal clusters ids were not lowered at the time.
+                # So to ensure this endpoint still works with old scripts we have to lower the id at first.
+                thermals_by_areas[area_id][cluster_id.lower()] = ThermalClusterInput(**values)
             thermals_map = self._thermal_manager.update_thermals_props(study, thermals_by_areas)
             data = {
                 f"{area_id} / {cluster_id}": cluster.model_dump(by_alias=True, exclude={"id", "name"})
@@ -230,7 +232,8 @@ def update_table_data(
             renewables_by_areas = collections.defaultdict(dict)
             for key, values in data.items():
                 area_id, cluster_id = key.split(" / ")
-                renewables_by_areas[area_id][cluster_id] = RenewableClusterInput(**values)
+                # Same reason as for thermal clusters
+                renewables_by_areas[area_id][cluster_id.lower()] = RenewableClusterInput(**values)
             renewables_map = self._renewable_manager.update_renewables_props(study, renewables_by_areas)
             data = {
                 f"{area_id} / {cluster_id}": cluster.model_dump(by_alias=True, exclude={"id", "name"})

antarest/study/storage/rawstudy/model/filesystem/config/cluster.py

@@ -21,6 +21,7 @@
 
 from pydantic import Field
 
+from antarest.core.model import LowerCaseStr
 from antarest.core.serialization import AntaresBaseModel
 
 
@@ -47,9 +48,9 @@ class ItemProperties(
     [('group-A', 'cluster-01'), ('GROUP-A', 'cluster-02'), ('Group-B', 'CLUSTER-01')]
     """
 
-    group: str = Field(default="", description="Cluster group")
+    group: LowerCaseStr = Field(default="", description="Cluster group")
 
-    name: str = Field(description="Cluster name", pattern=r"[a-zA-Z0-9_(),& -]+")
+    name: LowerCaseStr = Field(description="Cluster name", pattern=r"[a-zA-Z0-9_(),& -]+")
 
     def __lt__(self, other: t.Any) -> bool:
         """
@@ -58,7 +59,7 @@ def __lt__(self, other: t.Any) -> bool:
         This method may be used to sort and group clusters by `group` and `name`.
         """
         if isinstance(other, ItemProperties):
-            return (self.group.upper(), self.name.upper()).__lt__((other.group.upper(), other.name.upper()))
+            return (self.group, self.name).__lt__((other.group, other.name))
         return NotImplemented