fix(lowercase): revert lowercase evolution temporarily (#2280)

#2182 introduced a regression on variants that modify clusters
with upper case names, and on cluster modifications on all studies
 that contain clusters with upper case names.

The "filesystem" layer needs also to be updated to handle correctly
cluster IDs.

This PR temporarily reverts the evolution while we are fixing this.

Signed-off-by: Sylvain Leclerc <[email protected]>

antarest/core/model.py

@@ -13,9 +13,6 @@
 import enum
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
-import typing_extensions as te
-from pydantic import StringConstraints
-
 from antarest.core.serialization import AntaresBaseModel
 
 if TYPE_CHECKING:
@@ -25,7 +22,6 @@
 JSON = Dict[str, Any]
 ELEMENT = Union[str, int, float, bool, bytes]
 SUB_JSON = Union[ELEMENT, JSON, List[Any], None]
-LowerCaseStr = te.Annotated[str, StringConstraints(to_lower=True)]
 
 
 class PublicMode(enum.StrEnum):

antarest/launcher/extensions/adequacy_patch/extension.py

@@ -24,7 +24,7 @@
 from antarest.core.utils.utils import assert_this
 from antarest.launcher.extensions.interface import ILauncherExtension
 from antarest.study.service import StudyService
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 
 logger = logging.getLogger(__name__)

antarest/study/business/area_management.py

@@ -32,8 +32,7 @@
     ThermalAreasProperties,
     UIProperties,
 )
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
-from antarest.study.storage.rawstudy.model.filesystem.config.model import Area, DistrictSet
+from antarest.study.storage.rawstudy.model.filesystem.config.model import Area, DistrictSet, transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 from antarest.study.storage.storage_service import StudyStorageService
 from antarest.study.storage.variantstudy.model.command.create_area import CreateArea

antarest/study/business/areas/renewable_management.py

@@ -22,13 +22,12 @@
 from antarest.study.business.enum_ignore_case import EnumIgnoreCase
 from antarest.study.business.utils import execute_or_add_commands
 from antarest.study.model import Study
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.config.renewable import (
     RenewableConfig,
     RenewableConfigType,
     RenewableProperties,
     create_renewable_config,
-    create_renewable_properties,
 )
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 from antarest.study.storage.storage_service import StudyStorageService
@@ -275,6 +274,7 @@ def update_cluster(
         Raises:
             RenewableClusterNotFound: If the cluster to update is not found.
         """
+
         study_version = StudyVersion.parse(study.version)
         file_study = self._get_file_study(study)
         path = _CLUSTER_PATH.format(area_id=area_id, cluster_id=cluster_id)
@@ -284,19 +284,19 @@ def update_cluster(
         except KeyError:
             raise RenewableClusterNotFound(path, cluster_id) from None
         else:
-            old_properties = create_renewable_properties(study_version, **values)
+            old_config = create_renewable_config(study_version, **values)
 
         # use Python values to synchronize Config and Form values
         new_values = cluster_data.model_dump(by_alias=False, exclude_none=True)
-        new_properties = old_properties.copy(exclude={"id"}, update=new_values)
+        new_config = old_config.copy(exclude={"id"}, update=new_values)
+        new_data = new_config.model_dump(mode="json", by_alias=True, exclude={"id"})
 
         # create the dict containing the new values using aliases
         data: t.Dict[str, t.Any] = {}
-        for updated_field, updated_value in new_values.items():
-            if updated_field in old_properties.model_fields:
-                field_info = old_properties.model_fields[updated_field]
-                field_name = field_info.alias if field_info.alias else updated_field
-                data[field_name] = updated_value
+        for field_name, field in new_config.model_fields.items():
+            if field_name in new_values:
+                name = field.alias if field.alias else field_name
+                data[name] = new_data[name]
 
         # create the update config commands with the modified data
         command_context = self.storage_service.variant_study_service.command_factory.command_context
@@ -308,7 +308,7 @@ def update_cluster(
         ]
         execute_or_add_commands(study, file_study, commands, self.storage_service)
 
-        values = new_properties.model_dump(by_alias=False)
+        values = new_config.model_dump(by_alias=False)
         return RenewableClusterOutput(**values, id=cluster_id)
 
     def delete_clusters(self, study: Study, area_id: str, cluster_ids: t.Sequence[str]) -> None:
@@ -357,8 +357,9 @@ def duplicate_cluster(
         Raises:
             DuplicateRenewableCluster: If a cluster with the new name already exists in the area.
         """
-        new_id = transform_name_to_id(new_cluster_name)
-        if any(new_id == cluster.id for cluster in self.get_clusters(study, area_id)):
+        new_id = transform_name_to_id(new_cluster_name, lower=False)
+        lower_new_id = new_id.lower()
+        if any(lower_new_id == cluster.id.lower() for cluster in self.get_clusters(study, area_id)):
             raise DuplicateRenewableCluster(area_id, new_id)
 
         # Cluster duplication
@@ -370,8 +371,9 @@ def duplicate_cluster(
         create_cluster_cmd = self._make_create_cluster_cmd(area_id, new_config, study_version)
 
         # Matrix edition
-        source_path = f"input/renewables/series/{area_id}/{source_id}/series"
-        new_path = f"input/renewables/series/{area_id}/{new_id}/series"
+        lower_source_id = source_id.lower()
+        source_path = f"input/renewables/series/{area_id}/{lower_source_id}/series"
+        new_path = f"input/renewables/series/{area_id}/{lower_new_id}/series"
 
         # Prepare and execute commands
         storage_service = self.storage_service.get_storage(study)

antarest/study/business/areas/st_storage_management.py

@@ -33,7 +33,7 @@
 from antarest.study.business.all_optional_meta import all_optional_model, camel_case_model
 from antarest.study.business.utils import execute_or_add_commands
 from antarest.study.model import STUDY_VERSION_8_8, Study
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.config.st_storage import (
     STStorage880Config,
     STStorage880Properties,
@@ -305,7 +305,7 @@ def _make_create_cluster_cmd(
     ) -> CreateSTStorage:
         command = CreateSTStorage(
             area_id=area_id,
-            parameters=cluster.model_dump(mode="json", by_alias=True, exclude={"id"}),
+            parameters=cluster,
             command_context=self.storage_service.variant_study_service.command_factory.command_context,
             study_version=study_version,
         )
@@ -551,7 +551,8 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_clus
             ClusterAlreadyExists: If a cluster with the new name already exists in the area.
         """
         new_id = transform_name_to_id(new_cluster_name)
-        if any(new_id == storage.id for storage in self.get_storages(study, area_id)):
+        lower_new_id = new_id.lower()
+        if any(lower_new_id == storage.id.lower() for storage in self.get_storages(study, area_id)):
             raise DuplicateSTStorage(area_id, new_id)
 
         # Cluster duplication
@@ -570,13 +571,16 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_clus
         create_cluster_cmd = self._make_create_cluster_cmd(area_id, new_config, study_version)
 
         # Matrix edition
+        lower_source_id = source_id.lower()
         # noinspection SpellCheckingInspection
         ts_names = ["pmax_injection", "pmax_withdrawal", "lower_rule_curve", "upper_rule_curve", "inflows"]
         source_paths = [
-            _STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=source_id, ts_name=ts_name) for ts_name in ts_names
+            _STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=lower_source_id, ts_name=ts_name)
+            for ts_name in ts_names
         ]
         new_paths = [
-            _STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=new_id, ts_name=ts_name) for ts_name in ts_names
+            _STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=lower_new_id, ts_name=ts_name)
+            for ts_name in ts_names
         ]
 
         # Prepare and execute commands

antarest/study/business/areas/thermal_management.py

@@ -28,13 +28,12 @@
 from antarest.study.business.all_optional_meta import all_optional_model, camel_case_model
 from antarest.study.business.utils import execute_or_add_commands
 from antarest.study.model import STUDY_VERSION_8_7, Study
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.config.thermal import (
     Thermal870Config,
     Thermal870Properties,
     ThermalConfigType,
     create_thermal_config,
-    create_thermal_properties,
 )
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 from antarest.study.storage.storage_service import StudyStorageService
@@ -349,6 +348,7 @@ def update_cluster(
             ThermalClusterNotFound: If the provided `cluster_id` does not match the ID of the cluster
             in the provided cluster_data.
         """
+
         study_version = StudyVersion.parse(study.version)
         file_study = self._get_file_study(study)
         path = _CLUSTER_PATH.format(area_id=area_id, cluster_id=cluster_id)
@@ -357,19 +357,19 @@ def update_cluster(
         except KeyError:
             raise ThermalClusterNotFound(path, cluster_id) from None
         else:
-            old_properties = create_thermal_properties(study_version, **values)
+            old_config = create_thermal_config(study_version, **values)
 
         # Use Python values to synchronize Config and Form values
         new_values = cluster_data.model_dump(mode="json", by_alias=False, exclude_none=True)
-        new_properties = old_properties.copy(exclude={"id"}, update=new_values)
+        new_config = old_config.copy(exclude={"id"}, update=new_values)
+        new_data = new_config.model_dump(mode="json", by_alias=True, exclude={"id"})
 
         # create the dict containing the new values using aliases
         data: t.Dict[str, t.Any] = {}
-        for updated_field, updated_value in new_values.items():
-            if updated_field in old_properties.model_fields:
-                field_info = old_properties.model_fields[updated_field]
-                field_name = field_info.alias if field_info.alias else updated_field
-                data[field_name] = updated_value
+        for field_name, field in new_config.model_fields.items():
+            if field_name in new_values:
+                name = field.alias if field.alias else field_name
+                data[name] = new_data[name]
 
         # create the update config commands with the modified data
         command_context = self.storage_service.variant_study_service.command_factory.command_context
@@ -381,7 +381,7 @@ def update_cluster(
         ]
         execute_or_add_commands(study, file_study, commands, self.storage_service)
 
-        values = {**new_properties.model_dump(mode="json", by_alias=False), "id": cluster_id}
+        values = {**new_config.model_dump(mode="json", by_alias=False), "id": cluster_id}
         return ThermalClusterOutput.model_validate(values)
 
     def delete_clusters(self, study: Study, area_id: str, cluster_ids: t.Sequence[str]) -> None:
@@ -431,8 +431,9 @@ def duplicate_cluster(
         Raises:
             ClusterAlreadyExists: If a cluster with the new name already exists in the area.
         """
-        new_id = transform_name_to_id(new_cluster_name)
-        if any(new_id == cluster.id for cluster in self.get_clusters(study, area_id)):
+        new_id = transform_name_to_id(new_cluster_name, lower=False)
+        lower_new_id = new_id.lower()
+        if any(lower_new_id == cluster.id.lower() for cluster in self.get_clusters(study, area_id)):
             raise DuplicateThermalCluster(area_id, new_id)
 
         # Cluster duplication
@@ -444,22 +445,23 @@ def duplicate_cluster(
         create_cluster_cmd = self._make_create_cluster_cmd(area_id, new_config, study_version)
 
         # Matrix edition
+        lower_source_id = source_id.lower()
         source_paths = [
-            f"input/thermal/series/{area_id}/{source_id}/series",
-            f"input/thermal/prepro/{area_id}/{source_id}/modulation",
-            f"input/thermal/prepro/{area_id}/{source_id}/data",
+            f"input/thermal/series/{area_id}/{lower_source_id}/series",
+            f"input/thermal/prepro/{area_id}/{lower_source_id}/modulation",
+            f"input/thermal/prepro/{area_id}/{lower_source_id}/data",
         ]
         new_paths = [
-            f"input/thermal/series/{area_id}/{new_id}/series",
-            f"input/thermal/prepro/{area_id}/{new_id}/modulation",
-            f"input/thermal/prepro/{area_id}/{new_id}/data",
+            f"input/thermal/series/{area_id}/{lower_new_id}/series",
+            f"input/thermal/prepro/{area_id}/{lower_new_id}/modulation",
+            f"input/thermal/prepro/{area_id}/{lower_new_id}/data",
         ]
         study_version = StudyVersion.parse(study.version)
         if study_version >= STUDY_VERSION_8_7:
-            source_paths.append(f"input/thermal/series/{area_id}/{source_id}/CO2Cost")
-            source_paths.append(f"input/thermal/series/{area_id}/{source_id}/fuelCost")
-            new_paths.append(f"input/thermal/series/{area_id}/{new_id}/CO2Cost")
-            new_paths.append(f"input/thermal/series/{area_id}/{new_id}/fuelCost")
+            source_paths.append(f"input/thermal/series/{area_id}/{lower_source_id}/CO2Cost")
+            source_paths.append(f"input/thermal/series/{area_id}/{lower_source_id}/fuelCost")
+            new_paths.append(f"input/thermal/series/{area_id}/{lower_new_id}/CO2Cost")
+            new_paths.append(f"input/thermal/series/{area_id}/{lower_new_id}/fuelCost")
 
         # Prepare and execute commands
         commands: t.List[t.Union[CreateCluster, ReplaceMatrix]] = [create_cluster_cmd]
@@ -477,7 +479,8 @@ def duplicate_cluster(
         return ThermalClusterOutput(**new_config.model_dump(mode="json", by_alias=False))
 
     def validate_series(self, study: Study, area_id: str, cluster_id: str) -> bool:
-        thermal_cluster_path = Path(f"input/thermal/series/{area_id}/{cluster_id.lower()}")
+        lower_cluster_id = cluster_id.lower()
+        thermal_cluster_path = Path(f"input/thermal/series/{area_id}/{lower_cluster_id}")
         series_path = [thermal_cluster_path / "series"]
         if StudyVersion.parse(study.version) >= STUDY_VERSION_8_7:
             series_path.append(thermal_cluster_path / "CO2Cost")

antarest/study/business/binding_constraint_management.py

@@ -30,7 +30,7 @@
     MatrixWidthMismatchError,
     WrongMatrixHeightError,
 )
-from antarest.core.model import JSON, LowerCaseStr
+from antarest.core.model import JSON
 from antarest.core.requests import CaseInsensitiveDict
 from antarest.core.serialization import AntaresBaseModel
 from antarest.core.utils.string import to_camel_case
@@ -44,7 +44,7 @@
     BindingConstraintFrequency,
     BindingConstraintOperator,
 )
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 from antarest.study.storage.storage_service import StudyStorageService
 from antarest.study.storage.variantstudy.business.matrix_constants.binding_constraint.series_after_v87 import (
@@ -340,7 +340,7 @@ class ConstraintOutput830(ConstraintOutputBase):
 
 
 class ConstraintOutput870(ConstraintOutput830):
-    group: LowerCaseStr = DEFAULT_GROUP
+    group: str = DEFAULT_GROUP
 
 
 # WARNING: Do not change the order of the following line, it is used to determine

antarest/study/business/district_manager.py

@@ -16,7 +16,7 @@
 from antarest.core.serialization import AntaresBaseModel
 from antarest.study.business.utils import execute_or_add_commands
 from antarest.study.model import Study
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
 from antarest.study.storage.storage_service import StudyStorageService
 from antarest.study.storage.variantstudy.model.command.create_district import CreateDistrict, DistrictBaseFilter
 from antarest.study.storage.variantstudy.model.command.remove_district import RemoveDistrict

antarest/study/business/table_mode_management.py

@@ -219,9 +219,7 @@ def update_table_data(
             thermals_by_areas = collections.defaultdict(dict)
             for key, values in data.items():
                 area_id, cluster_id = key.split(" / ")
-                # Thermal clusters ids were not lowered at the time.
-                # So to ensure this endpoint still works with old scripts we have to lower the id at first.
-                thermals_by_areas[area_id][cluster_id.lower()] = ThermalClusterInput(**values)
+                thermals_by_areas[area_id][cluster_id] = ThermalClusterInput(**values)
             thermals_map = self._thermal_manager.update_thermals_props(study, thermals_by_areas)
             data = {
                 f"{area_id} / {cluster_id}": cluster.model_dump(by_alias=True, exclude={"id", "name"})
@@ -234,8 +232,7 @@ def update_table_data(
             renewables_by_areas = collections.defaultdict(dict)
             for key, values in data.items():
                 area_id, cluster_id = key.split(" / ")
-                # Same reason as for thermal clusters
-                renewables_by_areas[area_id][cluster_id.lower()] = RenewableClusterInput(**values)
+                renewables_by_areas[area_id][cluster_id] = RenewableClusterInput(**values)
             renewables_map = self._renewable_manager.update_renewables_props(study, renewables_by_areas)
             data = {
                 f"{area_id} / {cluster_id}": cluster.model_dump(by_alias=True, exclude={"id", "name"})

antarest/study/storage/rawstudy/model/filesystem/config/cluster.py

@@ -21,7 +21,6 @@
 
 from pydantic import Field
 
-from antarest.core.model import LowerCaseStr
 from antarest.core.serialization import AntaresBaseModel
 
 
@@ -48,9 +47,9 @@ class ItemProperties(
     [('group-A', 'cluster-01'), ('GROUP-A', 'cluster-02'), ('Group-B', 'CLUSTER-01')]
     """
 
-    group: LowerCaseStr = Field(default="", description="Cluster group")
+    group: str = Field(default="", description="Cluster group")
 
-    name: LowerCaseStr = Field(description="Cluster name", pattern=r"[a-zA-Z0-9_(),& -]+")
+    name: str = Field(description="Cluster name", pattern=r"[a-zA-Z0-9_(),& -]+")
 
     def __lt__(self, other: t.Any) -> bool:
         """
@@ -59,7 +58,7 @@ def __lt__(self, other: t.Any) -> bool:
         This method may be used to sort and group clusters by `group` and `name`.
         """
         if isinstance(other, ItemProperties):
-            return (self.group, self.name).__lt__((other.group, other.name))
+            return (self.group.upper(), self.name.upper()).__lt__((other.group.upper(), other.name.upper()))
         return NotImplemented