Skip to content

Commit

Permalink
WIP: allow matching on ID for clusters, but write name as defined in …
Browse files Browse the repository at this point in the history
…format

Signed-off-by: Sylvain Leclerc <sylvain.leclerc@rte-france.com>
sylvlecl committed Jan 23, 2025
1 parent ca2b53f commit 924e28f
Showing 18 changed files with 235 additions and 76 deletions.
2 changes: 1 addition & 1 deletion antarest/study/business/areas/renewable_management.py
Original file line number Diff line number Diff line change
@@ -223,7 +223,7 @@ def _make_create_cluster_cmd(
) -> CreateRenewablesCluster:
command = CreateRenewablesCluster(
area_id=area_id,
cluster_name=cluster.id,
cluster_name=cluster.name,
parameters=cluster.model_dump(mode="json", by_alias=True, exclude={"id"}),
command_context=self.storage_service.variant_study_service.command_factory.command_context,
study_version=study_version,
2 changes: 1 addition & 1 deletion antarest/study/business/areas/st_storage_management.py
Original file line number Diff line number Diff line change
@@ -441,7 +441,7 @@ def get_storage(
try:
config = file_study.tree.get(path.split("/"), depth=1)
except KeyError:
raise STStorageNotFound(path, storage_id) from None
raise STStorageNotFound(path, storage_id)
return create_storage_output(StudyVersion.parse(study.version), storage_id, config)

def update_storage(
2 changes: 1 addition & 1 deletion antarest/study/business/areas/thermal_management.py
Original file line number Diff line number Diff line change
@@ -317,7 +317,7 @@ def _make_create_cluster_cmd(
# between the cluster name and the cluster ID (which is a section name).
args = {
"area_id": area_id,
"cluster_name": cluster.id,
"cluster_name": cluster.name,
"parameters": cluster.model_dump(mode="json", by_alias=True, exclude={"id"}),
"command_context": self.storage_service.variant_study_service.command_factory.command_context,
"study_version": study_version,
34 changes: 20 additions & 14 deletions antarest/study/storage/rawstudy/ini_reader.py
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@

PrimitiveType = t.Union[str, int, float, bool]
ValueParser = Callable[[str], PrimitiveType]
SelectionPredicate = Callable[[str], bool]


def _lower_case(input: str) -> str:
@@ -70,8 +71,8 @@ class ReaderOptions:
option_regex: A compiled regex for matching option names.
"""

section_regex: t.Optional[t.Pattern[str]] = None
option_regex: t.Optional[t.Pattern[str]] = None
section_predicate: Optional[SelectionPredicate] = None
option_predicate: Optional[SelectionPredicate] = None

def select_section_option(self, section: str, option: str = "") -> bool:
"""
@@ -84,9 +85,9 @@ def select_section_option(self, section: str, option: str = "") -> bool:
Returns:
Whether the section and option match their respective regular expressions.
"""
if self.section_regex and not self.section_regex.fullmatch(section):
if self.section_predicate and not self.section_predicate(section):
return False
if self.option_regex and option and not self.option_regex.fullmatch(option):
if option and self.option_predicate and not self.option_predicate(option):
return False
return True

@@ -108,20 +109,25 @@ def ini_reader_options(
option: The option name to match (by default, all options are matched)
section_regex: The regex for matching section names.
option_regex: The regex for matching option names.
_unused: Placeholder for any unknown options.
Returns:
The newly created instance
"""
if section:
section_regex = re.compile(re.escape(section), re.IGNORECASE)
if option:
option_regex = re.compile(re.escape(option), re.IGNORECASE)
if isinstance(section_regex, str):
section_regex = re.compile(section_regex, re.IGNORECASE) if section_regex else None
if isinstance(option_regex, str):
option_regex = re.compile(option_regex, re.IGNORECASE) if option_regex else None
return ReaderOptions(section_regex=section_regex, option_regex=option_regex)
return ReaderOptions(
section_predicate=make_predicate(section, section_regex), option_predicate=make_predicate(option, option_regex)
)


def make_predicate(
value: str = "", regex: t.Optional[t.Union[str, t.Pattern[str]]] = None
) -> Optional[SelectionPredicate]:
if value:
option_regex = re.compile(re.escape(value), re.IGNORECASE)
elif isinstance(regex, str):
option_regex = re.compile(regex, re.IGNORECASE) if regex else None
else:
return None
return option_regex.fullmatch


class IReader(ABC):
4 changes: 2 additions & 2 deletions antarest/study/storage/rawstudy/ini_writer.py
Original file line number Diff line number Diff line change
@@ -70,7 +70,7 @@ def _write_line( # type:ignore
self, section_name, key, value
)
if self._value_serializers:
if serializer := self._get_serializer(key):
if serializer := self._get_serializer(section_name, key):
value = serializer(value)
if value is not None or not self._allow_no_value: # type:ignore
value = delimiter + str(value).replace("\n", "\n\t")
@@ -117,7 +117,7 @@ def write(self, data: JSON, path: Path) -> None:
data: JSON content.
path: path to `.ini` file.
"""
config_parser = IniConfigParser(special_keys=self.special_keys)
config_parser = IniConfigParser(special_keys=self.special_keys, value_serializers=self._value_serializers)
config_parser.read_dict(data)
with path.open("w") as fp:
config_parser.write(fp)
62 changes: 35 additions & 27 deletions antarest/study/storage/rawstudy/model/filesystem/ini_file_node.py
Original file line number Diff line number Diff line change
@@ -11,7 +11,6 @@
# This file is part of the Antares project.

import contextlib
import dataclasses
import functools
import io
import logging
@@ -21,7 +20,7 @@
import zipfile
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, List, Optional, Tuple
from typing import Callable, List, Optional

import py7zr
import pydantic_core
@@ -34,12 +33,13 @@
from antarest.study.storage.rawstudy.ini_reader import (
IniReader,
IReader,
OptionKey,
ReaderOptions,
ValueParser,
SelectionPredicate,
ini_reader_options,
make_predicate,
)
from antarest.study.storage.rawstudy.ini_writer import IniWriter
from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig
from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer
from antarest.study.storage.rawstudy.model.filesystem.inode import INode
@@ -52,6 +52,7 @@ def _lower_case(input: str) -> str:


LOWER_CASE_MATCHER: SectionMatcher = _lower_case
NAME_TO_ID_MATCHER: SectionMatcher = transform_name_to_id


@dataclass(frozen=True)
@@ -199,6 +200,12 @@ def _filter_for_url(self, data: JSON, depth: int, url: t.List[str]) -> JSON:
else:
return section_data

def _make_section_predicate(self, section: str) -> SelectionPredicate:
if self.section_matcher:
return lambda actual_section: self.section_matcher(section) == self.section_matcher(actual_section)
else:
return make_predicate(value=section)

# noinspection PyMethodMayBeStatic
def _build_options(self, url: t.List[str]) -> ReaderOptions:
"""
@@ -214,10 +221,13 @@ def _build_options(self, url: t.List[str]) -> ReaderOptions:
"""
if len(url) > 2:
raise ValueError(f"Invalid URL: {url!r}")
elif len(url) == 2:
return ini_reader_options(section=url[0], option=url[1])
elif len(url) == 1:
return ini_reader_options(section=url[0])
loc = url_to_location(url)
if loc.section and loc.key:
return ReaderOptions(
section_predicate=self._make_section_predicate(loc.section), option_predicate=make_predicate(loc.key)
)
elif loc.section:
return ReaderOptions(section_predicate=self._make_section_predicate(loc.section))
else:
return ini_reader_options()

@@ -306,35 +316,33 @@ def delete(self, url: t.Optional[t.List[str]] = None) -> None:
f"Cannot delete item {url!r}: URL should be fully resolved",
)

loc = url_to_location(url)

data = self.reader.read(self.path)

if url_len == 1:
section_name = url[0]
if loc.section and loc.key:
section = self._find_matching_section(data, loc.section)
if not section:
raise IniFileNodeWarning(
self.config,
f"Cannot delete section: Section [{loc.section}] not found",
)
try:
del data[section_name]
del data[section][loc.key]
except KeyError:
raise IniFileNodeWarning(
self.config,
f"Cannot delete section: Section [{section_name}] not found",
f"Cannot delete key: Key '{loc.key}' not found in section [{section}]",
) from None

elif url_len == 2:
section_name, key_name = url
try:
section = data[section_name]
except KeyError:
elif loc.section:
section = self._find_matching_section(data, loc.section)
if not section:
raise IniFileNodeWarning(
self.config,
f"Cannot delete key: Section [{section_name}] not found",
) from None
else:
try:
del section[key_name]
except KeyError:
raise IniFileNodeWarning(
self.config,
f"Cannot delete key: Key '{key_name}' not found in section [{section_name}]",
) from None
f"Cannot delete section: Section [{loc.section}] not found",
)
del data[section]

self.writer.write(data, self.path)

Original file line number Diff line number Diff line change
@@ -13,13 +13,15 @@
from typing_extensions import override

from antarest.study.storage.rawstudy.ini_reader import LOWER_CASE_PARSER, IniReader, OptionKey
from antarest.study.storage.rawstudy.ini_writer import LOWER_CASE_SERIALIZER, IniWriter
from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig
from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer
from antarest.study.storage.rawstudy.model.filesystem.folder_node import FolderNode
from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import LOWER_CASE_MATCHER, IniFileNode
from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import NAME_TO_ID_MATCHER, IniFileNode
from antarest.study.storage.rawstudy.model.filesystem.inode import TREE

_VALUE_PARSERS = {OptionKey(None, "group"): LOWER_CASE_PARSER}
_VALUE_SERIALIZERS = {OptionKey(None, "group"): LOWER_CASE_SERIALIZER}


class ClusteredRenewableClusterConfig(IniFileNode):
@@ -39,7 +41,12 @@ def __init__(
}
types = {cluster_id: section for cluster_id in config.get_renewable_ids(area)}
super().__init__(
context, config, types, reader=IniReader(value_parsers=_VALUE_PARSERS), section_matcher=LOWER_CASE_MATCHER
context,
config,
types,
reader=IniReader(value_parsers=_VALUE_PARSERS),
writer=IniWriter(value_serializers=_VALUE_SERIALIZERS),
section_matcher=NAME_TO_ID_MATCHER,
)


Original file line number Diff line number Diff line change
@@ -9,16 +9,12 @@
# SPDX-License-Identifier: MPL-2.0
#
# This file is part of the Antares project.
import typing as t

from typing_extensions import override

from antarest.core.model import SUB_JSON
from antarest.study.storage.rawstudy.ini_reader import LOWER_CASE_PARSER, IniReader, OptionKey
from antarest.study.storage.rawstudy.ini_writer import LOWER_CASE_SERIALIZER, IniWriter
from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig
from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer
from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import LOWER_CASE_MATCHER, IniFileNode
from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import NAME_TO_ID_MATCHER, IniFileNode

_VALUE_PARSERS = {OptionKey(None, "group"): LOWER_CASE_PARSER}
_VALUE_SERIALIZERS = {OptionKey(None, "group"): LOWER_CASE_SERIALIZER}
@@ -48,5 +44,5 @@ def __init__(
types,
reader=IniReader(value_parsers=_VALUE_PARSERS),
writer=IniWriter(value_serializers=_VALUE_SERIALIZERS),
section_matcher=LOWER_CASE_MATCHER,
section_matcher=NAME_TO_ID_MATCHER,
)
Original file line number Diff line number Diff line change
@@ -11,11 +11,13 @@
# This file is part of the Antares project.

from antarest.study.storage.rawstudy.ini_reader import LOWER_CASE_PARSER, IniReader, OptionKey
from antarest.study.storage.rawstudy.ini_writer import LOWER_CASE_SERIALIZER, IniWriter
from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig
from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer
from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import LOWER_CASE_MATCHER, IniFileNode
from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import NAME_TO_ID_MATCHER, IniFileNode

_VALUE_PARSERS = {OptionKey(None, "group"): LOWER_CASE_PARSER}
_VALUE_SERIALIZERS = {OptionKey(None, "group"): LOWER_CASE_SERIALIZER}


class InputThermalClustersAreaList(IniFileNode):
@@ -34,5 +36,10 @@ def __init__(
}
types = {th: section for th in config.get_thermal_ids(area)}
super().__init__(
context, config, types, reader=IniReader(value_parsers=_VALUE_PARSERS), section_matcher=LOWER_CASE_MATCHER
context,
config,
types,
reader=IniReader(value_parsers=_VALUE_PARSERS),
writer=IniWriter(value_serializers=_VALUE_SERIALIZERS),
section_matcher=NAME_TO_ID_MATCHER,
)
Original file line number Diff line number Diff line change
@@ -110,7 +110,7 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] =

cluster_id = data["cluster_id"]
config = study_data.tree.get(["input", "renewables", "clusters", self.area_id, "list"])
config[cluster_id] = self.parameters.model_dump(mode="json", by_alias=True)
config[self.cluster_name] = self.parameters.model_dump(mode="json", by_alias=True)

# Series identifiers are in lower case.
series_id = cluster_id.lower()
Original file line number Diff line number Diff line change
@@ -252,7 +252,7 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] =
# Fill-in the "list.ini" file with the parameters.
# On creation, it's better to write all the parameters in the file.
config = study_data.tree.get(["input", "st-storage", "clusters", self.area_id, "list"])
config[storage_id] = self.parameters.model_dump(mode="json", by_alias=True)
config[self.storage_name] = self.parameters.model_dump(mode="json", by_alias=True)

new_data: JSON = {
"input": {
11 changes: 4 additions & 7 deletions tests/integration/study_data_blueprint/test_lower_case.py
Original file line number Diff line number Diff line change
@@ -47,23 +47,20 @@ def test_clusters(self, client: TestClient, user_access_token: str, tmp_path: Pa
cluster_grp = "Battery"
url = "storages"
cluster_name = "Cluster 1"
lowered_name = cluster_name.lower()
lowered_grp = cluster_grp.lower()
res = client.post(
f"/v1/studies/{study_id}/areas/{area1_id}/{url}", json={"name": cluster_name, "group": cluster_grp}
)
assert res.status_code == 200, res.json()

# Asserts the fields are written in lower case inside the ini file
# Asserts the group fields are written in lower case inside the ini file
ini_path = study_path / "input" / cluster_type / "clusters" / area1_id / "list.ini"
ini_content = IniReader().read(ini_path)
assert list(ini_content.keys()) == [lowered_name]
assert ini_content[lowered_name]["group"] == lowered_grp
assert list(ini_content.keys()) == [cluster_name]
assert ini_content[cluster_name]["group"] == lowered_grp

# Rewrite the cluster name in MAJ to mimic legacy clusters
# Rewrite the group in MAJ to mimic legacy clusters
new_content = copy.deepcopy(ini_content)
new_content[cluster_name] = new_content.pop(lowered_name)
new_content[cluster_name]["name"] = cluster_name
new_content[cluster_name]["group"] = cluster_grp
IniWriter().write(new_content, ini_path)

6 changes: 2 additions & 4 deletions tests/storage/repository/antares_io/reader/test_ini_reader.py
Original file line number Diff line number Diff line change
@@ -346,18 +346,16 @@ def test_read__with_parser(self, tmp_path):
)
)

reader = IniReader()

def to_lower(input: str) -> str:
return input.lower()

value_parsers = {OptionKey("part2", "bar"): to_lower}
actual = reader.read(path, ini_reader_options(value_parsers=value_parsers))
actual = IniReader(value_parsers=value_parsers).read(path)
expected = {"part1": {"bar": "Hello"}, "part2": {"bar": "hello"}}
assert actual == expected

value_parsers = {OptionKey(None, "bar"): to_lower}
actual = reader.read(path, ini_reader_options(value_parsers=value_parsers))
actual = IniReader(value_parsers=value_parsers).read(path)
expected = {"part1": {"bar": "hello"}, "part2": {"bar": "hello"}}
assert actual == expected

29 changes: 29 additions & 0 deletions tests/storage/repository/antares_io/writer/test_ini_writer.py
Original file line number Diff line number Diff line change
@@ -15,6 +15,7 @@

import pytest

from antarest.study.storage.rawstudy.ini_reader import OptionKey
from antarest.study.storage.rawstudy.ini_writer import IniWriter


@@ -59,3 +60,31 @@ def test_write(tmp_path: str, ini_cleaner: Callable) -> None:
writer.write(json_data, path)

assert ini_cleaner(ini_content) == ini_cleaner(path.read_text())


@pytest.mark.unit_test
def test_write_with_custom_serializer(tmp_path: str, ini_cleaner: Callable) -> None:
path = Path(tmp_path) / "test.ini"

serializers = {OptionKey(None, "group"): lambda x: x.lower()}
writer = IniWriter(value_serializers=serializers)

expected = """
[part1]
group = gas
[part2]
group = gas
[part3]
other = Gas
"""

json_data = {
"part1": {"group": "Gas"},
"part2": {"group": "Gas"},
"part3": {"other": "Gas"},
}
writer.write(json_data, path)

assert ini_cleaner(path.read_text()) == ini_cleaner(expected)
Original file line number Diff line number Diff line change
@@ -87,15 +87,15 @@ def test_cluster_ini_list(tmp_path: Path, ini_node_cluster_class):
node.save(data)
# Asserts the data is saved correctly
ini_content = IniReader().read(file_path)
assert ini_content == data
# Asserts cluster group and ids are returned in lower case
assert ini_content == {"Cluster 1": {"group": "gas"}}
# Asserts cluster group is returned in lower case
content = node.get([])
assert content == {"Cluster 1": {"group": "gas"}}
# Asserts saving the group in upper case works and that it will be returned in lower case
node.save("NUCLEAR", ["Cluster 1", "group"])
content = node.get([])
assert content == {"Cluster 1": {"group": "nuclear"}}
# Asserts updating the file with an id not in lower case will be done correctly
# Asserts updating the file with an id in lower case will be done correctly
node.save({"params": "43"}, ["cluster 1"])
content = node.get([])
assert content == {"Cluster 1": {"params": 43}}
113 changes: 112 additions & 1 deletion tests/storage/repository/filesystem/test_ini_file_node.py
Original file line number Diff line number Diff line change
@@ -20,8 +20,13 @@

from antarest.core.model import JSON
from antarest.study.model import STUDY_VERSION_9_1
from antarest.study.storage.rawstudy.ini_writer import IniWriter
from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig
from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import IniFileNode
from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import (
NAME_TO_ID_MATCHER,
IniFileNode,
SectionMatcher,
)
from antarest.study.storage.rawstudy.model.filesystem.root.settings.scenariobuilder import ScenarioBuilder


@@ -293,6 +298,112 @@ def test_save(tmp_path: Path) -> None:
assert ini_path.read_text().strip() == expected.strip()


def _ini_node(tmp_path: Path, data: JSON = None, section_matcher: SectionMatcher = None) -> IniFileNode:
ini_path = tmp_path.joinpath("test.ini")
data = data or {
"part1": {
"key_float": 2.1,
"key_int": 1,
"key_str": "value1",
},
"part2": {
"key_float": 18,
"key_int": 5,
"key_str": "value2",
},
}
IniWriter().write(data=data, path=ini_path)

node = IniFileNode(
context=Mock(),
config=FileStudyTreeConfig(
study_path=tmp_path,
path=ini_path,
version=-1,
study_id="id",
areas={},
outputs={},
),
section_matcher=section_matcher,
)
return node


@pytest.fixture
def ini_node(tmp_path: Path) -> IniFileNode:
return _ini_node(tmp_path)


@pytest.fixture
def ini_node_with_id_matching(tmp_path: Path) -> IniFileNode:
return _ini_node(tmp_path, section_matcher=NAME_TO_ID_MATCHER)


@pytest.mark.unit_test
def test_delete_section(ini_node: IniFileNode) -> None:
ini_node.delete(["Part1"])
assert ini_node.get() == {
"part2": {
"key_float": 18,
"key_int": 5,
"key_str": "value2",
},
}


@pytest.mark.unit_test
def test_delete_section_with_matching(ini_node_with_id_matching: IniFileNode) -> None:
ini_node = ini_node_with_id_matching
ini_node.delete(["Part1"])
assert ini_node.get() == {
"part2": {
"key_float": 18,
"key_int": 5,
"key_str": "value2",
},
}


@pytest.mark.unit_test
def test_delete_section_with_matching_2(tmp_path: Path) -> None:
ini_node = _ini_node(
tmp_path, section_matcher=NAME_TO_ID_MATCHER, data={"Grand'Maison": {"key_float": 18, "key_int": 5}}
)
ini_node.delete(["grand maison"])
assert ini_node.get() == {}


@pytest.mark.unit_test
def test_get_section_with_matching(tmp_path: Path) -> None:
ini_node = _ini_node(
tmp_path, section_matcher=NAME_TO_ID_MATCHER, data={"Grand'Maison": {"key_float": 18, "key_int": 5}}
)
ini_node.get(["grand maison"])
assert ini_node.get() == {"Grand'Maison": {"key_float": 18, "key_int": 5}}


@pytest.mark.unit_test
def test_delete_option(ini_node: IniFileNode) -> None:
ini_node.delete(["part1", "key_int"])
assert ini_node.get() == {
"part1": {
"key_float": 2.1,
"key_str": "value1",
},
"part2": {
"key_float": 18,
"key_int": 5,
"key_str": "value2",
},
}


@pytest.mark.unit_test
def test_delete_file(ini_node: IniFileNode) -> None:
ini_node.delete()
assert ini_node.get() == {}


@pytest.mark.parametrize(
("ini_section", "url"),
[
2 changes: 1 addition & 1 deletion tests/variantstudy/model/command/test_create_cluster.py
Original file line number Diff line number Diff line change
@@ -137,7 +137,7 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext):

clusters = configparser.ConfigParser()
clusters.read(study_path / "input" / "thermal" / "clusters" / area_id / "list.ini")
section = clusters[cluster_name.lower()]
section = clusters[cluster_name]
assert str(section["name"]) == cluster_name
assert str(section["group"]) == parameters["group"]
assert int(section["unitcount"]) == int(parameters["unitcount"])
4 changes: 2 additions & 2 deletions tests/variantstudy/model/command/test_create_st_storage.py
Original file line number Diff line number Diff line change
@@ -305,7 +305,7 @@ def test_apply__nominal_case(self, recent_study: FileStudy, command_context: Com
# check the config
config = recent_study.tree.get(["input", "st-storage", "clusters", cmd.area_id, "list"])
expected = {
"storage1": {
"Storage1": {
"efficiency": 0.94,
"group": "battery",
"initiallevel": 0.5,
@@ -325,7 +325,7 @@ def test_apply__nominal_case(self, recent_study: FileStudy, command_context: Com
pmax_injection_id = service.create(pmax_injection)
inflows_id = service.create(inflows)
expected = {
"storage1": {
"Storage1": {
"pmax_injection": f"matrix://{pmax_injection_id}",
"pmax_withdrawal": constants.get_st_storage_pmax_withdrawal(),
"lower_rule_curve": constants.get_st_storage_lower_rule_curve(),

0 comments on commit 924e28f

Please sign in to comment.