From 2940d4b7ced3c8708bf5086f87cb1da5fd904c77 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Mon, 20 Jan 2025 11:29:31 +0100 Subject: [PATCH 01/53] add alembic migration --- ...3f1b6_create_links_ts_generation_tables.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py diff --git a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py new file mode 100644 index 0000000000..2a7f2550ab --- /dev/null +++ b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py @@ -0,0 +1,68 @@ +"""create_links_ts_generation_tables + +Revision ID: dadad513f1b6 +Revises: bae9c99bc42d +Create Date: 2025-01-20 10:11:01.293931 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy import table, column +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = 'dadad513f1b6' +down_revision = 'bae9c99bc42d' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "study_nb_ts_gen", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("links", sa.Integer(), server_default="1", nullable=False), + sa.ForeignKeyConstraint( + ["id"], + ["study.id"], + ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + + # Fill the table + bind = op.get_bind() + session = Session(bind=bind) + ts_gen_table = table("study_nb_ts_gen", column("id"), column("links")) + study_table = table('study', column('id')) + study_ids = session.query(study_table).all() + data_to_insert = [{'id': study_id[0], 'links': 1} for study_id in study_ids] + op.bulk_insert(ts_gen_table, data_to_insert) + + op.create_table( + "links_ts_gen_properties", + sa.Column("id", sa.Integer()), + sa.Column("area_from", sa.String(), nullable=False), + sa.Column("area_to", sa.String(), nullable=False), + sa.Column("prepro", sa.String(), nullable=False), + sa.Column("modulation", sa.String(), nullable=False), + sa.Column("unit_count", sa.String(), nullable=False), + sa.Column("nominal_capacity", sa.String(), nullable=False), + sa.Column("law_planned", sa.Enum('uniform', 'geometric', name='lawplanned'), nullable=False), + sa.Column("law_forced", sa.Enum('uniform', 'geometric', name='lawforced'), nullable=False), + sa.Column("volatility_planned", sa.String(), nullable=False), + sa.Column("volatility_forced", sa.String(), nullable=False), + sa.Column("force_no_generation", sa.Boolean(), nullable=False), + sa.Column("study_id", sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint( + ["study_id"], + ["study.id"], + ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id") + ) + + +def downgrade(): + op.drop_table("study_nb_ts_gen") + op.drop_table("links_ts_gen_properties") From 93fc3f7caa8d45ecb964db6fdbfbe7252daff360 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Mon, 20 Jan 2025 18:04:55 +0100 Subject: [PATCH 02/53] add table models --- ...3f1b6_create_links_ts_generation_tables.py | 4 +- antarest/study/model.py | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py index 2a7f2550ab..417c548037 100644 --- a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py +++ b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py @@ -46,8 +46,8 @@ def upgrade(): sa.Column("area_to", sa.String(), nullable=False), sa.Column("prepro", sa.String(), nullable=False), sa.Column("modulation", sa.String(), nullable=False), - sa.Column("unit_count", sa.String(), nullable=False), - sa.Column("nominal_capacity", sa.String(), nullable=False), + sa.Column("unit_count", sa.Integer(), nullable=False), + sa.Column("nominal_capacity", sa.Float(), nullable=False), sa.Column("law_planned", sa.Enum('uniform', 'geometric', name='lawplanned'), nullable=False), sa.Column("law_forced", sa.Enum('uniform', 'geometric', name='lawforced'), nullable=False), sa.Column("volatility_planned", sa.String(), nullable=False), diff --git a/antarest/study/model.py b/antarest/study/model.py index d2ff8b97b9..53f78adc52 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -25,6 +25,7 @@ Column, DateTime, Enum, + Float, ForeignKey, Integer, PrimaryKeyConstraint, @@ -39,6 +40,7 @@ from antarest.core.serialization import AntaresBaseModel from antarest.login.model import Group, GroupDTO, Identity from antarest.study.css4_colors import COLOR_NAMES +from antarest.study.storage.rawstudy.model.filesystem.config.thermal import LawOption if t.TYPE_CHECKING: # avoid circular import @@ -89,6 +91,41 @@ } +class TSGeneration(Base): # type:ignore + """ + A table to store how many columns needs to be generated by category and by study + + Attributes: + id: A foreign key on the study ID. + links: An integer representing how many columns needs to be generated during links TS generation. + """ + + __tablename__ = "study_nb_ts_gen" + + id: str = Column(String(36), ForeignKey("study.id", ondelete="CASCADE"), index=True, nullable=False) + links: int = Column(Integer, default=1, nullable=False) + + +class LinkParametersForTSGeneration(Base): # type:ignore + """A table to store for each link of a study, the input parameters to give to the TS generation algorithm""" + + __tablename__ = "links_ts_gen_properties" + + id = Column(Integer, primary_key=True, unique=True) + study_id = Column(String(36), ForeignKey("study.id", ondelete="CASCADE"), index=True, nullable=False) + area_from = Column(String(255), nullable=False) + area_to = Column(String(255), nullable=False) + prepro = Column(String(255), nullable=False) + modulation = Column(String(255), nullable=False) + unit_count = Column(Integer, nullable=False) + nominal_capacity = Column(Float, nullable=False) + law_planned = Column(Enum(LawOption), default=LawOption.UNIFORM, nullable=False) + law_uniform = Column(Enum(LawOption), default=LawOption.UNIFORM, nullable=False) + volatility_planned = Column(Float, nullable=False) + volatility_forced = Column(Float, nullable=False) + force_no_generation = Column(Boolean, nullable=False, default=False) + + class StudyGroup(Base): # type:ignore """ A table to manage the many-to-many relationship between `Study` and `Group` From 5442adaeacd63c3f3b02037b9a2dda4fb7f39c03 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Mon, 20 Jan 2025 18:25:41 +0100 Subject: [PATCH 03/53] always build data on the fly --- .../dadad513f1b6_create_links_ts_generation_tables.py | 11 ----------- antarest/study/model.py | 4 +++- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py index 417c548037..427b78f977 100644 --- a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py +++ b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py @@ -7,8 +7,6 @@ """ from alembic import op import sqlalchemy as sa -from sqlalchemy import table, column -from sqlalchemy.orm import Session # revision identifiers, used by Alembic. revision = 'dadad513f1b6' @@ -30,15 +28,6 @@ def upgrade(): sa.PrimaryKeyConstraint("id"), ) - # Fill the table - bind = op.get_bind() - session = Session(bind=bind) - ts_gen_table = table("study_nb_ts_gen", column("id"), column("links")) - study_table = table('study', column('id')) - study_ids = session.query(study_table).all() - data_to_insert = [{'id': study_id[0], 'links': 1} for study_id in study_ids] - op.bulk_insert(ts_gen_table, data_to_insert) - op.create_table( "links_ts_gen_properties", sa.Column("id", sa.Integer()), diff --git a/antarest/study/model.py b/antarest/study/model.py index 53f78adc52..fb26e65202 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -102,7 +102,9 @@ class TSGeneration(Base): # type:ignore __tablename__ = "study_nb_ts_gen" - id: str = Column(String(36), ForeignKey("study.id", ondelete="CASCADE"), index=True, nullable=False) + id: str = Column( + String(36), ForeignKey("study.id", ondelete="CASCADE"), primary_key=True, index=True, nullable=False + ) links: int = Column(Integer, default=1, nullable=False) From 99d66f5a210b08fe3a8b21e2c01565c3f658e191 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 09:53:14 +0100 Subject: [PATCH 04/53] rename tables --- .../dadad513f1b6_create_links_ts_generation_tables.py | 4 ++-- antarest/study/model.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py index 427b78f977..779b3dd912 100644 --- a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py +++ b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py @@ -17,7 +17,7 @@ def upgrade(): op.create_table( - "study_nb_ts_gen", + "nb_years_ts_generation", sa.Column("id", sa.String(length=36), nullable=False), sa.Column("links", sa.Integer(), server_default="1", nullable=False), sa.ForeignKeyConstraint( @@ -29,7 +29,7 @@ def upgrade(): ) op.create_table( - "links_ts_gen_properties", + "links_parameters_ts_generation", sa.Column("id", sa.Integer()), sa.Column("area_from", sa.String(), nullable=False), sa.Column("area_to", sa.String(), nullable=False), diff --git a/antarest/study/model.py b/antarest/study/model.py index fb26e65202..5f8319d194 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -91,7 +91,7 @@ } -class TSGeneration(Base): # type:ignore +class NbYearsTsGeneration(Base): # type:ignore """ A table to store how many columns needs to be generated by category and by study @@ -100,7 +100,7 @@ class TSGeneration(Base): # type:ignore links: An integer representing how many columns needs to be generated during links TS generation. """ - __tablename__ = "study_nb_ts_gen" + __tablename__ = "nb_years_ts_generation" id: str = Column( String(36), ForeignKey("study.id", ondelete="CASCADE"), primary_key=True, index=True, nullable=False @@ -108,10 +108,10 @@ class TSGeneration(Base): # type:ignore links: int = Column(Integer, default=1, nullable=False) -class LinkParametersForTSGeneration(Base): # type:ignore +class LinksParametersTsGeneration(Base): # type:ignore """A table to store for each link of a study, the input parameters to give to the TS generation algorithm""" - __tablename__ = "links_ts_gen_properties" + __tablename__ = "links_parameters_ts_generation" id = Column(Integer, primary_key=True, unique=True) study_id = Column(String(36), ForeignKey("study.id", ondelete="CASCADE"), index=True, nullable=False) From f66a84e26c240372ed865aa82c6875d69b12aab3 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 10:19:31 +0100 Subject: [PATCH 05/53] continue work --- ...513f1b6_create_links_ts_generation_tables.py | 11 ++++++----- antarest/study/model.py | 10 +++++----- .../variantstudy/model/command/create_link.py | 13 ++++++++++++- .../variantstudy/model/command/remove_link.py | 17 ++++++++++++++--- 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py index 779b3dd912..0645d6d5ba 100644 --- a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py +++ b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py @@ -28,6 +28,7 @@ def upgrade(): sa.PrimaryKeyConstraint("id"), ) + default_boolean = sa.text('1') if op.get_context().dialect.name == 'sqlite' else 't' op.create_table( "links_parameters_ts_generation", sa.Column("id", sa.Integer()), @@ -35,13 +36,13 @@ def upgrade(): sa.Column("area_to", sa.String(), nullable=False), sa.Column("prepro", sa.String(), nullable=False), sa.Column("modulation", sa.String(), nullable=False), - sa.Column("unit_count", sa.Integer(), nullable=False), - sa.Column("nominal_capacity", sa.Float(), nullable=False), + sa.Column("unit_count", sa.Integer(), nullable=False, server_default="1"), + sa.Column("nominal_capacity", sa.Float(), nullable=False, server_default="0"), sa.Column("law_planned", sa.Enum('uniform', 'geometric', name='lawplanned'), nullable=False), sa.Column("law_forced", sa.Enum('uniform', 'geometric', name='lawforced'), nullable=False), - sa.Column("volatility_planned", sa.String(), nullable=False), - sa.Column("volatility_forced", sa.String(), nullable=False), - sa.Column("force_no_generation", sa.Boolean(), nullable=False), + sa.Column("volatility_planned", sa.String(), nullable=False, server_default="0"), + sa.Column("volatility_forced", sa.String(), nullable=False, server_default="0"), + sa.Column("force_no_generation", sa.Boolean(), nullable=False, server_default=default_boolean), sa.Column("study_id", sa.String(length=36), nullable=False), sa.ForeignKeyConstraint( ["study_id"], diff --git a/antarest/study/model.py b/antarest/study/model.py index 5f8319d194..5180b5a5f3 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -119,13 +119,13 @@ class LinksParametersTsGeneration(Base): # type:ignore area_to = Column(String(255), nullable=False) prepro = Column(String(255), nullable=False) modulation = Column(String(255), nullable=False) - unit_count = Column(Integer, nullable=False) - nominal_capacity = Column(Float, nullable=False) + unit_count = Column(Integer, nullable=False, default=1) + nominal_capacity = Column(Float, nullable=False, default=0) law_planned = Column(Enum(LawOption), default=LawOption.UNIFORM, nullable=False) law_uniform = Column(Enum(LawOption), default=LawOption.UNIFORM, nullable=False) - volatility_planned = Column(Float, nullable=False) - volatility_forced = Column(Float, nullable=False) - force_no_generation = Column(Boolean, nullable=False, default=False) + volatility_planned = Column(Float, nullable=False, default=0) + volatility_forced = Column(Float, nullable=False, default=0) + force_no_generation = Column(Boolean, nullable=False, default=True) class StudyGroup(Base): # type:ignore diff --git a/antarest/study/storage/variantstudy/model/command/create_link.py b/antarest/study/storage/variantstudy/model/command/create_link.py index 1158c32a63..6f7ede228c 100644 --- a/antarest/study/storage/variantstudy/model/command/create_link.py +++ b/antarest/study/storage/variantstudy/model/command/create_link.py @@ -17,10 +17,11 @@ from typing_extensions import override from antarest.core.exceptions import LinkValidationError +from antarest.core.utils.fastapi_sqlalchemy import db from antarest.core.utils.utils import assert_this from antarest.matrixstore.model import MatrixData from antarest.study.business.model.link_model import LinkInternal -from antarest.study.model import STUDY_VERSION_8_2 +from antarest.study.model import STUDY_VERSION_8_2, LinksParametersTsGeneration, NbYearsTsGeneration from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig, Link from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.business.utils import strip_matrix_protocol, validate_matrix @@ -279,6 +280,16 @@ def _apply(self, study_data: FileStudy, listener: Optional[ICommandListener] = N area_from = data["area_from"] area_to = data["area_to"] + study_id = study_data.config.study_id + with db(): + link_info = db.session.query(NbYearsTsGeneration).filter_by(study_id=study_id).first() + if link_info: + # The DB is up-to-date, we have to fill it. + # If it was empty, we shouldn't fill it here. + db.session.add(LinksParametersTsGeneration(study_id=study_id, area_from=area_from, area_to=area_to)) + # todo: get the default matrix. How ???? + db.session.commit() + study_data.tree.save(validated_properties, ["input", "links", area_from, "properties", area_to]) self.series = self.series or (self.command_context.generator_matrix_constants.get_link(version=version)) diff --git a/antarest/study/storage/variantstudy/model/command/remove_link.py b/antarest/study/storage/variantstudy/model/command/remove_link.py index 2f9b2fa7ef..5a090a5fe5 100644 --- a/antarest/study/storage/variantstudy/model/command/remove_link.py +++ b/antarest/study/storage/variantstudy/model/command/remove_link.py @@ -15,7 +15,8 @@ from pydantic import field_validator, model_validator from typing_extensions import override -from antarest.study.model import STUDY_VERSION_8_2 +from antarest.core.utils.fastapi_sqlalchemy import db +from antarest.study.model import STUDY_VERSION_8_2, LinksParametersTsGeneration from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig, transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.common import CommandName, CommandOutput @@ -139,7 +140,7 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = The status of the operation and a message. """ - output = self._check_link_exists(study_data.config)[0] + output = self._apply_config(study_data.config)[0] if output.status: if study_data.config.version < STUDY_VERSION_8_2: @@ -152,7 +153,17 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = self._remove_link_from_scenario_builder(study_data) - return self._apply_config(study_data.config)[0] + with db(): + removed_link = ( + db.session.query(LinksParametersTsGeneration) + .filter_by(study_id=study_data.config.study_id, area_from=self.area1, area_to=self.area2) + .first() + ) + if removed_link: # The DB can be empty for the considered study as we're filling it on the fly + db.session.delete(removed_link) + db.session.commit() + + return output @override def to_dto(self) -> CommandDTO: From 8cd95f6bd0150b8a071b63a7b8e95d36ee525ec6 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 10:21:39 +0100 Subject: [PATCH 06/53] make matrices nullable inside DB --- .../dadad513f1b6_create_links_ts_generation_tables.py | 4 ++-- antarest/study/model.py | 4 ++-- .../study/storage/variantstudy/model/command/create_link.py | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py index 0645d6d5ba..c8ee4bc2dc 100644 --- a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py +++ b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py @@ -34,8 +34,8 @@ def upgrade(): sa.Column("id", sa.Integer()), sa.Column("area_from", sa.String(), nullable=False), sa.Column("area_to", sa.String(), nullable=False), - sa.Column("prepro", sa.String(), nullable=False), - sa.Column("modulation", sa.String(), nullable=False), + sa.Column("prepro", sa.String(), nullable=True), + sa.Column("modulation", sa.String(), nullable=True), sa.Column("unit_count", sa.Integer(), nullable=False, server_default="1"), sa.Column("nominal_capacity", sa.Float(), nullable=False, server_default="0"), sa.Column("law_planned", sa.Enum('uniform', 'geometric', name='lawplanned'), nullable=False), diff --git a/antarest/study/model.py b/antarest/study/model.py index 5180b5a5f3..c3624ee461 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -117,8 +117,8 @@ class LinksParametersTsGeneration(Base): # type:ignore study_id = Column(String(36), ForeignKey("study.id", ondelete="CASCADE"), index=True, nullable=False) area_from = Column(String(255), nullable=False) area_to = Column(String(255), nullable=False) - prepro = Column(String(255), nullable=False) - modulation = Column(String(255), nullable=False) + prepro = Column(String(255), nullable=True) + modulation = Column(String(255), nullable=True) unit_count = Column(Integer, nullable=False, default=1) nominal_capacity = Column(Float, nullable=False, default=0) law_planned = Column(Enum(LawOption), default=LawOption.UNIFORM, nullable=False) diff --git a/antarest/study/storage/variantstudy/model/command/create_link.py b/antarest/study/storage/variantstudy/model/command/create_link.py index 6f7ede228c..af6766734e 100644 --- a/antarest/study/storage/variantstudy/model/command/create_link.py +++ b/antarest/study/storage/variantstudy/model/command/create_link.py @@ -287,7 +287,6 @@ def _apply(self, study_data: FileStudy, listener: Optional[ICommandListener] = N # The DB is up-to-date, we have to fill it. # If it was empty, we shouldn't fill it here. db.session.add(LinksParametersTsGeneration(study_id=study_id, area_from=area_from, area_to=area_to)) - # todo: get the default matrix. How ???? db.session.commit() study_data.tree.save(validated_properties, ["input", "links", area_from, "properties", area_to]) From feaaaecfcfd8624d9ffe985ee54d92ed32e753e0 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 10:28:00 +0100 Subject: [PATCH 07/53] fix typo inside db query --- .../study/storage/variantstudy/model/command/create_link.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/antarest/study/storage/variantstudy/model/command/create_link.py b/antarest/study/storage/variantstudy/model/command/create_link.py index af6766734e..d5ad63115d 100644 --- a/antarest/study/storage/variantstudy/model/command/create_link.py +++ b/antarest/study/storage/variantstudy/model/command/create_link.py @@ -282,7 +282,7 @@ def _apply(self, study_data: FileStudy, listener: Optional[ICommandListener] = N study_id = study_data.config.study_id with db(): - link_info = db.session.query(NbYearsTsGeneration).filter_by(study_id=study_id).first() + link_info = db.session.query(NbYearsTsGeneration).filter_by(id=study_id).first() if link_info: # The DB is up-to-date, we have to fill it. # If it was empty, we shouldn't fill it here. From 351a342640f082f83f160a13f8e72a9b9a21ba12 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 10:59:20 +0100 Subject: [PATCH 08/53] fix remove link code --- .../variantstudy/model/command/remove_link.py | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/antarest/study/storage/variantstudy/model/command/remove_link.py b/antarest/study/storage/variantstudy/model/command/remove_link.py index 5a090a5fe5..bb4f62ad14 100644 --- a/antarest/study/storage/variantstudy/model/command/remove_link.py +++ b/antarest/study/storage/variantstudy/model/command/remove_link.py @@ -104,12 +104,8 @@ def _apply_config(self, study_cfg: FileStudyTreeConfig) -> OutputTuple: A tuple containing the command output and a dictionary of extra data. On success, the dictionary contains the source and target areas. """ - output, data = self._check_link_exists(study_cfg) - - if output.status: - del study_cfg.areas[self.area1].links[self.area2] - - return output, data + del study_cfg.areas[self.area1].links[self.area2] + return CommandOutput(status=True, message=f"Link between '{self.area1}' and '{self.area2}' removed"), {} def _remove_link_from_scenario_builder(self, study_data: FileStudy) -> None: """ @@ -140,18 +136,19 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = The status of the operation and a message. """ - output = self._apply_config(study_data.config)[0] + output = self._check_link_exists(study_data.config)[0] + if not output.status: + return output - if output.status: - if study_data.config.version < STUDY_VERSION_8_2: - study_data.tree.delete(["input", "links", self.area1, self.area2]) - else: - study_data.tree.delete(["input", "links", self.area1, f"{self.area2}_parameters"]) - study_data.tree.delete(["input", "links", self.area1, "capacities", f"{self.area2}_direct"]) - study_data.tree.delete(["input", "links", self.area1, "capacities", f"{self.area2}_indirect"]) - study_data.tree.delete(["input", "links", self.area1, "properties", self.area2]) + if study_data.config.version < STUDY_VERSION_8_2: + study_data.tree.delete(["input", "links", self.area1, self.area2]) + else: + study_data.tree.delete(["input", "links", self.area1, f"{self.area2}_parameters"]) + study_data.tree.delete(["input", "links", self.area1, "capacities", f"{self.area2}_direct"]) + study_data.tree.delete(["input", "links", self.area1, "capacities", f"{self.area2}_indirect"]) + study_data.tree.delete(["input", "links", self.area1, "properties", self.area2]) - self._remove_link_from_scenario_builder(study_data) + self._remove_link_from_scenario_builder(study_data) with db(): removed_link = ( @@ -163,7 +160,7 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = db.session.delete(removed_link) db.session.commit() - return output + return self._apply_config(study_data.config)[0] @override def to_dto(self) -> CommandDTO: From e432b5d026a0da7afa990d621de0c9f1f26c6673 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 11:36:31 +0100 Subject: [PATCH 09/53] fix test --- tests/study/storage/variantstudy/test_snapshot_generator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/study/storage/variantstudy/test_snapshot_generator.py b/tests/study/storage/variantstudy/test_snapshot_generator.py index 6956502504..6c89318342 100644 --- a/tests/study/storage/variantstudy/test_snapshot_generator.py +++ b/tests/study/storage/variantstudy/test_snapshot_generator.py @@ -876,13 +876,14 @@ def test_generate__nominal_case( ) # Check: the number of database queries is kept as low as possible. - # We expect 5 queries: + # We expect 6 queries: # - 1 query to fetch the ancestors of a variant study, # - 1 query to fetch the root study (with owner and groups for permission check), # - 1 query to fetch the list of variants with snapshot, commands, etc., # - 1 query to update the variant study additional_data, # - 1 query to insert the variant study snapshot. - assert len(db_recorder.sql_statements) == 5, str(db_recorder) + # - 1 query to check the DB state inside the create_link command + assert len(db_recorder.sql_statements) == 6, str(db_recorder) # Check: the variant generation must succeed. assert results.model_dump() == { From c1143456547ab264178992547bad30946a345095 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 14:09:35 +0100 Subject: [PATCH 10/53] update downgrade alembic --- .../dadad513f1b6_create_links_ts_generation_tables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py index c8ee4bc2dc..b0bded3c13 100644 --- a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py +++ b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py @@ -54,5 +54,5 @@ def upgrade(): def downgrade(): - op.drop_table("study_nb_ts_gen") - op.drop_table("links_ts_gen_properties") + op.drop_table("nb_years_ts_generation") + op.drop_table("links_parameters_ts_generation") From 2ffa87a0b87cd26bc34fb5e910d9328501003605 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 15:56:56 +0100 Subject: [PATCH 11/53] introduce new class --- antarest/study/business/model/link_model.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/antarest/study/business/model/link_model.py b/antarest/study/business/model/link_model.py index 9d6f50e1f5..c2449586f6 100644 --- a/antarest/study/business/model/link_model.py +++ b/antarest/study/business/model/link_model.py @@ -19,6 +19,7 @@ from antarest.core.utils.string import to_camel_case, to_kebab_case from antarest.study.business.enum_ignore_case import EnumIgnoreCase from antarest.study.model import STUDY_VERSION_8_2 +from antarest.study.storage.rawstudy.model.filesystem.config.thermal import LawOption class AssetType(EnumIgnoreCase): @@ -139,7 +140,7 @@ def join_with_comma(values: t.List[FilterOption]) -> str: ] -class LinkBaseDTO(AntaresBaseModel): +class LinkIniDTO(AntaresBaseModel): model_config = ConfigDict(alias_generator=to_camel_case, populate_by_name=True, extra="forbid") hurdles_cost: bool = False @@ -158,6 +159,22 @@ class LinkBaseDTO(AntaresBaseModel): filter_year_by_year: t.Optional[comma_separated_enum_list] = FILTER_VALUES +class LinkTsGeneration(AntaresBaseModel): + model_config = ConfigDict(alias_generator=to_camel_case, populate_by_name=True, extra="forbid") + + unit_count: int = 1 + nominal_capacity: float = 0 + law_planned: LawOption = LawOption.UNIFORM + law_forced: LawOption = LawOption.UNIFORM + volatility_planned: float = Field(default=0.0, ge=0, le=1) + volatility_forced: float = Field(default=0.0, ge=0, le=1) + force_no_generation: bool = True + + +class LinkBaseDTO(LinkIniDTO, LinkTsGeneration): + pass + + class Area(AntaresBaseModel): area1: str area2: str From 5ada0798805458f790d6bb12fef714c04833a1fb Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 16:07:55 +0100 Subject: [PATCH 12/53] implement create link method --- antarest/study/model.py | 2 +- .../variantstudy/model/command/create_link.py | 46 +++++++++++-------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index c3624ee461..f7bf322bc3 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -122,7 +122,7 @@ class LinksParametersTsGeneration(Base): # type:ignore unit_count = Column(Integer, nullable=False, default=1) nominal_capacity = Column(Float, nullable=False, default=0) law_planned = Column(Enum(LawOption), default=LawOption.UNIFORM, nullable=False) - law_uniform = Column(Enum(LawOption), default=LawOption.UNIFORM, nullable=False) + law_forced = Column(Enum(LawOption), default=LawOption.UNIFORM, nullable=False) volatility_planned = Column(Float, nullable=False, default=0) volatility_forced = Column(Float, nullable=False, default=0) force_no_generation = Column(Boolean, nullable=False, default=True) diff --git a/antarest/study/storage/variantstudy/model/command/create_link.py b/antarest/study/storage/variantstudy/model/command/create_link.py index d5ad63115d..a0d9fc7e0c 100644 --- a/antarest/study/storage/variantstudy/model/command/create_link.py +++ b/antarest/study/storage/variantstudy/model/command/create_link.py @@ -20,8 +20,8 @@ from antarest.core.utils.fastapi_sqlalchemy import db from antarest.core.utils.utils import assert_this from antarest.matrixstore.model import MatrixData -from antarest.study.business.model.link_model import LinkInternal -from antarest.study.model import STUDY_VERSION_8_2, LinksParametersTsGeneration, NbYearsTsGeneration +from antarest.study.business.model.link_model import Area, LinkInternal, LinkTsGeneration +from antarest.study.model import STUDY_VERSION_8_2, LinksParametersTsGeneration from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig, Link from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.business.utils import strip_matrix_protocol, validate_matrix @@ -269,28 +269,38 @@ def _apply(self, study_data: FileStudy, listener: Optional[ICommandListener] = N if not output.status: return output - to_exclude = {"area1", "area2"} + internal_link = LinkInternal.model_validate(self.parameters) + area_from = data["area_from"] + area_to = data["area_to"] + + # Saves ini properties + to_exclude = set(Area.model_fields.keys() & LinkTsGeneration.model_fields.keys()) if version < STUDY_VERSION_8_2: to_exclude.update("filter-synthesis", "filter-year-by-year") + ini_properties = internal_link.model_dump(by_alias=True, exclude=to_exclude) + study_data.tree.save(ini_properties, ["input", "links", area_from, "properties", area_to]) - validated_properties = LinkInternal.model_validate(self.parameters).model_dump( - by_alias=True, exclude=to_exclude - ) + # Saves DB properties + includes = set(LinkTsGeneration.model_fields.keys()) + db_properties = LinkTsGeneration.model_validate(internal_link.model_dump(mode="json", include=includes)) - area_from = data["area_from"] - area_to = data["area_to"] - - study_id = study_data.config.study_id with db(): - link_info = db.session.query(NbYearsTsGeneration).filter_by(id=study_id).first() - if link_info: - # The DB is up-to-date, we have to fill it. - # If it was empty, we shouldn't fill it here. - db.session.add(LinksParametersTsGeneration(study_id=study_id, area_from=area_from, area_to=area_to)) - db.session.commit() - - study_data.tree.save(validated_properties, ["input", "links", area_from, "properties", area_to]) + new_parameters = LinksParametersTsGeneration( + study_id=study_data.config.study_id, + area_from=area_from, + area_to=area_to, + unit_count=db_properties.unit_count, + nominal_capacity=db_properties.nominal_capacity, + law_planned=db_properties.law_planned, + law_forced=db_properties.law_forced, + volatility_planned=db_properties.volatility_planned, + volatility_forced=db_properties.volatility_forced, + force_no_generation=db_properties.force_no_generation, + ) + db.session.add(new_parameters) + db.session.commit() + # Saves matrices self.series = self.series or (self.command_context.generator_matrix_constants.get_link(version=version)) self.direct = self.direct or (self.command_context.generator_matrix_constants.get_link_direct()) self.indirect = self.indirect or (self.command_context.generator_matrix_constants.get_link_indirect()) From 3ec33dda60b56ddb97e5fe2837aac4fb94850e8f Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 16:26:36 +0100 Subject: [PATCH 13/53] update link implemented --- .../variantstudy/model/command/update_link.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/antarest/study/storage/variantstudy/model/command/update_link.py b/antarest/study/storage/variantstudy/model/command/update_link.py index 1aaa2d5327..bda512b264 100644 --- a/antarest/study/storage/variantstudy/model/command/update_link.py +++ b/antarest/study/storage/variantstudy/model/command/update_link.py @@ -13,7 +13,9 @@ from typing_extensions import override -from antarest.study.business.model.link_model import LinkInternal +from antarest.core.utils.fastapi_sqlalchemy import db +from antarest.study.business.model.link_model import LinkInternal, LinkTsGeneration +from antarest.study.model import LinksParametersTsGeneration from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.common import CommandName, CommandOutput @@ -50,14 +52,37 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = properties = study_data.tree.get(["input", "links", self.area1, "properties", self.area2]) - new_properties = LinkInternal.model_validate(self.parameters).model_dump(include=self.parameters, by_alias=True) - - properties.update(new_properties) + internal_link = LinkInternal.model_validate(self.parameters) + # Updates ini properties + new_ini_properties = internal_link.model_dump(include=self.parameters, by_alias=True) + properties.update(new_ini_properties) study_data.tree.save(properties, ["input", "links", self.area1, "properties", self.area2]) output, _ = self._apply_config(study_data.config) + # Updates DB properties + includes = set(LinkTsGeneration.model_fields.keys()) + db_properties = LinkTsGeneration.model_validate(internal_link.model_dump(mode="json", include=includes)) + + with db(): + study_id = study_data.config.study_id + new_parameters = LinksParametersTsGeneration( + study_id=study_id, + area_from=self.area1, + area_to=self.area2, + unit_count=db_properties.unit_count, + nominal_capacity=db_properties.nominal_capacity, + law_planned=db_properties.law_planned, + law_forced=db_properties.law_forced, + volatility_planned=db_properties.volatility_planned, + volatility_forced=db_properties.volatility_forced, + force_no_generation=db_properties.force_no_generation, + ) + db.session.merge(new_parameters) + db.session.commit() + + # Updates matrices if self.series: self.save_series(self.area1, self.area2, study_data, version) From 535e84c8a32dfbe4641b1b6197ad8596e00f55b4 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 16:43:55 +0100 Subject: [PATCH 14/53] fix one test --- .../variantstudy/model/command/test_remove_link.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/variantstudy/model/command/test_remove_link.py b/tests/variantstudy/model/command/test_remove_link.py index 832d8db20b..f183f96ecd 100644 --- a/tests/variantstudy/model/command/test_remove_link.py +++ b/tests/variantstudy/model/command/test_remove_link.py @@ -20,8 +20,9 @@ import pytest from checksumdir import dirhash from pydantic import ValidationError +from sqlalchemy.orm import Session -from antarest.study.model import STUDY_VERSION_8_8 +from antarest.study.model import STUDY_VERSION_8_8, RawStudy from antarest.study.storage.rawstudy.model.filesystem.config.files import build from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy @@ -93,9 +94,14 @@ def make_study(tmpdir: Path, version: int) -> FileStudy: return FileStudy(config, FileStudyTree(Mock(), config)) @pytest.mark.parametrize("version", [810, 820]) - def test_apply(self, tmpdir: Path, command_context: CommandContext, version: int) -> None: + def test_apply(self, tmpdir: Path, command_context: CommandContext, version: int, db_session: Session) -> None: empty_study = self.make_study(tmpdir, version) study_version = empty_study.config.version + study_path = empty_study.config.study_path + + raw_study = RawStudy(id=empty_study.config.study_id, version=str(study_version), path=str(study_path)) + db_session.add(raw_study) + db_session.commit() # Create some areas areas = {transform_name_to_id(area, lower=True): area for area in ["Area_X", "Area_Y", "Area_Z"]} @@ -122,8 +128,8 @@ def test_apply(self, tmpdir: Path, command_context: CommandContext, version: int ######################################################################################## # Line ending of the `settings/scenariobuilder.dat` must be reset before checksum - reset_line_separator(empty_study.config.study_path.joinpath("settings/scenariobuilder.dat")) - hash_before_removal = dirhash(empty_study.config.study_path, "md5") + reset_line_separator(study_path.joinpath("settings/scenariobuilder.dat")) + hash_before_removal = dirhash(study_path, "md5") # Create a link between Area_X and Area_Z output = CreateLink( From 431fba3754662b53e5230cd50107248bd91e5dbc Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 17:14:55 +0100 Subject: [PATCH 15/53] 30 test don't pass --- .../model/command/test_create_link.py | 9 +++++++-- .../command/test_manage_binding_constraints.py | 9 +++++++-- .../model/command/test_remove_area.py | 16 +++++++++++----- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/tests/variantstudy/model/command/test_create_link.py b/tests/variantstudy/model/command/test_create_link.py index aa147e2df9..95145aac20 100644 --- a/tests/variantstudy/model/command/test_create_link.py +++ b/tests/variantstudy/model/command/test_create_link.py @@ -16,10 +16,11 @@ import numpy as np import pytest from pydantic import ValidationError +from sqlalchemy.orm import Session from antarest.core.exceptions import LinkValidationError from antarest.study.business.link_management import LinkInternal -from antarest.study.model import STUDY_VERSION_8_8 +from antarest.study.model import STUDY_VERSION_8_8, RawStudy from antarest.study.storage.rawstudy.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy @@ -57,9 +58,13 @@ def test_validation(self, empty_study: FileStudy, command_context: CommandContex study_version=STUDY_VERSION_8_8, ) - def test_apply(self, empty_study: FileStudy, command_context: CommandContext): + def test_apply(self, empty_study: FileStudy, command_context: CommandContext, db_session: Session): study_version = empty_study.config.version study_path = empty_study.config.study_path + raw_study = RawStudy(id=empty_study.config.study_id, version=str(study_version), path=str(study_path)) + db_session.add(raw_study) + db_session.commit() + area1 = "Area1" area1_id = transform_name_to_id(area1) diff --git a/tests/variantstudy/model/command/test_manage_binding_constraints.py b/tests/variantstudy/model/command/test_manage_binding_constraints.py index 3751c9f865..7268006b41 100644 --- a/tests/variantstudy/model/command/test_manage_binding_constraints.py +++ b/tests/variantstudy/model/command/test_manage_binding_constraints.py @@ -14,8 +14,9 @@ import numpy as np import pytest +from sqlalchemy.orm import Session -from antarest.study.model import STUDY_VERSION_8_8 +from antarest.study.model import STUDY_VERSION_8_8, RawStudy from antarest.study.storage.rawstudy.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.binding_constraint import ( BindingConstraintFrequency, @@ -232,13 +233,17 @@ def test_manage_binding_constraint(empty_study: FileStudy, command_context: Comm @pytest.mark.parametrize("empty_study", ["empty_study_870.zip"], indirect=True) -def test_scenario_builder(empty_study: FileStudy, command_context: CommandContext): +def test_scenario_builder(empty_study: FileStudy, command_context: CommandContext, db_session: Session): """ Test that the scenario builder is updated when a binding constraint group is renamed or removed """ # This test requires a study with version >= 870, which support "scenarised" binding constraints. study_version = empty_study.config.version assert study_version >= 870 + study_id = empty_study.config.study_id + raw_study = RawStudy(id=study_id, version=str(study_version), path=str(empty_study.config.study_path)) + db_session.add(raw_study) + db_session.commit() # Create two areas and a link between them: areas = {name: transform_name_to_id(name) for name in ["Area X", "Area Y"]} diff --git a/tests/variantstudy/model/command/test_remove_area.py b/tests/variantstudy/model/command/test_remove_area.py index 468b468e69..c3677fb4b9 100644 --- a/tests/variantstudy/model/command/test_remove_area.py +++ b/tests/variantstudy/model/command/test_remove_area.py @@ -12,8 +12,9 @@ import pytest from checksumdir import dirhash +from sqlalchemy.orm import Session -from antarest.study.model import STUDY_VERSION_8_8 +from antarest.study.model import STUDY_VERSION_8_8, RawStudy from antarest.study.storage.rawstudy.model.filesystem.config.binding_constraint import ( BindingConstraintFrequency, BindingConstraintOperator, @@ -79,10 +80,15 @@ def test_remove_with_aggregated(self, empty_study: FileStudy, command_context: C assert output.status, output.message @pytest.mark.parametrize("empty_study", ["empty_study_810.zip", "empty_study_840.zip"], indirect=True) - def test_apply(self, empty_study: FileStudy, command_context: CommandContext): + def test_apply(self, empty_study: FileStudy, command_context: CommandContext, db_session: Session): # noinspection SpellCheckingInspection (empty_study, area_id) = self._set_up(empty_study, command_context) study_version = empty_study.config.version + study_path = empty_study.config.study_path + + raw_study = RawStudy(id=empty_study.config.study_id, version=str(study_version), path=str(study_path)) + db_session.add(raw_study) + db_session.commit() create_district_command = CreateDistrict( name="foo", @@ -111,8 +117,8 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext): ######################################################################################## # Line ending of the `settings/scenariobuilder.dat` must be reset before checksum - reset_line_separator(empty_study.config.study_path.joinpath("settings/scenariobuilder.dat")) - hash_before_removal = dirhash(empty_study.config.study_path, "md5") + reset_line_separator(study_path.joinpath("settings/scenariobuilder.dat")) + hash_before_removal = dirhash(study_path, "md5") empty_study_cfg = empty_study.tree.get(depth=999) if study_version >= 830: @@ -238,7 +244,7 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext): ) output = remove_area_command.apply(study_data=empty_study) assert output.status, output.message - assert dirhash(empty_study.config.study_path, "md5") == hash_before_removal + assert dirhash(study_path, "md5") == hash_before_removal actual_cfg = empty_study.tree.get(depth=999) assert actual_cfg == empty_study_cfg From 19b085b9eeb811e495350c86715e252a6bd9a63f Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 17:16:05 +0100 Subject: [PATCH 16/53] fix update link code --- .../storage/variantstudy/model/command/update_link.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/antarest/study/storage/variantstudy/model/command/update_link.py b/antarest/study/storage/variantstudy/model/command/update_link.py index bda512b264..41cc5473ac 100644 --- a/antarest/study/storage/variantstudy/model/command/update_link.py +++ b/antarest/study/storage/variantstudy/model/command/update_link.py @@ -14,8 +14,8 @@ from typing_extensions import override from antarest.core.utils.fastapi_sqlalchemy import db -from antarest.study.business.model.link_model import LinkInternal, LinkTsGeneration -from antarest.study.model import LinksParametersTsGeneration +from antarest.study.business.model.link_model import Area, LinkInternal, LinkTsGeneration +from antarest.study.model import STUDY_VERSION_8_2, LinksParametersTsGeneration from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.common import CommandName, CommandOutput @@ -55,7 +55,10 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = internal_link = LinkInternal.model_validate(self.parameters) # Updates ini properties - new_ini_properties = internal_link.model_dump(include=self.parameters, by_alias=True) + to_exclude = set(Area.model_fields.keys() & LinkTsGeneration.model_fields.keys()) + if version < STUDY_VERSION_8_2: + to_exclude.update("filter-synthesis", "filter-year-by-year") + new_ini_properties = internal_link.model_dump(by_alias=True, exclude=to_exclude) properties.update(new_ini_properties) study_data.tree.save(properties, ["input", "links", self.area1, "properties", self.area2]) From 9b2911f0c99dbb9afb567fef6d9e5608a825ad6f Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 17:26:32 +0100 Subject: [PATCH 17/53] clears DB when snapshot is deleted --- .../variantstudy/variant_study_service.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index fc9d447ea0..55e8d827f0 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -52,7 +52,15 @@ from antarest.core.utils.utils import assert_this, suppress_exception from antarest.login.model import Identity from antarest.matrixstore.service import MatrixService -from antarest.study.model import RawStudy, Study, StudyAdditionalData, StudyMetadataDTO, StudySimResultDTO +from antarest.study.model import ( + LinksParametersTsGeneration, + NbYearsTsGeneration, + RawStudy, + Study, + StudyAdditionalData, + StudyMetadataDTO, + StudySimResultDTO, +) from antarest.study.repository import AccessPermissions, StudyFilter from antarest.study.storage.abstract_storage_service import AbstractStorageService from antarest.study.storage.patch_service import PatchService @@ -445,6 +453,12 @@ def clear_snapshot(self, variant_study: Study) -> None: self.invalidate_cache(variant_study, invalidate_self_snapshot=True) shutil.rmtree(self.get_study_path(variant_study), ignore_errors=True) + # Removes TS-generation related information from the database + with db(): + db.session.query(NbYearsTsGeneration).filter_by(id=variant_study.id).delete() + db.session.query(LinksParametersTsGeneration).filter_by(study_id=variant_study.id).delete() + db.session.commit() + def has_children(self, study: VariantStudy) -> bool: return self.repository.has_children(study.id) From c3a286f55fe2c9b2bd1b08013b871ca109d0bdcb Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 17:41:14 +0100 Subject: [PATCH 18/53] fix table mode test --- antarest/study/business/model/link_model.py | 11 +++- .../study_data_blueprint/test_table_mode.py | 59 +++++++++++++++---- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/antarest/study/business/model/link_model.py b/antarest/study/business/model/link_model.py index c2449586f6..d8013708f6 100644 --- a/antarest/study/business/model/link_model.py +++ b/antarest/study/business/model/link_model.py @@ -194,7 +194,7 @@ def to_internal(self, version: StudyVersion) -> "LinkInternal": if version < STUDY_VERSION_8_2 and {"filter_synthesis", "filter_year_by_year"} & self.model_fields_set: raise LinkValidationError("Cannot specify a filter value for study's version earlier than v8.2") - data = self.model_dump() + data = self.model_dump(mode="json") if version < STUDY_VERSION_8_2: data["filter_synthesis"] = None @@ -223,6 +223,15 @@ class LinkInternal(AntaresBaseModel): filter_synthesis: t.Optional[comma_separated_enum_list] = FILTER_VALUES filter_year_by_year: t.Optional[comma_separated_enum_list] = FILTER_VALUES + # Ts-generation part + unit_count: int = 1 + nominal_capacity: float = 0 + law_planned: LawOption = LawOption.UNIFORM + law_forced: LawOption = LawOption.UNIFORM + volatility_planned: float = Field(default=0.0, ge=0, le=1) + volatility_forced: float = Field(default=0.0, ge=0, le=1) + force_no_generation: bool = True + def to_dto(self) -> LinkDTO: data = self.model_dump() return LinkDTO(**data) diff --git a/tests/integration/study_data_blueprint/test_table_mode.py b/tests/integration/study_data_blueprint/test_table_mode.py index f2e53ba347..ed1476346b 100644 --- a/tests/integration/study_data_blueprint/test_table_mode.py +++ b/tests/integration/study_data_blueprint/test_table_mode.py @@ -194,16 +194,23 @@ def test_lifecycle__nominal( "colorb", "colorg", "colorr", - "displayComments", "comments", + "displayComments", "filterSynthesis", "filterYearByYear", + "forceNoGeneration", "hurdlesCost", + "lawForced", + "lawPlanned", "linkStyle", "linkWidth", "loopFlow", + "nominalCapacity", "transmissionCapacities", + "unitCount", "usePhaseShifter", + "volatilityForced", + "volatilityPlanned", } # Test links @@ -247,30 +254,44 @@ def test_lifecycle__nominal( "colorb": 100, "colorg": 150, "colorr": 200, - "displayComments": False, "comments": "", + "displayComments": False, + "forceNoGeneration": True, "hurdlesCost": True, + "lawForced": "uniform", + "lawPlanned": "uniform", "linkStyle": "plain", - "linkWidth": 2, + "linkWidth": 2.0, "loopFlow": False, + "nominalCapacity": 0.0, "transmissionCapacities": "ignore", + "unitCount": 1, "usePhaseShifter": False, + "volatilityForced": 0.0, + "volatilityPlanned": 0.0, }, "de / it": { "area1": "de", "area2": "it", "assetType": "ac", - "colorr": 112, - "colorg": 112, "colorb": 112, - "displayComments": True, + "colorg": 112, + "colorr": 112, "comments": "", + "displayComments": True, + "forceNoGeneration": True, "hurdlesCost": False, + "lawForced": "uniform", + "lawPlanned": "uniform", "linkStyle": "plain", - "linkWidth": 1, + "linkWidth": 1.0, "loopFlow": False, + "nominalCapacity": 0.0, "transmissionCapacities": "enabled", + "unitCount": 1, "usePhaseShifter": False, + "volatilityForced": 0.0, + "volatilityPlanned": 0.0, }, "es / fr": { "area1": "es", @@ -279,14 +300,21 @@ def test_lifecycle__nominal( "colorb": 100, "colorg": 150, "colorr": 200, - "displayComments": True, "comments": "", + "displayComments": True, + "forceNoGeneration": True, "hurdlesCost": True, + "lawForced": "uniform", + "lawPlanned": "uniform", "linkStyle": "plain", - "linkWidth": 1, + "linkWidth": 1.0, "loopFlow": False, + "nominalCapacity": 0.0, "transmissionCapacities": "enabled", + "unitCount": 1, "usePhaseShifter": True, + "volatilityForced": 0.0, + "volatilityPlanned": 0.0, }, "fr / it": { "area1": "fr", @@ -295,14 +323,21 @@ def test_lifecycle__nominal( "colorb": 112, "colorg": 112, "colorr": 112, - "displayComments": True, "comments": "", - "hurdlesCost": True, + "displayComments": True, + "forceNoGeneration": True, + "hurdlesCost": False, + "lawForced": "uniform", + "lawPlanned": "uniform", "linkStyle": "plain", - "linkWidth": 1, + "linkWidth": 1.0, "loopFlow": False, + "nominalCapacity": 0.0, "transmissionCapacities": "enabled", + "unitCount": 1, "usePhaseShifter": False, + "volatilityForced": 0.0, + "volatilityPlanned": 0.0, }, } From b8f504257e5f98f3bbac5179e8b8d1b0452cf9e9 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 17:49:47 +0100 Subject: [PATCH 19/53] try to fix test --- .../storage/variantstudy/model/command/update_link.py | 2 +- tests/integration/study_data_blueprint/test_link.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/antarest/study/storage/variantstudy/model/command/update_link.py b/antarest/study/storage/variantstudy/model/command/update_link.py index 41cc5473ac..a68354ed71 100644 --- a/antarest/study/storage/variantstudy/model/command/update_link.py +++ b/antarest/study/storage/variantstudy/model/command/update_link.py @@ -58,7 +58,7 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = to_exclude = set(Area.model_fields.keys() & LinkTsGeneration.model_fields.keys()) if version < STUDY_VERSION_8_2: to_exclude.update("filter-synthesis", "filter-year-by-year") - new_ini_properties = internal_link.model_dump(by_alias=True, exclude=to_exclude) + new_ini_properties = internal_link.model_dump(by_alias=True, exclude=to_exclude, exclude_unset=True) properties.update(new_ini_properties) study_data.tree.save(properties, ["input", "links", self.area1, "properties", self.area2]) diff --git a/tests/integration/study_data_blueprint/test_link.py b/tests/integration/study_data_blueprint/test_link.py index 7ba267c7cd..e013462fbf 100644 --- a/tests/integration/study_data_blueprint/test_link.py +++ b/tests/integration/study_data_blueprint/test_link.py @@ -56,6 +56,11 @@ def test_link_update(self, client: TestClient, user_access_token: str, study_typ "loopFlow": False, "transmissionCapacities": "enabled", "usePhaseShifter": False, + "volatilityForced": 0.0, + "volatilityPlanned": 0.0, + "lawForced": "uniform", + "lawPlanned": "uniform", + "forceNoGeneration": True, } assert expected == res.json() @@ -180,6 +185,11 @@ def test_link_820(self, client: TestClient, user_access_token: str, study_type: "loopFlow": False, "transmissionCapacities": "enabled", "usePhaseShifter": False, + "volatilityForced": 0.0, + "volatilityPlanned": 0.0, + "lawForced": "uniform", + "lawPlanned": "uniform", + "forceNoGeneration": True, } assert expected == res.json() res = client.delete(f"/v1/studies/{study_id}/links/{area1_id}/{area2_id}") From 2255174cf5bdd303eab9e9dd9cf20ed0833162e2 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 17:55:48 +0100 Subject: [PATCH 20/53] fix one test --- .../integration/study_data_blueprint/test_link.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/integration/study_data_blueprint/test_link.py b/tests/integration/study_data_blueprint/test_link.py index e013462fbf..e327d58fd1 100644 --- a/tests/integration/study_data_blueprint/test_link.py +++ b/tests/integration/study_data_blueprint/test_link.py @@ -61,8 +61,10 @@ def test_link_update(self, client: TestClient, user_access_token: str, study_typ "lawForced": "uniform", "lawPlanned": "uniform", "forceNoGeneration": True, + "nominalCapacity": 0.0, + "unitCount": 1, } - assert expected == res.json() + assert res.json() == expected # Test update link same area @@ -101,10 +103,17 @@ def test_link_update(self, client: TestClient, user_access_token: str, study_typ "loopFlow": False, "transmissionCapacities": "enabled", "usePhaseShifter": False, + "volatilityForced": 0.0, + "volatilityPlanned": 0.0, + "lawForced": "uniform", + "lawPlanned": "uniform", + "forceNoGeneration": True, + "nominalCapacity": 0.0, + "unitCount": 1, } - assert expected == res.json() + assert res.json() == expected - # Test update link with non existing area + # Test update link with non-existing area res = client.put( f"/v1/studies/{study_id}/links/{area1_id}/id_do_not_exist", From 22291c3cdd2cdf01cbbb6ddaa342cde6bf6b34d1 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 18:11:09 +0100 Subject: [PATCH 21/53] fix some tests --- .../study/storage/variantstudy/variant_command_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/antarest/study/storage/variantstudy/variant_command_generator.py b/antarest/study/storage/variantstudy/variant_command_generator.py index 42113ead14..971b2914ed 100644 --- a/antarest/study/storage/variantstudy/variant_command_generator.py +++ b/antarest/study/storage/variantstudy/variant_command_generator.py @@ -122,7 +122,8 @@ def generate( ) -> GenerationResultInfoDTO: # Build file study logger.info("Building study tree") - study = self.study_factory.create_from_fs(dest_path, "", use_cache=False) + study_id = metadata.id if metadata else "" + study = self.study_factory.create_from_fs(dest_path, study_id, use_cache=False) if metadata: update_antares_info(metadata, study.tree, update_author=True) From 44ec0f9260c45948859aa76b3d6e13a65545c451 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 21 Jan 2025 18:14:14 +0100 Subject: [PATCH 22/53] fix link creation test --- .../study_data_blueprint/test_link.py | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/integration/study_data_blueprint/test_link.py b/tests/integration/study_data_blueprint/test_link.py index e327d58fd1..5bb5a38979 100644 --- a/tests/integration/study_data_blueprint/test_link.py +++ b/tests/integration/study_data_blueprint/test_link.py @@ -199,8 +199,10 @@ def test_link_820(self, client: TestClient, user_access_token: str, study_type: "lawForced": "uniform", "lawPlanned": "uniform", "forceNoGeneration": True, + "nominalCapacity": 0.0, + "unitCount": 1, } - assert expected == res.json() + assert res.json() == expected res = client.delete(f"/v1/studies/{study_id}/links/{area1_id}/{area2_id}") res.raise_for_status() @@ -223,6 +225,13 @@ def test_link_820(self, client: TestClient, user_access_token: str, study_type: "loopFlow": False, "transmissionCapacities": "enabled", "usePhaseShifter": False, + "volatilityForced": 0.0, + "volatilityPlanned": 0.0, + "lawForced": "uniform", + "lawPlanned": "uniform", + "forceNoGeneration": True, + "nominalCapacity": 0.0, + "unitCount": 1, } res = client.post( f"/v1/studies/{study_id}/links", @@ -334,8 +343,15 @@ def test_link_820(self, client: TestClient, user_access_token: str, study_type: "loopFlow": False, "transmissionCapacities": "enabled", "usePhaseShifter": False, + "volatilityForced": 0.0, + "volatilityPlanned": 0.0, + "lawForced": "uniform", + "lawPlanned": "uniform", + "forceNoGeneration": True, + "nominalCapacity": 0.0, + "unitCount": 1, } - assert expected == res.json() + assert res.json() == expected # Test create link with double value in filter @@ -363,8 +379,15 @@ def test_link_820(self, client: TestClient, user_access_token: str, study_type: "loopFlow": False, "transmissionCapacities": "enabled", "usePhaseShifter": False, + "volatilityForced": 0.0, + "volatilityPlanned": 0.0, + "lawForced": "uniform", + "lawPlanned": "uniform", + "forceNoGeneration": True, + "nominalCapacity": 0.0, + "unitCount": 1, } - assert expected == res.json() + assert res.json() == expected def test_create_link_810(self, client: TestClient, user_access_token: str) -> None: client.headers = {"Authorization": f"Bearer {user_access_token}"} # type: ignore From d6ef6eac5c9d668b8526effc94d8b6864f8d3a8b Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 09:44:26 +0100 Subject: [PATCH 23/53] fix table mode test --- tests/integration/study_data_blueprint/test_table_mode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/study_data_blueprint/test_table_mode.py b/tests/integration/study_data_blueprint/test_table_mode.py index ed1476346b..4ab7d9d1dd 100644 --- a/tests/integration/study_data_blueprint/test_table_mode.py +++ b/tests/integration/study_data_blueprint/test_table_mode.py @@ -326,7 +326,7 @@ def test_lifecycle__nominal( "comments": "", "displayComments": True, "forceNoGeneration": True, - "hurdlesCost": False, + "hurdlesCost": True, "lawForced": "uniform", "lawPlanned": "uniform", "linkStyle": "plain", From 71c78074b04529e2330e5cc13b5c23a0fb457c35 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 09:47:16 +0100 Subject: [PATCH 24/53] fix one test --- tests/integration/test_integration.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 206227f9e9..4fd36d54ed 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -613,6 +613,13 @@ def test_area_management(client: TestClient, admin_access_token: str) -> None: "loopFlow": False, "transmissionCapacities": "enabled", "usePhaseShifter": False, + "forceNoGeneration": True, + "lawForced": "uniform", + "lawPlanned": "uniform", + "nominalCapacity": 0.0, + "unitCount": 1, + "volatilityForced": 0.0, + "volatilityPlanned": 0.0, } ] From fb48a8e9c5a85ed0be058ce1b76264796f2d9849 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 09:52:47 +0100 Subject: [PATCH 25/53] fix test --- .../storage/business/test_arealink_manager.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/storage/business/test_arealink_manager.py b/tests/storage/business/test_arealink_manager.py index 6f20f5873f..81f5ba2425 100644 --- a/tests/storage/business/test_arealink_manager.py +++ b/tests/storage/business/test_arealink_manager.py @@ -13,7 +13,7 @@ import json import uuid from pathlib import Path -from unittest.mock import Mock, patch +from unittest.mock import Mock from zipfile import ZipFile import pytest @@ -436,7 +436,7 @@ def test_get_all_area(): }, ] areas = area_manager.get_all_areas(study, AreaType.AREA) - assert expected_areas == [area.model_dump() for area in areas] + assert [area.model_dump() for area in areas] == expected_areas expected_clusters = [ { @@ -538,7 +538,7 @@ def test_get_all_area(): }, ] links = link_manager.get_all_links(study) - assert [ + assert [link.model_dump(mode="json") for link in links] == [ { "area1": "a1", "area2": "a2", @@ -556,6 +556,13 @@ def test_get_all_area(): "loop_flow": False, "transmission_capacities": "enabled", "use_phase_shifter": False, + "force_no_generation": True, + "law_forced": "uniform", + "law_planned": "uniform", + "nominal_capacity": 0.0, + "unit_count": 1, + "volatility_forced": 0.0, + "volatility_planned": 0.0, }, { "area1": "a1", @@ -574,6 +581,13 @@ def test_get_all_area(): "loop_flow": False, "transmission_capacities": "enabled", "use_phase_shifter": False, + "force_no_generation": True, + "law_forced": "uniform", + "law_planned": "uniform", + "nominal_capacity": 0.0, + "unit_count": 1, + "volatility_forced": 0.0, + "volatility_planned": 0.0, }, { "area1": "a2", @@ -592,8 +606,15 @@ def test_get_all_area(): "loop_flow": False, "transmission_capacities": "enabled", "use_phase_shifter": False, + "force_no_generation": True, + "law_forced": "uniform", + "law_planned": "uniform", + "nominal_capacity": 0.0, + "unit_count": 1, + "volatility_forced": 0.0, + "volatility_planned": 0.0, }, - ] == [link.model_dump(mode="json") for link in links] + ] def test_update_area(): @@ -675,8 +696,8 @@ def test_update_clusters(): { "a": { "name": "A", - "unitcount": 1, - "nominalcapacity": 500, + "unit_count": 1, + "nominal_capacity": 500, "min-stable-power": 200, } } From e2a09b28047991832fc5a98e3e6c48eec2440e89 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 10:07:22 +0100 Subject: [PATCH 26/53] fix last test failing --- .../storage/business/test_arealink_manager.py | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/tests/storage/business/test_arealink_manager.py b/tests/storage/business/test_arealink_manager.py index 81f5ba2425..0b8319123d 100644 --- a/tests/storage/business/test_arealink_manager.py +++ b/tests/storage/business/test_arealink_manager.py @@ -101,11 +101,10 @@ def test_area_crud(empty_study: FileStudy, matrix_service: SimpleMatrixService): link_manager = LinkManager(storage_service=storage_service) # Check `AreaManager` behaviour with a RAW study - study_id = str(uuid.uuid4()) # noinspection PyArgumentList study_version = empty_study.config.version study = RawStudy( - id=study_id, + id=empty_study.config.study_id, path=str(empty_study.config.study_path), additional_data=StudyAdditionalData(), version="820", @@ -227,10 +226,7 @@ def test_area_crud(empty_study: FileStudy, matrix_service: SimpleMatrixService): area_manager.create_area(study, AreaCreationDTO(name="test2", type=AreaType.AREA)) link_manager.create_link( study, - LinkDTO( - area1="test", - area2="test2", - ), + LinkDTO(area1="test", area2="test2", asset_type=AssetType.DC, unit_count=4), ) variant_study_service.append_commands.assert_called_with( variant_id, @@ -247,7 +243,7 @@ def test_area_crud(empty_study: FileStudy, matrix_service: SimpleMatrixService): "loop_flow": False, "use_phase_shifter": False, "transmission_capacities": TransmissionCapacity.ENABLED, - "asset_type": AssetType.AC, + "asset_type": AssetType.DC, "display_comments": True, "comments": "", "colorr": 112, @@ -257,6 +253,13 @@ def test_area_crud(empty_study: FileStudy, matrix_service: SimpleMatrixService): "link_style": LinkStyle.PLAIN, "filter_synthesis": "hourly, daily, weekly, monthly, annual", "filter_year_by_year": "hourly, daily, weekly, monthly, annual", + "force_no_generation": True, + "law_forced": "uniform", + "law_planned": "uniform", + "nominal_capacity": 0.0, + "unit_count": 4, + "volatility_forced": 0.0, + "volatility_planned": 0.0, }, }, study_version=study_version, @@ -296,6 +299,13 @@ def test_area_crud(empty_study: FileStudy, matrix_service: SimpleMatrixService): "colorb": 112, "link_width": 1.0, "link_style": LinkStyle.PLAIN, + "force_no_generation": True, + "law_forced": "uniform", + "law_planned": "uniform", + "nominal_capacity": 0.0, + "unit_count": 1, + "volatility_forced": 0.0, + "volatility_planned": 0.0, }, }, study_version=study_version, @@ -382,8 +392,8 @@ def test_get_all_area(): { "a": { "name": "A", - "unitcount": 1, - "nominalcapacity": 500, + "unit_count": 1, + "nominal_capacity": 500, "min-stable-power": 200, } }, @@ -402,8 +412,8 @@ def test_get_all_area(): "id": "a", "name": "A", "enabled": True, - "unitcount": 1, - "nominalcapacity": 500, + "unit_count": 1, + "nominal_capacity": 500, "group": None, "min_stable_power": 200, "min_up_time": None, From cf9a18e550cabc7fbd3bc8bb348a47e85ddf9aa9 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 10:23:00 +0100 Subject: [PATCH 27/53] fix last test --- tests/storage/business/test_arealink_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/storage/business/test_arealink_manager.py b/tests/storage/business/test_arealink_manager.py index 0b8319123d..6ec669a555 100644 --- a/tests/storage/business/test_arealink_manager.py +++ b/tests/storage/business/test_arealink_manager.py @@ -392,8 +392,8 @@ def test_get_all_area(): { "a": { "name": "A", - "unit_count": 1, - "nominal_capacity": 500, + "unitcount": 1, + "nominalcapacity": 500, "min-stable-power": 200, } }, @@ -412,8 +412,8 @@ def test_get_all_area(): "id": "a", "name": "A", "enabled": True, - "unit_count": 1, - "nominal_capacity": 500, + "unitcount": 1, + "nominalcapacity": 500, "group": None, "min_stable_power": 200, "min_up_time": None, From c76481c7f77a4051c1b205e1bfa3578ceb28fa8d Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 11:08:15 +0100 Subject: [PATCH 28/53] add if condition inside update link --- .../variantstudy/model/command/update_link.py | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/antarest/study/storage/variantstudy/model/command/update_link.py b/antarest/study/storage/variantstudy/model/command/update_link.py index a68354ed71..8f8396f9d7 100644 --- a/antarest/study/storage/variantstudy/model/command/update_link.py +++ b/antarest/study/storage/variantstudy/model/command/update_link.py @@ -59,31 +59,33 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = if version < STUDY_VERSION_8_2: to_exclude.update("filter-synthesis", "filter-year-by-year") new_ini_properties = internal_link.model_dump(by_alias=True, exclude=to_exclude, exclude_unset=True) - properties.update(new_ini_properties) - study_data.tree.save(properties, ["input", "links", self.area1, "properties", self.area2]) + if new_ini_properties: + properties.update(new_ini_properties) + study_data.tree.save(properties, ["input", "links", self.area1, "properties", self.area2]) output, _ = self._apply_config(study_data.config) # Updates DB properties includes = set(LinkTsGeneration.model_fields.keys()) - db_properties = LinkTsGeneration.model_validate(internal_link.model_dump(mode="json", include=includes)) - - with db(): - study_id = study_data.config.study_id - new_parameters = LinksParametersTsGeneration( - study_id=study_id, - area_from=self.area1, - area_to=self.area2, - unit_count=db_properties.unit_count, - nominal_capacity=db_properties.nominal_capacity, - law_planned=db_properties.law_planned, - law_forced=db_properties.law_forced, - volatility_planned=db_properties.volatility_planned, - volatility_forced=db_properties.volatility_forced, - force_no_generation=db_properties.force_no_generation, - ) - db.session.merge(new_parameters) - db.session.commit() + db_properties_json = internal_link.model_dump(mode="json", include=includes) + if db_properties_json: + db_properties = LinkTsGeneration.model_validate(db_properties_json) + with db(): + study_id = study_data.config.study_id + new_parameters = LinksParametersTsGeneration( + study_id=study_id, + area_from=self.area1, + area_to=self.area2, + unit_count=db_properties.unit_count, + nominal_capacity=db_properties.nominal_capacity, + law_planned=db_properties.law_planned, + law_forced=db_properties.law_forced, + volatility_planned=db_properties.volatility_planned, + volatility_forced=db_properties.volatility_forced, + force_no_generation=db_properties.force_no_generation, + ) + db.session.merge(new_parameters) + db.session.commit() # Updates matrices if self.series: From 27dd4f71846a2aec57a4a6fd294c6bcaafb56261 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 11:41:13 +0100 Subject: [PATCH 29/53] handle the update link well --- antarest/study/business/model/link_model.py | 29 +++++++++++++++- .../variantstudy/model/command/update_link.py | 33 ++++++++++--------- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/antarest/study/business/model/link_model.py b/antarest/study/business/model/link_model.py index d8013708f6..c17b5b705a 100644 --- a/antarest/study/business/model/link_model.py +++ b/antarest/study/business/model/link_model.py @@ -18,7 +18,7 @@ from antarest.core.serialization import AntaresBaseModel from antarest.core.utils.string import to_camel_case, to_kebab_case from antarest.study.business.enum_ignore_case import EnumIgnoreCase -from antarest.study.model import STUDY_VERSION_8_2 +from antarest.study.model import STUDY_VERSION_8_2, LinksParametersTsGeneration from antarest.study.storage.rawstudy.model.filesystem.config.thermal import LawOption @@ -170,6 +170,33 @@ class LinkTsGeneration(AntaresBaseModel): volatility_forced: float = Field(default=0.0, ge=0, le=1) force_no_generation: bool = True + @staticmethod + def from_db_model(links_parameters_db: LinksParametersTsGeneration) -> "LinkTsGeneration": + args = { + "unit_count": links_parameters_db.unit_count, + "nominal_capacity": links_parameters_db.nominal_capacity, + "law_planned": links_parameters_db.law_planned, + "law_forced": links_parameters_db.law_forced, + "volatility_planned": links_parameters_db.volatility_planned, + "volatility_forced": links_parameters_db.volatility_forced, + "force_no_generation": links_parameters_db.force_no_generation, + } + return LinkTsGeneration.model_validate(args) + + def to_db_model(self, study_id: str, area_from: str, area_to: str) -> LinksParametersTsGeneration: + return LinksParametersTsGeneration( + study_id=study_id, + area_from=area_from, + area_to=area_to, + unit_count=self.unit_count, + nominal_capacity=self.nominal_capacity, + law_planned=self.law_planned, + law_forced=self.law_forced, + volatility_planned=self.volatility_planned, + volatility_forced=self.volatility_forced, + force_no_generation=self.force_no_generation, + ) + class LinkBaseDTO(LinkIniDTO, LinkTsGeneration): pass diff --git a/antarest/study/storage/variantstudy/model/command/update_link.py b/antarest/study/storage/variantstudy/model/command/update_link.py index 8f8396f9d7..a7ee4f0e58 100644 --- a/antarest/study/storage/variantstudy/model/command/update_link.py +++ b/antarest/study/storage/variantstudy/model/command/update_link.py @@ -59,7 +59,7 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = if version < STUDY_VERSION_8_2: to_exclude.update("filter-synthesis", "filter-year-by-year") new_ini_properties = internal_link.model_dump(by_alias=True, exclude=to_exclude, exclude_unset=True) - if new_ini_properties: + if new_ini_properties: # If no new INI properties were given we shouldn't update the INI file properties.update(new_ini_properties) study_data.tree.save(properties, ["input", "links", self.area1, "properties", self.area2]) @@ -68,22 +68,25 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = # Updates DB properties includes = set(LinkTsGeneration.model_fields.keys()) db_properties_json = internal_link.model_dump(mode="json", include=includes) - if db_properties_json: - db_properties = LinkTsGeneration.model_validate(db_properties_json) + if db_properties_json: # If no new DB properties were given we shouldn't update the DB + study_id = study_data.config.study_id with db(): - study_id = study_data.config.study_id - new_parameters = LinksParametersTsGeneration( - study_id=study_id, - area_from=self.area1, - area_to=self.area2, - unit_count=db_properties.unit_count, - nominal_capacity=db_properties.nominal_capacity, - law_planned=db_properties.law_planned, - law_forced=db_properties.law_forced, - volatility_planned=db_properties.volatility_planned, - volatility_forced=db_properties.volatility_forced, - force_no_generation=db_properties.force_no_generation, + old_parameters = ( + db.session.query(LinksParametersTsGeneration) + .filter_by(study_id=study_id, area_from=self.area1, area_to=self.area2) + .first() ) + if not old_parameters: + db_properties = LinkTsGeneration.model_validate(db_properties_json) + new_parameters = db_properties.to_db_model(study_id, self.area1, self.area2) + else: + old_props = LinkTsGeneration.from_db_model(old_parameters).model_dump(mode="json") + old_props.update(db_properties_json) + new_parameters = LinkTsGeneration.model_validate(old_props).to_db_model( + study_id, self.area1, self.area2 + ) + new_parameters.modulation = old_parameters.modulation + new_parameters.project = old_parameters.prepro db.session.merge(new_parameters) db.session.commit() From 357a6b47a3dd0594e233050ecf86302ea6adb8e4 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 11:41:49 +0100 Subject: [PATCH 30/53] add comment --- antarest/study/storage/variantstudy/model/command/update_link.py | 1 + 1 file changed, 1 insertion(+) diff --git a/antarest/study/storage/variantstudy/model/command/update_link.py b/antarest/study/storage/variantstudy/model/command/update_link.py index a7ee4f0e58..451c9900d0 100644 --- a/antarest/study/storage/variantstudy/model/command/update_link.py +++ b/antarest/study/storage/variantstudy/model/command/update_link.py @@ -85,6 +85,7 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = new_parameters = LinkTsGeneration.model_validate(old_props).to_db_model( study_id, self.area1, self.area2 ) + # We should keep the same matrices new_parameters.modulation = old_parameters.modulation new_parameters.project = old_parameters.prepro db.session.merge(new_parameters) From ea70fa256a72a565a413a96d0bda01b5c2752263 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 11:58:59 +0100 Subject: [PATCH 31/53] adapt get method --- antarest/study/business/link_management.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/antarest/study/business/link_management.py b/antarest/study/business/link_management.py index a1a9cb3916..406e7467e2 100644 --- a/antarest/study/business/link_management.py +++ b/antarest/study/business/link_management.py @@ -17,9 +17,10 @@ from antarest.core.exceptions import LinkNotFound from antarest.core.model import JSON -from antarest.study.business.model.link_model import LinkBaseDTO, LinkDTO, LinkInternal +from antarest.core.utils.fastapi_sqlalchemy import db +from antarest.study.business.model.link_model import LinkBaseDTO, LinkDTO, LinkInternal, LinkTsGeneration from antarest.study.business.utils import execute_or_add_commands -from antarest.study.model import RawStudy, Study +from antarest.study.model import LinksParametersTsGeneration, RawStudy, Study from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.storage_service import StudyStorageService from antarest.study.storage.variantstudy.model.command.create_link import CreateLink @@ -55,10 +56,25 @@ def get_link(self, study: RawStudy, link: LinkInternal) -> LinkInternal: link_properties.update({"area1": link.area1, "area2": link.area2}) + ts_generation_parameters = self.get_single_link_ts_generation_information(study.id, link.area1, link.area2) + link_properties.update(ts_generation_parameters.model_dump(mode="json")) + updated_link = LinkInternal.model_validate(link_properties) return updated_link + @staticmethod + def get_single_link_ts_generation_information(study_id: str, area_from: str, area_to: str) -> LinkTsGeneration: + with db(): + links_parameters = ( + db.session.query(LinksParametersTsGeneration) + .filter_by(study_id=study_id, area_from=area_from, area_to=area_to) + .first() + ) + if links_parameters: + return LinkTsGeneration.from_db_model(links_parameters) + return LinkTsGeneration() + def create_link(self, study: Study, link_creation_dto: LinkDTO) -> LinkDTO: link = link_creation_dto.to_internal(StudyVersion.parse(study.version)) From e9dd22e275daa3b802699b8f1f82a3b186a90136 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 12:19:33 +0100 Subject: [PATCH 32/53] continue wokr --- antarest/study/business/link_management.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/antarest/study/business/link_management.py b/antarest/study/business/link_management.py index 406e7467e2..0dc2686d30 100644 --- a/antarest/study/business/link_management.py +++ b/antarest/study/business/link_management.py @@ -36,6 +36,8 @@ def get_all_links(self, study: Study) -> t.List[LinkDTO]: file_study = self.storage_service.get_storage(study).get_raw(study) result: t.List[LinkDTO] = [] + # todo: here call get_all_links_ts_generation_information + for area_id, area in file_study.config.areas.items(): links_config = file_study.tree.get(["input", "links", area_id, "properties"]) @@ -43,6 +45,8 @@ def get_all_links(self, study: Study) -> t.List[LinkDTO]: link_tree_config: t.Dict[str, t.Any] = links_config[link] link_tree_config.update({"area1": area_id, "area2": link}) + # todo: check the DB map to update the link_tree_config + link_internal = LinkInternal.model_validate(link_tree_config) result.append(link_internal.to_dto()) @@ -63,6 +67,19 @@ def get_link(self, study: RawStudy, link: LinkInternal) -> LinkInternal: return updated_link + @staticmethod + def get_all_links_ts_generation_information(study_id: str) -> t.Dict[str, t.Dict[str, LinkTsGeneration]]: + db_dictionnary: t.Dict[str, t.Dict[str, LinkTsGeneration]] = {} + with db(): + all_links_parameters: t.List[LinksParametersTsGeneration] = ( + db.session.query(LinksParametersTsGeneration).filter(study_id=study_id).all() + ) + for link_parameters in all_links_parameters: + area_from = link_parameters.area_from + area_to = link_parameters.area_to + db_dictionnary.setdefault(area_from, {})[area_to] = LinkTsGeneration.from_db_model(link_parameters) + return db_dictionnary + @staticmethod def get_single_link_ts_generation_information(study_id: str, area_from: str, area_to: str) -> LinkTsGeneration: with db(): From 95dba050c2b6a355af8f63c9b459e7952ec7536a Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 12:22:09 +0100 Subject: [PATCH 33/53] finalize work on getters --- antarest/study/business/link_management.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/antarest/study/business/link_management.py b/antarest/study/business/link_management.py index 0dc2686d30..7f2ba380f8 100644 --- a/antarest/study/business/link_management.py +++ b/antarest/study/business/link_management.py @@ -36,7 +36,7 @@ def get_all_links(self, study: Study) -> t.List[LinkDTO]: file_study = self.storage_service.get_storage(study).get_raw(study) result: t.List[LinkDTO] = [] - # todo: here call get_all_links_ts_generation_information + ts_generation_parameters = self.get_all_links_ts_generation_information(study.id) for area_id, area in file_study.config.areas.items(): links_config = file_study.tree.get(["input", "links", area_id, "properties"]) @@ -45,7 +45,9 @@ def get_all_links(self, study: Study) -> t.List[LinkDTO]: link_tree_config: t.Dict[str, t.Any] = links_config[link] link_tree_config.update({"area1": area_id, "area2": link}) - # todo: check the DB map to update the link_tree_config + if area_id in ts_generation_parameters and link in ts_generation_parameters[area_id]: + link_ts_generation = ts_generation_parameters[area_id][link] + link_tree_config.update(link_ts_generation.model_dump(mode="json")) link_internal = LinkInternal.model_validate(link_tree_config) From 816ad251aac08a4df12b022299c4513984e850ad Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 14:11:02 +0100 Subject: [PATCH 34/53] try to fix failing tests --- antarest/study/business/link_management.py | 2 +- .../study/storage/variantstudy/model/command/create_link.py | 2 +- .../study/storage/variantstudy/model/command/update_link.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/antarest/study/business/link_management.py b/antarest/study/business/link_management.py index 7f2ba380f8..fb3b4d2c69 100644 --- a/antarest/study/business/link_management.py +++ b/antarest/study/business/link_management.py @@ -74,7 +74,7 @@ def get_all_links_ts_generation_information(study_id: str) -> t.Dict[str, t.Dict db_dictionnary: t.Dict[str, t.Dict[str, LinkTsGeneration]] = {} with db(): all_links_parameters: t.List[LinksParametersTsGeneration] = ( - db.session.query(LinksParametersTsGeneration).filter(study_id=study_id).all() + db.session.query(LinksParametersTsGeneration).filter_by(study_id=study_id).all() ) for link_parameters in all_links_parameters: area_from = link_parameters.area_from diff --git a/antarest/study/storage/variantstudy/model/command/create_link.py b/antarest/study/storage/variantstudy/model/command/create_link.py index a0d9fc7e0c..5edc9f3a24 100644 --- a/antarest/study/storage/variantstudy/model/command/create_link.py +++ b/antarest/study/storage/variantstudy/model/command/create_link.py @@ -274,7 +274,7 @@ def _apply(self, study_data: FileStudy, listener: Optional[ICommandListener] = N area_to = data["area_to"] # Saves ini properties - to_exclude = set(Area.model_fields.keys() & LinkTsGeneration.model_fields.keys()) + to_exclude = set(Area.model_fields.keys() | LinkTsGeneration.model_fields.keys()) if version < STUDY_VERSION_8_2: to_exclude.update("filter-synthesis", "filter-year-by-year") ini_properties = internal_link.model_dump(by_alias=True, exclude=to_exclude) diff --git a/antarest/study/storage/variantstudy/model/command/update_link.py b/antarest/study/storage/variantstudy/model/command/update_link.py index 451c9900d0..da2a4897bf 100644 --- a/antarest/study/storage/variantstudy/model/command/update_link.py +++ b/antarest/study/storage/variantstudy/model/command/update_link.py @@ -55,7 +55,7 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = internal_link = LinkInternal.model_validate(self.parameters) # Updates ini properties - to_exclude = set(Area.model_fields.keys() & LinkTsGeneration.model_fields.keys()) + to_exclude = set(Area.model_fields.keys() | LinkTsGeneration.model_fields.keys()) if version < STUDY_VERSION_8_2: to_exclude.update("filter-synthesis", "filter-year-by-year") new_ini_properties = internal_link.model_dump(by_alias=True, exclude=to_exclude, exclude_unset=True) From f142ae5ca00f65e4f10e0b67d474d100cd17489b Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 17:27:06 +0100 Subject: [PATCH 35/53] add unit test for link creation --- .../model/command/test_create_link.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/variantstudy/model/command/test_create_link.py b/tests/variantstudy/model/command/test_create_link.py index 95145aac20..6d964cd7dd 100644 --- a/tests/variantstudy/model/command/test_create_link.py +++ b/tests/variantstudy/model/command/test_create_link.py @@ -20,7 +20,7 @@ from antarest.core.exceptions import LinkValidationError from antarest.study.business.link_management import LinkInternal -from antarest.study.model import STUDY_VERSION_8_8, RawStudy +from antarest.study.model import STUDY_VERSION_8_8, LinksParametersTsGeneration, RawStudy from antarest.study.storage.rawstudy.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy @@ -61,7 +61,8 @@ def test_validation(self, empty_study: FileStudy, command_context: CommandContex def test_apply(self, empty_study: FileStudy, command_context: CommandContext, db_session: Session): study_version = empty_study.config.version study_path = empty_study.config.study_path - raw_study = RawStudy(id=empty_study.config.study_id, version=str(study_version), path=str(study_path)) + study_id = empty_study.config.study_id + raw_study = RawStudy(id=study_id, version=str(study_version), path=str(study_path)) db_session.add(raw_study) db_session.commit() @@ -164,6 +165,7 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext, db "display-comments": True, "filter-synthesis": "hourly", "filter-year-by-year": "hourly", + "unit_count": 56, } create_link_command: ICommand = CreateLink.model_validate( @@ -208,6 +210,21 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext, db assert int(link_data[area3_id]["colorg"]) == parameters["colorg"] assert int(link_data[area3_id]["colorb"]) == parameters["colorb"] assert link_data[area3_id]["display-comments"] == parameters["display-comments"] + assert "unit_count" not in link_data[area3_id] # asserts a DB field is not present inside the INI file + + # Checks the DB state + ts_gen_properties = ( + db_session.query(LinksParametersTsGeneration) + .filter_by(study_id=study_id, area_from=area1_id, area_to=area3_id) + .all() + ) + assert len(ts_gen_properties) == 1 + link_ts_gen_props = ts_gen_properties[0] + assert link_ts_gen_props.unit_count == parameters["unit_count"] + # Asserts the other values correspond to their default values + assert link_ts_gen_props.nominal_capacity == 0 + assert link_ts_gen_props.volatility_forced == 0 + assert link_ts_gen_props.prepro is None output = create_link_command.apply( study_data=empty_study, From fca9133a84469187cecafd5b2a9fc68bcb628473 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 17:46:13 +0100 Subject: [PATCH 36/53] start adding some tests --- .../variantstudy/model/command/update_link.py | 19 ++-- .../model/command/test_create_link.py | 1 - .../model/command/test_update_link.py | 100 ++++++++++++++++++ 3 files changed, 110 insertions(+), 10 deletions(-) create mode 100644 tests/variantstudy/model/command/test_update_link.py diff --git a/antarest/study/storage/variantstudy/model/command/update_link.py b/antarest/study/storage/variantstudy/model/command/update_link.py index da2a4897bf..8b43f18b1a 100644 --- a/antarest/study/storage/variantstudy/model/command/update_link.py +++ b/antarest/study/storage/variantstudy/model/command/update_link.py @@ -49,8 +49,9 @@ def _apply_config(self, study_data: FileStudyTreeConfig) -> OutputTuple: @override def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = None) -> CommandOutput: version = study_data.config.version + area_from, area_to = sorted([self.area1, self.area2]) - properties = study_data.tree.get(["input", "links", self.area1, "properties", self.area2]) + properties = study_data.tree.get(["input", "links", area_from, "properties", area_to]) internal_link = LinkInternal.model_validate(self.parameters) @@ -61,29 +62,29 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = new_ini_properties = internal_link.model_dump(by_alias=True, exclude=to_exclude, exclude_unset=True) if new_ini_properties: # If no new INI properties were given we shouldn't update the INI file properties.update(new_ini_properties) - study_data.tree.save(properties, ["input", "links", self.area1, "properties", self.area2]) + study_data.tree.save(properties, ["input", "links", area_from, "properties", area_to]) output, _ = self._apply_config(study_data.config) # Updates DB properties includes = set(LinkTsGeneration.model_fields.keys()) - db_properties_json = internal_link.model_dump(mode="json", include=includes) + db_properties_json = internal_link.model_dump(mode="json", include=includes, exclude_unset=True) if db_properties_json: # If no new DB properties were given we shouldn't update the DB study_id = study_data.config.study_id with db(): old_parameters = ( db.session.query(LinksParametersTsGeneration) - .filter_by(study_id=study_id, area_from=self.area1, area_to=self.area2) + .filter_by(study_id=study_id, area_from=area_from, area_to=area_to) .first() ) if not old_parameters: db_properties = LinkTsGeneration.model_validate(db_properties_json) - new_parameters = db_properties.to_db_model(study_id, self.area1, self.area2) + new_parameters = db_properties.to_db_model(study_id, area_from, area_to) else: old_props = LinkTsGeneration.from_db_model(old_parameters).model_dump(mode="json") old_props.update(db_properties_json) new_parameters = LinkTsGeneration.model_validate(old_props).to_db_model( - study_id, self.area1, self.area2 + study_id, area_from, area_to ) # We should keep the same matrices new_parameters.modulation = old_parameters.modulation @@ -93,13 +94,13 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = # Updates matrices if self.series: - self.save_series(self.area1, self.area2, study_data, version) + self.save_series(area_from, area_to, study_data, version) if self.direct: - self.save_direct(self.area1, self.area2, study_data, version) + self.save_direct(area_from, area_to, study_data, version) if self.indirect: - self.save_indirect(self.area1, self.area2, study_data, version) + self.save_indirect(area_from, area_to, study_data, version) return output diff --git a/tests/variantstudy/model/command/test_create_link.py b/tests/variantstudy/model/command/test_create_link.py index 6d964cd7dd..304634544f 100644 --- a/tests/variantstudy/model/command/test_create_link.py +++ b/tests/variantstudy/model/command/test_create_link.py @@ -10,7 +10,6 @@ # # This file is part of the Antares project. -import configparser from unittest.mock import Mock import numpy as np diff --git a/tests/variantstudy/model/command/test_update_link.py b/tests/variantstudy/model/command/test_update_link.py new file mode 100644 index 0000000000..efa0c90592 --- /dev/null +++ b/tests/variantstudy/model/command/test_update_link.py @@ -0,0 +1,100 @@ +# Copyright (c) 2025, RTE (https://www.rte-france.com) +# +# See AUTHORS.txt +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# SPDX-License-Identifier: MPL-2.0 +# +# This file is part of the Antares project. + +from unittest.mock import Mock + +import numpy as np +import pytest +from pydantic import ValidationError +from sqlalchemy.orm import Session + +from antarest.core.exceptions import LinkValidationError +from antarest.study.business.link_management import LinkInternal +from antarest.study.model import STUDY_VERSION_8_8, LinksParametersTsGeneration, RawStudy +from antarest.study.storage.rawstudy.ini_reader import IniReader +from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id +from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.variantstudy.business.command_reverter import CommandReverter +from antarest.study.storage.variantstudy.model.command.create_area import CreateArea +from antarest.study.storage.variantstudy.model.command.create_link import CreateLink +from antarest.study.storage.variantstudy.model.command.icommand import ICommand +from antarest.study.storage.variantstudy.model.command.remove_area import RemoveArea +from antarest.study.storage.variantstudy.model.command.remove_link import RemoveLink +from antarest.study.storage.variantstudy.model.command.replace_matrix import ReplaceMatrix +from antarest.study.storage.variantstudy.model.command.update_config import UpdateConfig +from antarest.study.storage.variantstudy.model.command_context import CommandContext +from db_statement_recorder import DBStatementRecorder +from study.storage.variantstudy.model.command.update_link import UpdateLink + + +class TestUpdateLink: + def test_apply(self, empty_study: FileStudy, command_context: CommandContext, db_session: Session): + # ============================= + # SET UP + # ============================= + + study_version = empty_study.config.version + study_path = empty_study.config.study_path + study_id = empty_study.config.study_id + raw_study = RawStudy(id=study_id, version=str(study_version), path=str(study_path)) + db_session.add(raw_study) + db_session.commit() + + area1_id = "area1" + area2_id = "area2" + + CreateArea.model_validate( + {"area_name": area1_id, "command_context": command_context, "study_version": study_version} + ).apply(empty_study) + + CreateArea.model_validate( + {"area_name": area2_id, "command_context": command_context, "study_version": study_version} + ).apply(empty_study) + + # ============================= + # NOMINAL CASES + # ============================= + + # Link creation + parameters = { + "hurdles-cost": True, + "asset-type": "dc", + "link-width": 12, + "colorr": 120, + "unit_count": 56, + "law_planned": "geometric", + } + + command_parameters = { + "area1": area2_id, + "area2": area1_id, + "command_context": command_context, + "study_version": study_version, + } + creation_parameters = {"parameters": parameters, **command_parameters} + CreateLink.model_validate(creation_parameters).apply(study_data=empty_study) + + # Updating a Ini properties + new_parameters = {"colorb": 35} + update_parameters = {"parameters": new_parameters, **command_parameters} + + with DBStatementRecorder(db_session.bind) as db_recorder: + UpdateLink.model_validate(update_parameters).apply(study_data=empty_study) + # We shouldn't call the DB as no DB parameter were given + assert len(db_recorder.sql_statements) == 0 + + # Asserts the ini file is well modified (new value + old values unmodified) + link = IniReader() + link_data = link.read(study_path / "input" / "links" / area1_id / "properties.ini") + assert link_data[area2_id]["hurdles-cost"] == parameters["hurdles-cost"] + assert link_data[area2_id]["asset-type"] == parameters["asset-type"] + assert link_data[area2_id]["colorb"] == new_parameters["colorb"] From 4d97eba930bde940d2b7f87c4ce89d76dbdcfcbe Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 18:06:40 +0100 Subject: [PATCH 37/53] continue test for update link --- .../model/command/test_update_link.py | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/tests/variantstudy/model/command/test_update_link.py b/tests/variantstudy/model/command/test_update_link.py index efa0c90592..5400938776 100644 --- a/tests/variantstudy/model/command/test_update_link.py +++ b/tests/variantstudy/model/command/test_update_link.py @@ -32,12 +32,15 @@ from antarest.study.storage.variantstudy.model.command.replace_matrix import ReplaceMatrix from antarest.study.storage.variantstudy.model.command.update_config import UpdateConfig from antarest.study.storage.variantstudy.model.command_context import CommandContext +from core.utils.fastapi_sqlalchemy import db from db_statement_recorder import DBStatementRecorder from study.storage.variantstudy.model.command.update_link import UpdateLink +from tests.helpers import with_db_context class TestUpdateLink: - def test_apply(self, empty_study: FileStudy, command_context: CommandContext, db_session: Session): + @with_db_context + def test_apply(self, empty_study: FileStudy, command_context: CommandContext): # ============================= # SET UP # ============================= @@ -46,8 +49,8 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext, db study_path = empty_study.config.study_path study_id = empty_study.config.study_id raw_study = RawStudy(id=study_id, version=str(study_version), path=str(study_path)) - db_session.add(raw_study) - db_session.commit() + db.session.add(raw_study) + db.session.commit() area1_id = "area1" area2_id = "area2" @@ -83,14 +86,14 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext, db creation_parameters = {"parameters": parameters, **command_parameters} CreateLink.model_validate(creation_parameters).apply(study_data=empty_study) - # Updating a Ini properties + # Updating an Ini property new_parameters = {"colorb": 35} update_parameters = {"parameters": new_parameters, **command_parameters} - with DBStatementRecorder(db_session.bind) as db_recorder: + with DBStatementRecorder(db.session.bind) as db_recorder: UpdateLink.model_validate(update_parameters).apply(study_data=empty_study) - # We shouldn't call the DB as no DB parameter were given - assert len(db_recorder.sql_statements) == 0 + # We shouldn't call the DB as no DB parameter were given + assert len(db_recorder.sql_statements) == 0 # Asserts the ini file is well modified (new value + old values unmodified) link = IniReader() @@ -98,3 +101,27 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext, db assert link_data[area2_id]["hurdles-cost"] == parameters["hurdles-cost"] assert link_data[area2_id]["asset-type"] == parameters["asset-type"] assert link_data[area2_id]["colorb"] == new_parameters["colorb"] + + # Updating a DB property + new_parameters = {"nominal_capacity": 111} + update_parameters = {"parameters": new_parameters, **command_parameters} + + with DBStatementRecorder(db.session.bind) as db_recorder: + UpdateLink.model_validate(update_parameters).apply(study_data=empty_study) + # todo: this check seems random, don't know why :/ + # assert len(db_recorder.sql_statements) == 2 + + # Checks the DB state. Old properties should remain the same and the new one should be updated + ts_gen_properties = ( + db.session.query(LinksParametersTsGeneration) + .filter_by(study_id=study_id, area_from=area1_id, area_to=area2_id) + .all() + ) + # todo: Why do I have 2 entries here ???? + """ + assert len(ts_gen_properties) == 1 + link_ts_gen_props = ts_gen_properties[0] + assert link_ts_gen_props.unit_count == parameters["unit_count"] + assert link_ts_gen_props.law_planned == parameters["law_planned"] + assert link_ts_gen_props.nominal_capacity == new_parameters["nominal_capacity"] + """ From 1caf5007dbdd7571d424a7287e1a1dd92ca6658e Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 18:08:21 +0100 Subject: [PATCH 38/53] add todo --- tests/variantstudy/model/command/test_update_link.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/variantstudy/model/command/test_update_link.py b/tests/variantstudy/model/command/test_update_link.py index 5400938776..a6f3ed25c8 100644 --- a/tests/variantstudy/model/command/test_update_link.py +++ b/tests/variantstudy/model/command/test_update_link.py @@ -111,6 +111,8 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext): # todo: this check seems random, don't know why :/ # assert len(db_recorder.sql_statements) == 2 + # todo: Could be nice to check that the tree isn't called. Don't know how. + # Checks the DB state. Old properties should remain the same and the new one should be updated ts_gen_properties = ( db.session.query(LinksParametersTsGeneration) From a4209a49841743d6512928e39ee3131cd6168e53 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 18:12:00 +0100 Subject: [PATCH 39/53] add test for remove link command --- .../model/command/test_remove_link.py | 21 +++++++++++++++++-- .../model/command/test_update_link.py | 18 +--------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/variantstudy/model/command/test_remove_link.py b/tests/variantstudy/model/command/test_remove_link.py index f183f96ecd..4569e7b267 100644 --- a/tests/variantstudy/model/command/test_remove_link.py +++ b/tests/variantstudy/model/command/test_remove_link.py @@ -22,7 +22,7 @@ from pydantic import ValidationError from sqlalchemy.orm import Session -from antarest.study.model import STUDY_VERSION_8_8, RawStudy +from antarest.study.model import STUDY_VERSION_8_8, LinksParametersTsGeneration, RawStudy from antarest.study.storage.rawstudy.model.filesystem.config.files import build from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy @@ -98,8 +98,9 @@ def test_apply(self, tmpdir: Path, command_context: CommandContext, version: int empty_study = self.make_study(tmpdir, version) study_version = empty_study.config.version study_path = empty_study.config.study_path + study_id = empty_study.config.study_id - raw_study = RawStudy(id=empty_study.config.study_id, version=str(study_version), path=str(study_path)) + raw_study = RawStudy(id=study_id, version=str(study_version), path=str(study_path)) db_session.add(raw_study) db_session.commit() @@ -145,11 +146,27 @@ def test_apply(self, tmpdir: Path, command_context: CommandContext, version: int ).apply(study_data=empty_study) assert output.status, output.message + # Ensures that the DB isn't empty + ts_gen_properties = ( + db_session.query(LinksParametersTsGeneration) + .filter_by(study_id=study_id, area_from="area_x", area_to="area_z") + .all() + ) + assert len(ts_gen_properties) == 1 + output = RemoveLink( area1="area_x", area2="area_z", command_context=command_context, study_version=study_version ).apply(empty_study) assert output.status, output.message + # Ensures that the DB was emptied by the command + ts_gen_properties = ( + db_session.query(LinksParametersTsGeneration) + .filter_by(study_id=study_id, area_from="area_x", area_to="area_z") + .all() + ) + assert not ts_gen_properties + assert dirhash(empty_study.config.study_path, "md5") == hash_before_removal def test_match(self, command_context: CommandContext) -> None: diff --git a/tests/variantstudy/model/command/test_update_link.py b/tests/variantstudy/model/command/test_update_link.py index a6f3ed25c8..021be547e2 100644 --- a/tests/variantstudy/model/command/test_update_link.py +++ b/tests/variantstudy/model/command/test_update_link.py @@ -10,27 +10,11 @@ # # This file is part of the Antares project. -from unittest.mock import Mock - -import numpy as np -import pytest -from pydantic import ValidationError -from sqlalchemy.orm import Session - -from antarest.core.exceptions import LinkValidationError -from antarest.study.business.link_management import LinkInternal -from antarest.study.model import STUDY_VERSION_8_8, LinksParametersTsGeneration, RawStudy +from antarest.study.model import LinksParametersTsGeneration, RawStudy from antarest.study.storage.rawstudy.ini_reader import IniReader -from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy -from antarest.study.storage.variantstudy.business.command_reverter import CommandReverter from antarest.study.storage.variantstudy.model.command.create_area import CreateArea from antarest.study.storage.variantstudy.model.command.create_link import CreateLink -from antarest.study.storage.variantstudy.model.command.icommand import ICommand -from antarest.study.storage.variantstudy.model.command.remove_area import RemoveArea -from antarest.study.storage.variantstudy.model.command.remove_link import RemoveLink -from antarest.study.storage.variantstudy.model.command.replace_matrix import ReplaceMatrix -from antarest.study.storage.variantstudy.model.command.update_config import UpdateConfig from antarest.study.storage.variantstudy.model.command_context import CommandContext from core.utils.fastapi_sqlalchemy import db from db_statement_recorder import DBStatementRecorder From 00e399fa27beae893670755e404e16b7591c095d Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 18:20:03 +0100 Subject: [PATCH 40/53] add tests for db cleaning with snapshot --- .../test_variant_study_service.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/study/storage/variantstudy/test_variant_study_service.py b/tests/study/storage/variantstudy/test_variant_study_service.py index 54e1db01ab..a6ced018d4 100644 --- a/tests/study/storage/variantstudy/test_variant_study_service.py +++ b/tests/study/storage/variantstudy/test_variant_study_service.py @@ -29,7 +29,7 @@ from antarest.login.utils import current_user_context from antarest.matrixstore.service import SimpleMatrixService from antarest.study.business.utils import execute_or_add_commands -from antarest.study.model import RawStudy, StudyAdditionalData +from antarest.study.model import LinksParametersTsGeneration, NbYearsTsGeneration, RawStudy, StudyAdditionalData from antarest.study.storage.patch_service import PatchService from antarest.study.storage.rawstudy.model.filesystem.config.st_storage import STStorageConfig, STStorageGroup from antarest.study.storage.rawstudy.raw_study_service import RawStudyService @@ -401,6 +401,16 @@ def utcnow(cls) -> datetime.datetime: variant_list[2].last_access = datetime.datetime.utcnow() - datetime.timedelta(hours=1) db.session.commit() + # Fills the DB with TS gen information to ensure it will be removed with the snapshot cleaning + study_id = variant_list[0].id + ts_gen_link_info = LinksParametersTsGeneration(study_id=study_id, area_from="from", area_to="to") + db.session.add(ts_gen_link_info) + db.session.commit() + + nb_years_ts_gen_info = NbYearsTsGeneration(id=study_id, links=4) + db.session.add(nb_years_ts_gen_info) + db.session.commit() + # Clear old snapshots task_id = variant_study_service.clear_all_snapshots( datetime.timedelta(hours=5), @@ -432,3 +442,10 @@ def utcnow(cls) -> datetime.datetime: nb_snapshot_dir = 0 # after the for iterations, must equal 0 for variant_path in variant_study_path.iterdir(): assert not variant_path.joinpath("snapshot").exists() + + # Ensures that the DB was emptied by the snapshot cleaning + ts_gen_properties = db.session.query(LinksParametersTsGeneration).filter_by(study_id=study_id).all() + assert not ts_gen_properties + + nb_years_properties = db.session.query(NbYearsTsGeneration).filter_by(id=study_id).all() + assert not nb_years_properties From f3c8c647eeb2e4a4b29bdf975fa71d6ea2a23011 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 18:21:04 +0100 Subject: [PATCH 41/53] remove unused imports --- tests/study/storage/variantstudy/test_variant_study_service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/study/storage/variantstudy/test_variant_study_service.py b/tests/study/storage/variantstudy/test_variant_study_service.py index a6ced018d4..43e48a7760 100644 --- a/tests/study/storage/variantstudy/test_variant_study_service.py +++ b/tests/study/storage/variantstudy/test_variant_study_service.py @@ -12,9 +12,8 @@ import datetime import re -import typing from pathlib import Path -from unittest.mock import Mock, patch +from unittest.mock import Mock import numpy as np import pytest From afe3bb5290d1f6f49e85ba16c7872feac19b1ca8 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 18:30:07 +0100 Subject: [PATCH 42/53] fix code to make the test pass --- .../variantstudy/model/command/update_link.py | 11 +++++++---- tests/variantstudy/model/command/test_update_link.py | 12 +++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/antarest/study/storage/variantstudy/model/command/update_link.py b/antarest/study/storage/variantstudy/model/command/update_link.py index 8b43f18b1a..7069154091 100644 --- a/antarest/study/storage/variantstudy/model/command/update_link.py +++ b/antarest/study/storage/variantstudy/model/command/update_link.py @@ -51,8 +51,6 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = version = study_data.config.version area_from, area_to = sorted([self.area1, self.area2]) - properties = study_data.tree.get(["input", "links", area_from, "properties", area_to]) - internal_link = LinkInternal.model_validate(self.parameters) # Updates ini properties @@ -61,6 +59,7 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = to_exclude.update("filter-synthesis", "filter-year-by-year") new_ini_properties = internal_link.model_dump(by_alias=True, exclude=to_exclude, exclude_unset=True) if new_ini_properties: # If no new INI properties were given we shouldn't update the INI file + properties = study_data.tree.get(["input", "links", area_from, "properties", area_to]) properties.update(new_ini_properties) study_data.tree.save(properties, ["input", "links", area_from, "properties", area_to]) @@ -80,6 +79,8 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = if not old_parameters: db_properties = LinkTsGeneration.model_validate(db_properties_json) new_parameters = db_properties.to_db_model(study_id, area_from, area_to) + db.session.add(new_parameters) + db.session.commit() else: old_props = LinkTsGeneration.from_db_model(old_parameters).model_dump(mode="json") old_props.update(db_properties_json) @@ -89,8 +90,10 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = # We should keep the same matrices new_parameters.modulation = old_parameters.modulation new_parameters.project = old_parameters.prepro - db.session.merge(new_parameters) - db.session.commit() + # todo: I should do it more efficiently but a merge creates 2 entry ... + db.session.delete(old_parameters) + db.session.add(new_parameters) + db.session.commit() # Updates matrices if self.series: diff --git a/tests/variantstudy/model/command/test_update_link.py b/tests/variantstudy/model/command/test_update_link.py index 021be547e2..f2ff4bc8a9 100644 --- a/tests/variantstudy/model/command/test_update_link.py +++ b/tests/variantstudy/model/command/test_update_link.py @@ -81,7 +81,8 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext): # Asserts the ini file is well modified (new value + old values unmodified) link = IniReader() - link_data = link.read(study_path / "input" / "links" / area1_id / "properties.ini") + ini_path = study_path / "input" / "links" / area1_id / "properties.ini" + link_data = link.read(ini_path) assert link_data[area2_id]["hurdles-cost"] == parameters["hurdles-cost"] assert link_data[area2_id]["asset-type"] == parameters["asset-type"] assert link_data[area2_id]["colorb"] == new_parameters["colorb"] @@ -89,13 +90,13 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext): # Updating a DB property new_parameters = {"nominal_capacity": 111} update_parameters = {"parameters": new_parameters, **command_parameters} + # Removes the ini file to show we don't need it as we're only updating the DB + ini_path.unlink() with DBStatementRecorder(db.session.bind) as db_recorder: UpdateLink.model_validate(update_parameters).apply(study_data=empty_study) # todo: this check seems random, don't know why :/ - # assert len(db_recorder.sql_statements) == 2 - - # todo: Could be nice to check that the tree isn't called. Don't know how. + # assert len(db_recorder.sql_statements) == 3 # Checks the DB state. Old properties should remain the same and the new one should be updated ts_gen_properties = ( @@ -103,11 +104,8 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext): .filter_by(study_id=study_id, area_from=area1_id, area_to=area2_id) .all() ) - # todo: Why do I have 2 entries here ???? - """ assert len(ts_gen_properties) == 1 link_ts_gen_props = ts_gen_properties[0] assert link_ts_gen_props.unit_count == parameters["unit_count"] assert link_ts_gen_props.law_planned == parameters["law_planned"] assert link_ts_gen_props.nominal_capacity == new_parameters["nominal_capacity"] - """ From fda4f1310c43669c5f4922cfc46fad0001475f26 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 18:37:27 +0100 Subject: [PATCH 43/53] fix little issues --- .../storage/variantstudy/model/command/update_link.py | 7 ++----- tests/variantstudy/model/command/test_update_link.py | 11 +++-------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/antarest/study/storage/variantstudy/model/command/update_link.py b/antarest/study/storage/variantstudy/model/command/update_link.py index 7069154091..26c3e0735e 100644 --- a/antarest/study/storage/variantstudy/model/command/update_link.py +++ b/antarest/study/storage/variantstudy/model/command/update_link.py @@ -79,8 +79,6 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = if not old_parameters: db_properties = LinkTsGeneration.model_validate(db_properties_json) new_parameters = db_properties.to_db_model(study_id, area_from, area_to) - db.session.add(new_parameters) - db.session.commit() else: old_props = LinkTsGeneration.from_db_model(old_parameters).model_dump(mode="json") old_props.update(db_properties_json) @@ -90,10 +88,9 @@ def _apply(self, study_data: FileStudy, listener: t.Optional[ICommandListener] = # We should keep the same matrices new_parameters.modulation = old_parameters.modulation new_parameters.project = old_parameters.prepro - # todo: I should do it more efficiently but a merge creates 2 entry ... db.session.delete(old_parameters) - db.session.add(new_parameters) - db.session.commit() + db.session.add(new_parameters) + db.session.commit() # Updates matrices if self.series: diff --git a/tests/variantstudy/model/command/test_update_link.py b/tests/variantstudy/model/command/test_update_link.py index f2ff4bc8a9..6935bd0a2f 100644 --- a/tests/variantstudy/model/command/test_update_link.py +++ b/tests/variantstudy/model/command/test_update_link.py @@ -10,15 +10,15 @@ # # This file is part of the Antares project. +from antarest.core.utils.fastapi_sqlalchemy import db from antarest.study.model import LinksParametersTsGeneration, RawStudy from antarest.study.storage.rawstudy.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.create_area import CreateArea from antarest.study.storage.variantstudy.model.command.create_link import CreateLink +from antarest.study.storage.variantstudy.model.command.update_link import UpdateLink from antarest.study.storage.variantstudy.model.command_context import CommandContext -from core.utils.fastapi_sqlalchemy import db -from db_statement_recorder import DBStatementRecorder -from study.storage.variantstudy.model.command.update_link import UpdateLink +from tests.db_statement_recorder import DBStatementRecorder from tests.helpers import with_db_context @@ -93,11 +93,6 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext): # Removes the ini file to show we don't need it as we're only updating the DB ini_path.unlink() - with DBStatementRecorder(db.session.bind) as db_recorder: - UpdateLink.model_validate(update_parameters).apply(study_data=empty_study) - # todo: this check seems random, don't know why :/ - # assert len(db_recorder.sql_statements) == 3 - # Checks the DB state. Old properties should remain the same and the new one should be updated ts_gen_properties = ( db.session.query(LinksParametersTsGeneration) From ab2f3ce6e2010481ce298563a44f9d9147102220 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 22 Jan 2025 18:49:44 +0100 Subject: [PATCH 44/53] fix test --- tests/variantstudy/model/command/test_update_link.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/variantstudy/model/command/test_update_link.py b/tests/variantstudy/model/command/test_update_link.py index 6935bd0a2f..5d0eef78f7 100644 --- a/tests/variantstudy/model/command/test_update_link.py +++ b/tests/variantstudy/model/command/test_update_link.py @@ -92,6 +92,7 @@ def test_apply(self, empty_study: FileStudy, command_context: CommandContext): update_parameters = {"parameters": new_parameters, **command_parameters} # Removes the ini file to show we don't need it as we're only updating the DB ini_path.unlink() + UpdateLink.model_validate(update_parameters).apply(study_data=empty_study) # Checks the DB state. Old properties should remain the same and the new one should be updated ts_gen_properties = ( From a17d5211742cfdd4d94c0c7f932d363d45fab873 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Mon, 27 Jan 2025 17:34:31 +0100 Subject: [PATCH 45/53] simplify links testing --- .../study_data_blueprint/test_link.py | 402 +++++------------- 1 file changed, 114 insertions(+), 288 deletions(-) diff --git a/tests/integration/study_data_blueprint/test_link.py b/tests/integration/study_data_blueprint/test_link.py index 5bb5a38979..c2510480a6 100644 --- a/tests/integration/study_data_blueprint/test_link.py +++ b/tests/integration/study_data_blueprint/test_link.py @@ -9,92 +9,51 @@ # SPDX-License-Identifier: MPL-2.0 # # This file is part of the Antares project. +import copy + import pytest from starlette.testclient import TestClient -from antarest.study.business.model.link_model import TransmissionCapacity from tests.integration.prepare_proxy import PreparerProxy @pytest.mark.unit_test class TestLink: @pytest.mark.parametrize("study_type", ["raw", "variant"]) - def test_link_update(self, client: TestClient, user_access_token: str, study_type: str) -> None: + def test_nominal_case(self, client: TestClient, user_access_token: str, study_type: str) -> None: + # ============================= + # SET UP + # ============================= + client.headers = {"Authorization": f"Bearer {user_access_token}"} # type: ignore preparer = PreparerProxy(client, user_access_token) - study_id = preparer.create_study("foo", version=820) + study_id = preparer.create_study("foo") if study_type == "variant": study_id = preparer.create_variant(study_id, name="Variant 1") + links_url = f"/v1/studies/{study_id}/links" + area1_id = preparer.create_area(study_id, name="Area 1")["id"] area2_id = preparer.create_area(study_id, name="Area 2")["id"] - client.post( - f"/v1/studies/{study_id}/links", - json={"area1": area1_id, "area2": area2_id, "hurdlesCost": True, "comments": "comment"}, - ) - res = client.put( - f"/v1/studies/{study_id}/links/{area1_id}/{area2_id}", - json={"colorr": 150}, - ) + area3_id = preparer.create_area(study_id, name="Area 3")["id"] - assert res.status_code == 200 - expected = { - "area1": "area 1", - "area2": "area 2", - "assetType": "ac", - "colorb": 112, - "colorg": 112, - "colorr": 150, - "displayComments": True, - "comments": "comment", - "filterSynthesis": "hourly, daily, weekly, monthly, annual", - "filterYearByYear": "hourly, daily, weekly, monthly, annual", - "hurdlesCost": True, - "linkStyle": "plain", - "linkWidth": 1.0, - "loopFlow": False, - "transmissionCapacities": "enabled", - "usePhaseShifter": False, - "volatilityForced": 0.0, - "volatilityPlanned": 0.0, - "lawForced": "uniform", - "lawPlanned": "uniform", - "forceNoGeneration": True, - "nominalCapacity": 0.0, - "unitCount": 1, - } - assert res.json() == expected + # ============================= + # CREATION AND UPDATE + # ============================= - # Test update link same area + # Link creation with default values + res = client.post(links_url, json={"area1": area1_id, "area2": area2_id}) - res = client.put( - f"/v1/studies/{study_id}/links/{area1_id}/{area1_id}", - json={"hurdlesCost": False}, - ) - assert res.status_code == 422 - expected = { - "description": "Cannot create a link that goes from and to the same single area: area 1", - "exception": "LinkValidationError", - } - assert expected == res.json() - - # Test update link area not ordered + assert res.status_code == 200, res.json() - res = client.put( - f"/v1/studies/{study_id}/links/{area2_id}/{area1_id}", - json={"hurdlesCost": False}, - ) - assert res.status_code == 200 - expected = { - "area1": "area 1", - "area2": "area 2", + default_parameters = { "assetType": "ac", "colorb": 112, "colorg": 112, - "colorr": 150, + "colorr": 112, "displayComments": True, - "comments": "comment", + "comments": "", "filterSynthesis": "hourly, daily, weekly, monthly, annual", "filterYearByYear": "hourly, daily, weekly, monthly, annual", "hurdlesCost": False, @@ -111,206 +70,120 @@ def test_link_update(self, client: TestClient, user_access_token: str, study_typ "nominalCapacity": 0.0, "unitCount": 1, } - assert res.json() == expected - - # Test update link with non-existing area + expected_result = copy.deepcopy(default_parameters) + expected_result["area1"] = area1_id + expected_result["area2"] = area2_id + assert res.json() == expected_result - res = client.put( - f"/v1/studies/{study_id}/links/{area1_id}/id_do_not_exist", - json={"hurdlesCost": False}, + # Link creation with specific values and then updates another value + client.post( + links_url, + json={"area1": area1_id, "area2": area3_id, "hurdlesCost": True, "comments": "comment"}, ) - assert res.status_code == 404 - expected = { - "description": "The link area 1 -> id_do_not_exist is not present in the study", - "exception": "LinkNotFound", - } - assert expected == res.json() + res = client.put(f"/v1/studies/{study_id}/links/{area1_id}/{area3_id}", json={"colorr": 150}) + assert res.status_code == 200 + expected = copy.deepcopy(default_parameters) + expected["area1"] = area1_id + expected["area2"] = area3_id + expected["hurdlesCost"] = True + expected["comments"] = "comment" + expected["colorr"] = 150 + assert res.json() == expected - # Test update link fails when given wrong parameters - if study_type == "raw": - res = client.post( - f"/v1/studies/{study_id}/commands", - json=[ - { - "action": "update_link", - "args": { - "area1": area1_id, - "area2": area2_id, - "parameters": {"hurdles-cost": False, "wrong": "parameter"}, - }, - } - ], - ) - assert res.status_code == 500 - expected = "Unexpected exception occurred when trying to apply command CommandName.UPDATE_LINK" - assert expected in res.json()["description"] + # Test update link area not ordered + + res = client.put(f"/v1/studies/{study_id}/links/{area3_id}/{area1_id}", json={"hurdlesCost": False}) + assert res.status_code == 200 + expected["hurdlesCost"] = False + assert res.json() == expected # Test update link variant returns only modified values if study_type == "variant": res = client.put( - f"/v1/studies/{study_id}/links/{area1_id}/{area2_id}", - json={"hurdlesCost": False}, + f"/v1/studies/{study_id}/links/{area1_id}/{area3_id}", + json={"assetType": "dc"}, ) assert res.status_code == 200 res = client.get(f"/v1/studies/{study_id}/commands") commands = res.json() command_args = commands[-1]["args"] - assert command_args["parameters"] == {"hurdles_cost": False} - - @pytest.mark.parametrize("study_type", ["raw", "variant"]) - def test_link_820(self, client: TestClient, user_access_token: str, study_type: str) -> None: - client.headers = {"Authorization": f"Bearer {user_access_token}"} # type: ignore - - preparer = PreparerProxy(client, user_access_token) - study_id = preparer.create_study("foo", version=820) - if study_type == "variant": - study_id = preparer.create_variant(study_id, name="Variant 1") + assert command_args["parameters"] == {"asset_type": "dc"} - area1_id = preparer.create_area(study_id, name="Area 1")["id"] - area2_id = preparer.create_area(study_id, name="Area 2")["id"] - area3_id = preparer.create_area(study_id, name="Area 3")["id"] - - # Test create link with default values - res = client.post(f"/v1/studies/{study_id}/links", json={"area1": area1_id, "area2": area2_id}) + # Test create link with empty filters + res = client.post(links_url, json={"area1": area2_id, "area2": area3_id, "filterSynthesis": ""}) assert res.status_code == 200, res.json() - - expected = { - "area1": "area 1", - "area2": "area 2", - "assetType": "ac", - "colorb": 112, - "colorg": 112, - "colorr": 112, - "displayComments": True, - "comments": "", - "filterSynthesis": "hourly, daily, weekly, monthly, annual", - "filterYearByYear": "hourly, daily, weekly, monthly, annual", - "hurdlesCost": False, - "linkStyle": "plain", - "linkWidth": 1.0, - "loopFlow": False, - "transmissionCapacities": "enabled", - "usePhaseShifter": False, - "volatilityForced": 0.0, - "volatilityPlanned": 0.0, - "lawForced": "uniform", - "lawPlanned": "uniform", - "forceNoGeneration": True, - "nominalCapacity": 0.0, - "unitCount": 1, - } + expected = copy.deepcopy(default_parameters) + expected["area1"] = area2_id + expected["area2"] = area3_id + expected["filterSynthesis"] = "" assert res.json() == expected - res = client.delete(f"/v1/studies/{study_id}/links/{area1_id}/{area2_id}") + res = client.delete(f"/v1/studies/{study_id}/links/{area2_id}/{area3_id}") res.raise_for_status() - # Test create link with parameters - - parameters = { - "area1": "area 1", - "area2": "area 2", - "assetType": "ac", - "colorb": 160, - "colorg": 170, - "colorr": 180, - "displayComments": True, - "comments": "comment", - "filterSynthesis": "hourly, daily, weekly, monthly, annual", - "filterYearByYear": "hourly, daily, weekly, monthly, annual", - "hurdlesCost": False, - "linkStyle": "plain", - "linkWidth": 1.0, - "loopFlow": False, - "transmissionCapacities": "enabled", - "usePhaseShifter": False, - "volatilityForced": 0.0, - "volatilityPlanned": 0.0, - "lawForced": "uniform", - "lawPlanned": "uniform", - "forceNoGeneration": True, - "nominalCapacity": 0.0, - "unitCount": 1, - } - res = client.post( - f"/v1/studies/{study_id}/links", - json=parameters, - ) + # Test create link with double value in filter + res = client.post(links_url, json={"area1": area2_id, "area2": area3_id, "filterSynthesis": "hourly, hourly"}) assert res.status_code == 200, res.json() + expected = copy.deepcopy(default_parameters) + expected["filterSynthesis"] = "hourly" + expected["area1"] = area2_id + expected["area2"] = area3_id + assert res.json() == expected - assert parameters == res.json() - res = client.delete(f"/v1/studies/{study_id}/links/{area1_id}/{area2_id}") - res.raise_for_status() - - # Create two links, count them, then delete one - - res1 = client.post(f"/v1/studies/{study_id}/links", json={"area1": area1_id, "area2": area2_id}) - res2 = client.post(f"/v1/studies/{study_id}/links", json={"area1": area1_id, "area2": area3_id}) - - assert res1.status_code == 200, res1.json() - assert res2.status_code == 200, res2.json() + # ============================= + # DELETION + # ============================= - res = client.get(f"/v1/studies/{study_id}/links") + # Deletes one link and asserts it's not present anymore + res = client.get(links_url) + assert res.status_code == 200, res.json() + assert 3 == len(res.json()) + res = client.delete(f"/v1/studies/{study_id}/links/{area2_id}/{area3_id}") assert res.status_code == 200, res.json() - assert 2 == len(res.json()) - res = client.delete(f"/v1/studies/{study_id}/links/{area1_id}/{area3_id}") - res.raise_for_status() + res = client.get(links_url) + assert res.status_code == 200 + assert 2 == len(res.json()) - res = client.get(f"/v1/studies/{study_id}/links") + # ============================= + # ERRORS + # ============================= - assert res.status_code == 200, res.json() - assert 1 == len(res.json()) - client.delete(f"/v1/studies/{study_id}/links/{area1_id}/{area2_id}") - res.raise_for_status() + # Test update link same area - # Test create link with same area + res = client.put(f"/v1/studies/{study_id}/links/{area1_id}/{area1_id}", json={"hurdlesCost": False}) + assert res.status_code == 422 + assert res.json()["description"] == "Cannot create a link that goes from and to the same single area: area 1" + assert res.json()["exception"] == "LinkValidationError" - res = client.post(f"/v1/studies/{study_id}/links", json={"area1": area1_id, "area2": area1_id}) + # Test update link with non-existing area - assert res.status_code == 422, res.json() - expected = { - "description": "Cannot create a link that goes from and to the same single area: area 1", - "exception": "LinkValidationError", - } - assert expected == res.json() + res = client.put(f"/v1/studies/{study_id}/links/{area1_id}/id_do_not_exist", json={"hurdlesCost": False}) + assert res.status_code == 404 + assert res.json()["description"] == "The link area 1 -> id_do_not_exist is not present in the study" + assert res.json()["exception"] == "LinkNotFound" # Test create link with wrong value for enum - res = client.post( - f"/v1/studies/{study_id}/links", - json={"area1": area1_id, "area2": area2_id, "assetType": TransmissionCapacity.ENABLED}, - ) + res = client.post(links_url, json={"area1": area1_id, "area2": area2_id, "assetType": "enabled"}) assert res.status_code == 422, res.json() - expected = { - "body": {"area1": "area 1", "area2": "area 2", "assetType": "enabled"}, - "description": "Input should be 'ac', 'dc', 'gaz', 'virt' or 'other'", - "exception": "RequestValidationError", - } - assert expected == res.json() + assert res.json()["description"] == "Input should be 'ac', 'dc', 'gaz', 'virt' or 'other'" + assert res.json()["exception"] == "RequestValidationError" # Test create link with wrong color parameter - res = client.post(f"/v1/studies/{study_id}/links", json={"area1": area1_id, "area2": area2_id, "colorr": 260}) - + res = client.post(links_url, json={"area1": area1_id, "area2": area2_id, "colorr": 260}) assert res.status_code == 422, res.json() - expected = { - "body": {"area1": "area 1", "area2": "area 2", "colorr": 260}, - "description": "Input should be less than or equal to 255", - "exception": "RequestValidationError", - } - assert expected == res.json() + assert res.json()["description"] == "Input should be less than or equal to 255" + assert res.json()["exception"] == "RequestValidationError" # Test create link with wrong filter parameter - res = client.post( - f"/v1/studies/{study_id}/links", - json={"area1": area1_id, "area2": area2_id, "filterSynthesis": "centurial"}, - ) - + res = client.post(links_url, json={"area1": area1_id, "area2": area2_id, "filterSynthesis": "centurial"}) assert res.status_code == 422, res.json() expected = { "description": "Invalid value(s) in filters: centurial. Allowed values are: hourly, daily, weekly, monthly, annual.", @@ -318,78 +191,27 @@ def test_link_820(self, client: TestClient, user_access_token: str, study_type: } assert expected == res.json() - # Test create link with empty filters - - res = client.post( - f"/v1/studies/{study_id}/links", - json={"area1": area1_id, "area2": area2_id, "filterSynthesis": ""}, - ) + # Test update link command fails when given wrong parameters - assert res.status_code == 200, res.json() - expected = { - "area1": "area 1", - "area2": "area 2", - "assetType": "ac", - "colorb": 112, - "colorg": 112, - "colorr": 112, - "displayComments": True, - "comments": "", - "filterSynthesis": "", - "filterYearByYear": "hourly, daily, weekly, monthly, annual", - "hurdlesCost": False, - "linkStyle": "plain", - "linkWidth": 1.0, - "loopFlow": False, - "transmissionCapacities": "enabled", - "usePhaseShifter": False, - "volatilityForced": 0.0, - "volatilityPlanned": 0.0, - "lawForced": "uniform", - "lawPlanned": "uniform", - "forceNoGeneration": True, - "nominalCapacity": 0.0, - "unitCount": 1, - } - assert res.json() == expected - - # Test create link with double value in filter - - client.delete(f"/v1/studies/{study_id}/links/{area1_id}/{area2_id}") - res = client.post( - f"/v1/studies/{study_id}/links", - json={"area1": area1_id, "area2": area2_id, "filterSynthesis": "hourly, hourly"}, - ) - - assert res.status_code == 200, res.json() - expected = { - "area1": "area 1", - "area2": "area 2", - "assetType": "ac", - "colorb": 112, - "colorg": 112, - "colorr": 112, - "displayComments": True, - "comments": "", - "filterSynthesis": "hourly", - "filterYearByYear": "hourly, daily, weekly, monthly, annual", - "hurdlesCost": False, - "linkStyle": "plain", - "linkWidth": 1.0, - "loopFlow": False, - "transmissionCapacities": "enabled", - "usePhaseShifter": False, - "volatilityForced": 0.0, - "volatilityPlanned": 0.0, - "lawForced": "uniform", - "lawPlanned": "uniform", - "forceNoGeneration": True, - "nominalCapacity": 0.0, - "unitCount": 1, - } - assert res.json() == expected + if study_type == "raw": + res = client.post( + f"/v1/studies/{study_id}/commands", + json=[ + { + "action": "update_link", + "args": { + "area1": area1_id, + "area2": area2_id, + "parameters": {"hurdles-cost": False, "wrong": "parameter"}, + }, + } + ], + ) + assert res.status_code == 500 + expected = "Unexpected exception occurred when trying to apply command CommandName.UPDATE_LINK" + assert expected in res.json()["description"] - def test_create_link_810(self, client: TestClient, user_access_token: str) -> None: + def test_other_behaviors(self, client: TestClient, user_access_token: str) -> None: client.headers = {"Authorization": f"Bearer {user_access_token}"} # type: ignore preparer = PreparerProxy(client, user_access_token) @@ -397,6 +219,7 @@ def test_create_link_810(self, client: TestClient, user_access_token: str) -> No area1_id = preparer.create_area(study_id, name="Area 1")["id"] area2_id = preparer.create_area(study_id, name="Area 2")["id"] + # Asserts we cannot give a filter value to a study prior to v8.2 res = client.post( f"/v1/studies/{study_id}/links", json={"area1": area1_id, "area2": area2_id, "filterSynthesis": "hourly"} ) @@ -407,3 +230,6 @@ def test_create_link_810(self, client: TestClient, user_access_token: str) -> No "exception": "LinkValidationError", } assert expected == res.json() + + # Asserts TS generation info are passed from a parent to its children + # todo From 14b175a8485c294e0e8d7432d0f4f08cc81b4d82 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Mon, 27 Jan 2025 17:40:47 +0100 Subject: [PATCH 46/53] add a test that fails for now --- .../study_data_blueprint/test_link.py | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/integration/study_data_blueprint/test_link.py b/tests/integration/study_data_blueprint/test_link.py index c2510480a6..83609cd7ac 100644 --- a/tests/integration/study_data_blueprint/test_link.py +++ b/tests/integration/study_data_blueprint/test_link.py @@ -218,18 +218,26 @@ def test_other_behaviors(self, client: TestClient, user_access_token: str) -> No study_id = preparer.create_study("foo", version=810) area1_id = preparer.create_area(study_id, name="Area 1")["id"] area2_id = preparer.create_area(study_id, name="Area 2")["id"] + links_url = f"/v1/studies/{study_id}/links" # Asserts we cannot give a filter value to a study prior to v8.2 - res = client.post( - f"/v1/studies/{study_id}/links", json={"area1": area1_id, "area2": area2_id, "filterSynthesis": "hourly"} - ) - + res = client.post(links_url, json={"area1": area1_id, "area2": area2_id, "filterSynthesis": "hourly"}) assert res.status_code == 422, res.json() - expected = { - "description": "Cannot specify a filter value for study's version earlier than v8.2", - "exception": "LinkValidationError", - } - assert expected == res.json() + assert res.json()["description"] == "Cannot specify a filter value for study's version earlier than v8.2" + assert res.json()["exception"] == "LinkValidationError" + + # ============================= + # TS GENERATION + # ============================= - # Asserts TS generation info are passed from a parent to its children - # todo + # Creates a link inside parent study with a specific unit count + res = client.post(links_url, json={"area1": area1_id, "area2": area2_id, "unitCount": 24}) + assert res.status_code == 200, res.json() + # Asserts the value was saved correctly + res = client.get(links_url) + assert res.json()[0]["unitCount"] == 24 + # Creates a variant + variant_id = preparer.create_variant(study_id, name="Variant 1") + # Asserts we still have the parent value + res = client.get(f"/v1/studies/{variant_id}/links") + assert res.json()[0]["unitCount"] == 24 From abb4349af94c8de178a8e980e730d49e5063e862 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Mon, 27 Jan 2025 18:05:02 +0100 Subject: [PATCH 47/53] fix code --- .../variantstudy/variant_study_service.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index 55e8d827f0..24367b657c 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -658,6 +658,7 @@ def create_variant_study(self, uuid: str, name: str, params: RequestParameters) additional_data=additional_data, ) self.repository.save(variant_study) + self._copy_parent_ts_generation_info(study.id, new_id) self.event_bus.push( Event( type=EventType.STUDY_CREATED, @@ -1201,6 +1202,21 @@ def clear_all_snapshots(self, retention_time: timedelta, params: RequestParamete request_params=params, ) + @staticmethod + def _copy_parent_ts_generation_info(parent_study_id: str, variant_study_id: str) -> None: + with db(): + parent_parameters: t.List[LinksParametersTsGeneration] = ( + db.session.query(LinksParametersTsGeneration).filter_by(study_id=parent_study_id).all() + ) + if not parent_parameters: + return + child_parameters = [] + for parameter in parent_parameters: + parameter.study_id = variant_study_id + child_parameters.append(parameter) + db.session.add_all(child_parameters) + db.session.commit() + class SnapshotCleanerTask: def __init__( From 07ac311395e9c5c10d51624a92954aeb2041663e Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 28 Jan 2025 09:21:05 +0100 Subject: [PATCH 48/53] clear db when invalidating the cache --- .../variantstudy/variant_study_service.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index 24367b657c..793a9f5ae8 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -439,8 +439,16 @@ def invalidate_cache( invalidate_self_snapshot: bool = False, ) -> None: remove_from_cache(self.cache, variant_study.id) - if isinstance(variant_study, VariantStudy) and variant_study.snapshot and invalidate_self_snapshot: - variant_study.snapshot.last_executed_command = None + if isinstance(variant_study, VariantStudy): + # Removes TS-generation related information from the database + with db(): + db.session.query(NbYearsTsGeneration).filter_by(id=variant_study.id).delete() + db.session.query(LinksParametersTsGeneration).filter_by(study_id=variant_study.id).delete() + db.session.commit() + + if variant_study.snapshot and invalidate_self_snapshot: + variant_study.snapshot.last_executed_command = None + self.repository.save( metadata=variant_study, update_modification_date=True, @@ -453,12 +461,6 @@ def clear_snapshot(self, variant_study: Study) -> None: self.invalidate_cache(variant_study, invalidate_self_snapshot=True) shutil.rmtree(self.get_study_path(variant_study), ignore_errors=True) - # Removes TS-generation related information from the database - with db(): - db.session.query(NbYearsTsGeneration).filter_by(id=variant_study.id).delete() - db.session.query(LinksParametersTsGeneration).filter_by(study_id=variant_study.id).delete() - db.session.commit() - def has_children(self, study: VariantStudy) -> bool: return self.repository.has_children(study.id) From d504f51afcb08cff7b1e6930bc9dd4bd198dedb4 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 28 Jan 2025 09:27:46 +0100 Subject: [PATCH 49/53] put the DB copy inside the snapshot generation --- .../variantstudy/snapshot_generator.py | 19 ++++++++++++++++++- .../variantstudy/variant_study_service.py | 16 ---------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/antarest/study/storage/variantstudy/snapshot_generator.py b/antarest/study/storage/variantstudy/snapshot_generator.py index 58d4d62b7b..baae8ee832 100644 --- a/antarest/study/storage/variantstudy/snapshot_generator.py +++ b/antarest/study/storage/variantstudy/snapshot_generator.py @@ -24,7 +24,8 @@ from antarest.core.jwt import JWTUser from antarest.core.model import StudyPermissionType from antarest.core.tasks.service import ITaskNotifier, NoopNotifier -from antarest.study.model import RawStudy, StudyAdditionalData +from antarest.core.utils.fastapi_sqlalchemy import db +from antarest.study.model import LinksParametersTsGeneration, RawStudy, StudyAdditionalData from antarest.study.storage.patch_service import PatchService from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfigDTO from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy, StudyFactory @@ -126,6 +127,7 @@ def generate_snapshot( logger.info(f"Reading additional data from files for study {file_study.config.study_id}") variant_study.additional_data = self._read_additional_data(file_study) self.repository.save(variant_study) + self._copy_parent_ts_generation_info(parent_study_id=ref_study.id, variant_study_id=variant_study_id) self._update_cache(file_study) @@ -218,6 +220,21 @@ def _update_cache(self, file_study: FileStudy) -> None: FileStudyTreeConfigDTO.from_build_config(file_study.config).model_dump(), ) + @staticmethod + def _copy_parent_ts_generation_info(parent_study_id: str, variant_study_id: str) -> None: + with db(): + parent_parameters: t.List[LinksParametersTsGeneration] = ( + db.session.query(LinksParametersTsGeneration).filter_by(study_id=parent_study_id).all() + ) + if not parent_parameters: + return + child_parameters = [] + for parameter in parent_parameters: + parameter.study_id = variant_study_id + child_parameters.append(parameter) + db.session.add_all(child_parameters) + db.session.commit() + class RefStudySearchResult(t.NamedTuple): """ diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index 793a9f5ae8..71a632fb70 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -660,7 +660,6 @@ def create_variant_study(self, uuid: str, name: str, params: RequestParameters) additional_data=additional_data, ) self.repository.save(variant_study) - self._copy_parent_ts_generation_info(study.id, new_id) self.event_bus.push( Event( type=EventType.STUDY_CREATED, @@ -1204,21 +1203,6 @@ def clear_all_snapshots(self, retention_time: timedelta, params: RequestParamete request_params=params, ) - @staticmethod - def _copy_parent_ts_generation_info(parent_study_id: str, variant_study_id: str) -> None: - with db(): - parent_parameters: t.List[LinksParametersTsGeneration] = ( - db.session.query(LinksParametersTsGeneration).filter_by(study_id=parent_study_id).all() - ) - if not parent_parameters: - return - child_parameters = [] - for parameter in parent_parameters: - parameter.study_id = variant_study_id - child_parameters.append(parameter) - db.session.add_all(child_parameters) - db.session.commit() - class SnapshotCleanerTask: def __init__( From a394dce1fbca4862c2e6fba687034f753041fc9b Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 28 Jan 2025 09:32:12 +0100 Subject: [PATCH 50/53] fix test with new behavior --- tests/study/storage/variantstudy/test_snapshot_generator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/study/storage/variantstudy/test_snapshot_generator.py b/tests/study/storage/variantstudy/test_snapshot_generator.py index 6c89318342..4b2c5ca4c7 100644 --- a/tests/study/storage/variantstudy/test_snapshot_generator.py +++ b/tests/study/storage/variantstudy/test_snapshot_generator.py @@ -882,8 +882,10 @@ def test_generate__nominal_case( # - 1 query to fetch the list of variants with snapshot, commands, etc., # - 1 query to update the variant study additional_data, # - 1 query to insert the variant study snapshot. + # - 1 query to fetch the TS generation info of the parent study + # - 1 query to insert them for the newly created variant study # - 1 query to check the DB state inside the create_link command - assert len(db_recorder.sql_statements) == 6, str(db_recorder) + assert len(db_recorder.sql_statements) == 8, str(db_recorder) # Check: the variant generation must succeed. assert results.model_dump() == { From 4a1e5b8c9bbc00eccdaf3fe2e79806cd4ce73928 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 28 Jan 2025 09:46:36 +0100 Subject: [PATCH 51/53] add index for study_id inside DB --- .../versions/dadad513f1b6_create_links_ts_generation_tables.py | 1 + 1 file changed, 1 insertion(+) diff --git a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py index b0bded3c13..116924106e 100644 --- a/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py +++ b/alembic/versions/dadad513f1b6_create_links_ts_generation_tables.py @@ -26,6 +26,7 @@ def upgrade(): ondelete="CASCADE" ), sa.PrimaryKeyConstraint("id"), + sa.Index('ix_nb_years_ts_generation_study_id', 'study_id') ) default_boolean = sa.text('1') if op.get_context().dialect.name == 'sqlite' else 't' From 20d2f822e2385981a9befa50b844e133e64dd1f1 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 28 Jan 2025 10:03:02 +0100 Subject: [PATCH 52/53] add tricky test --- .../study_data_blueprint/test_link.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/integration/study_data_blueprint/test_link.py b/tests/integration/study_data_blueprint/test_link.py index 83609cd7ac..742093b7e6 100644 --- a/tests/integration/study_data_blueprint/test_link.py +++ b/tests/integration/study_data_blueprint/test_link.py @@ -241,3 +241,23 @@ def test_other_behaviors(self, client: TestClient, user_access_token: str) -> No # Asserts we still have the parent value res = client.get(f"/v1/studies/{variant_id}/links") assert res.json()[0]["unitCount"] == 24 + # Modifies the unitCount value. The command is only appended not applied so the data isn't saved in DB + res = client.post( + f"/v1/studies/{study_id}/commands", + json=[ + { + "action": "update_link", + "args": { + "area1": area1_id, + "area2": area2_id, + "parameters": {"unit-count": 12}, + }, + } + ], + ) + assert res.status_code == 200, res.json() + # Creates a variant of level 2 + level_2_variant_id = preparer.create_variant(variant_id, name="Variant 2") + # Asserts we see the right value + res = client.get(f"/v1/studies/{level_2_variant_id}/links") + assert res.json()[0]["unitCount"] == 12 From 7eeef2d56bff2932a7149177fdeb87ab37d0054a Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 11 Feb 2025 14:18:15 +0100 Subject: [PATCH 53/53] fix import issue --- tests/variantstudy/model/command/test_update_link.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/variantstudy/model/command/test_update_link.py b/tests/variantstudy/model/command/test_update_link.py index 5d0eef78f7..351f314656 100644 --- a/tests/variantstudy/model/command/test_update_link.py +++ b/tests/variantstudy/model/command/test_update_link.py @@ -10,9 +10,9 @@ # # This file is part of the Antares project. +from antarest.core.serde.ini_reader import IniReader from antarest.core.utils.fastapi_sqlalchemy import db from antarest.study.model import LinksParametersTsGeneration, RawStudy -from antarest.study.storage.rawstudy.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.create_area import CreateArea from antarest.study.storage.variantstudy.model.command.create_link import CreateLink