feat(launcher): add information about which user launched a study (#1761

)

Co-authored-by: Laurent LAPORTE <[email protected]>
(cherry picked from commit ccd849e)

alembic/versions/d495746853cc_add_owner_id_to_jobresult.py

@@ -0,0 +1,27 @@
+"""Add owner_id to JobResult
+
+Revision ID: d495746853cc
+Revises: e65e0c04606b
+Create Date: 2023-10-19 13:16:29.969047
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "d495746853cc"
+down_revision = "e65e0c04606b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("job_result", schema=None) as batch_op:
+        batch_op.add_column(sa.Column('owner_id', sa.Integer(), default=None))
+        batch_op.create_foreign_key("fk_job_result_owner_id", "identities", ["owner_id"], ["id"], ondelete="SET NULL")
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("job_result", schema=None) as batch_op:
+        batch_op.drop_column("owner_id")

antarest/launcher/model.py

@@ -7,7 +7,7 @@
 from sqlalchemy.orm import relationship  # type: ignore
 
 from antarest.core.persistence import Base
-from antarest.core.utils.utils import DTO
+from antarest.login.model import Identity
 
 
 class XpansionParametersDTO(BaseModel):
@@ -23,7 +23,7 @@ class LauncherParametersDTO(BaseModel):
     adequacy_patch: Optional[Dict[str, Any]] = None
     nb_cpu: Optional[int] = None
     post_processing: bool = False
-    time_limit: Optional[int] = None  # 3600 <= time_limit < 864000 (10 days)
+    time_limit: Optional[int] = None  # 3600 ≤ time_limit < 864000 (10 days)
     xpansion: Union[XpansionParametersDTO, bool, None] = None
     xpansion_r_version: bool = False
     archive_output: bool = True
@@ -51,7 +51,7 @@ def to_suffix(self) -> str:
             return "out.log"
         elif self == LogType.STDERR:
             return "err.log"
-        else:
+        else:  # pragma: no cover
             return "out.log"
 
 
@@ -68,6 +68,24 @@ class JobLogType(str, enum.Enum):
 
 
 class JobResultDTO(BaseModel):
+    """
+    A data transfer object (DTO) representing the job result.
+
+    - id: The unique identifier for the task (UUID).
+    - study_id: The unique identifier for the Antares study (UUID).
+    - launcher: The name of the launcher for a simulation task, with possible values "local", "slurm" or `None`.
+    - launcher_params: Parameters related to the launcher.
+    - status: The status of the task. It can be one of the following: "pending", "failed", "success", or "running".
+    - creation_date: The date of creation of the task.
+    - completion_date: The date of completion of the task, if available.
+    - msg: A message associated with the task, either for the user or for error description.
+    - output_id: The identifier of the simulation results.
+    - exit_code: The exit code associated with the task.
+    - solver_stats: Global statistics related to the simulation, including processing time,
+      call count, optimization issues, and study-specific statistics (INI file-like format).
+    - owner_id: The unique identifier of the user or bot that executed the task.
+    """
+
     id: str
     study_id: str
     launcher: Optional[str]
@@ -79,47 +97,52 @@ class JobResultDTO(BaseModel):
     output_id: Optional[str]
     exit_code: Optional[int]
     solver_stats: Optional[str]
+    owner_id: Optional[int]
 
 
-class JobLog(DTO, Base):  # type: ignore
+class JobLog(Base):  # type: ignore
     __tablename__ = "launcherjoblog"
 
-    id = Column(Integer(), Sequence("launcherjoblog_id_sequence"), primary_key=True)
-    message = Column(String, nullable=False)
-    job_id = Column(
+    id: str = Column(Integer(), Sequence("launcherjoblog_id_sequence"), primary_key=True)
+    message: str = Column(String, nullable=False)
+    job_id: str = Column(
         String(),
         ForeignKey("job_result.id", name="fk_log_job_result_id"),
     )
-    log_type = Column(String, nullable=False)
-
-    def __eq__(self, other: Any) -> bool:
-        if not isinstance(other, JobLog):
-            return False
-        return bool(
-            other.id == self.id
-            and other.message == self.message
-            and other.log_type == self.log_type
-            and other.job_id == self.job_id
-        )
+    log_type: str = Column(String, nullable=False)
+
+    # SQLAlchemy provides its own way to handle object comparison, which ensures
+    # that the comparison is based on the database identity of the objects.
+    # So, implementing `__eq__` and `__ne__` is not necessary.
+
+    def __str__(self) -> str:
+        return f"Job log #{self.id} {self.log_type}: '{self.message}'"
 
     def __repr__(self) -> str:
-        return f"id={self.id}, message={self.message}, log_type={self.log_type}, job_id={self.job_id}"
+        return (
+            f"<JobLog(id={self.id!r},"
+            f" message={self.message!r},"
+            f" job_id={self.job_id!r},"
+            f" log_type={self.log_type!r})>"
+        )
 
 
-class JobResult(DTO, Base):  # type: ignore
+class JobResult(Base):  # type: ignore
     __tablename__ = "job_result"
 
-    id = Column(String(36), primary_key=True)
-    study_id = Column(String(36))
-    launcher = Column(String)
-    launcher_params = Column(String, nullable=True)
-    job_status = Column(Enum(JobStatus))
+    id: str = Column(String(36), primary_key=True)
+    study_id: str = Column(String(36))
+    launcher: Optional[str] = Column(String)
+    launcher_params: Optional[str] = Column(String, nullable=True)
+    job_status: JobStatus = Column(Enum(JobStatus))
     creation_date = Column(DateTime, default=datetime.utcnow)
     completion_date = Column(DateTime)
-    msg = Column(String())
-    output_id = Column(String())
-    exit_code = Column(Integer)
-    solver_stats = Column(String(), nullable=True)
+    msg: Optional[str] = Column(String())
+    output_id: Optional[str] = Column(String())
+    exit_code: Optional[int] = Column(Integer)
+    solver_stats: Optional[str] = Column(String(), nullable=True)
+    owner_id: Optional[int] = Column(Integer(), ForeignKey(Identity.id, ondelete="SET NULL"), nullable=True)
+
     logs = relationship(JobLog, uselist=True, cascade="all, delete, delete-orphan")
 
     def to_dto(self) -> JobResultDTO:
@@ -135,18 +158,31 @@ def to_dto(self) -> JobResultDTO:
             output_id=self.output_id,
             exit_code=self.exit_code,
             solver_stats=self.solver_stats,
+            owner_id=self.owner_id,
         )
 
-    def __eq__(self, o: Any) -> bool:
-        if not isinstance(o, JobResult):
-            return False
-        return o.to_dto().dict() == self.to_dto().dict()
+    # SQLAlchemy provides its own way to handle object comparison, which ensures
+    # that the comparison is based on the database identity of the objects.
+    # So, implementing `__eq__` and `__ne__` is not necessary.
 
     def __str__(self) -> str:
-        return str(self.to_dto().dict())
+        return f"Job result #{self.id} (study '{self.study_id}'): {self.job_status}"
 
     def __repr__(self) -> str:
-        return self.__str__()
+        return (
+            f"<JobResult(id={self.id!r},"
+            f" study_id={self.study_id!r},"
+            f" launcher={self.launcher!r},"
+            f" launcher_params={self.launcher_params!r},"
+            f" job_status={self.job_status!r},"
+            f" creation_date={self.creation_date!r},"
+            f" completion_date={self.completion_date!r},"
+            f" msg={self.msg!r},"
+            f" output_id={self.output_id!r},"
+            f" exit_code={self.exit_code!r},"
+            f" solver_stats={self.solver_stats!r},"
+            f" owner_id={self.owner_id!r})>"
+        )
 
 
 class JobCreationDTO(BaseModel):

antarest/launcher/service.py

@@ -225,12 +225,16 @@ def run_study(
             study=study_info,
             permission_type=StudyPermissionType.RUN,
         )
+        owner_id: int = 0
+        if params.user:
+            owner_id = params.user.impersonator if params.user.type == "bots" else params.user.id
         job_status = JobResult(
             id=job_uuid,
             study_id=study_uuid,
             job_status=JobStatus.PENDING,
             launcher=launcher,
             launcher_params=launcher_parameters.json() if launcher_parameters else None,
+            owner_id=(owner_id or None),
         )
         self.job_result_repository.save(job_status)
 
@@ -254,25 +258,33 @@ def run_study(
     def kill_job(self, job_id: str, params: RequestParameters) -> JobResult:
         logger.info(f"Trying to cancel job {job_id}")
         job_result = self.job_result_repository.get(job_id)
-        assert job_result
+        if job_result is None:
+            raise ValueError(f"Job {job_id} not found")
+
         study_uuid = job_result.study_id
-        launcher = job_result.launcher
         study = self.study_service.get_study(study_uuid)
         assert_permission(
             user=params.user,
             study=study,
             permission_type=StudyPermissionType.RUN,
         )
 
+        launcher = job_result.launcher
+        if launcher is None:
+            raise ValueError(f"Job {job_id} has no launcher")
         self._assert_launcher_is_initialized(launcher)
 
         self.launchers[launcher].kill_job(job_id=job_id)
 
+        owner_id = 0
+        if params.user:
+            owner_id = params.user.impersonator if params.user.type == "bots" else params.user.id
         job_status = JobResult(
             id=str(job_id),
             study_id=study_uuid,
             job_status=JobStatus.FAILED,
             launcher=launcher,
+            owner_id=(owner_id or None),
         )
         self.job_result_repository.save(job_status)
         self.event_bus.push(
@@ -373,6 +385,8 @@ def get_log(self, job_id: str, log_type: LogType, params: RequestParameters) ->
                     or ""
                 )
             else:
+                if job_result.launcher is None:
+                    raise ValueError(f"Job {job_id} has no launcher")
                 self._assert_launcher_is_initialized(job_result.launcher)
                 launcher_logs = str(self.launchers[job_result.launcher].get_log(job_id, log_type) or "")
             if log_type == LogType.STDOUT:
@@ -667,5 +681,7 @@ def get_launch_progress(self, job_id: str, params: RequestParameters) -> float:
             permission_type=StudyPermissionType.READ,
         )
 
+        if launcher is None:
+            raise ValueError(f"Job {job_id} has no launcher")
         launch_progress_json = self.launchers[launcher].cache.get(id=f"Launch_Progress_{job_id}") or {"progress": 0}
         return launch_progress_json.get("progress", 0)

docs/architecture/1-database.md

@@ -1,71 +1,67 @@
-# Database management
+# Database Management
 
-We support two database types :
-- postgresql (for production deployment)
-- sqlite (for the local desktop application)
+We support two types of databases:
+- PostgreSQL (for production deployment)
+- SQLite (for the local desktop application)
 
 ## SQLAlchemy & Alembic
 
-We use [sqlalchemy](https://www.sqlalchemy.org/) and [alembic](https://alembic.sqlalchemy.org/en/latest/)
-to manage database and database entities.
+We utilize [SQLAlchemy](https://www.sqlalchemy.org/) and [Alembic](https://alembic.sqlalchemy.org/en/latest/) for managing databases and their entities.
 
-Schema is described by sqlalchemy models that are grouped and imported within
-the file [dbmodel.py](https://github.com/AntaresSimulatorTeam/AntaREST/blob/master/antarest/dbmodel.py).  
-This file is then used by alembic [env file](https://github.com/AntaresSimulatorTeam/AntaREST/blob/master/alembic/env.py) 
-to create the [database migration scripts](https://github.com/AntaresSimulatorTeam/AntaREST/blob/master/alembic/versions).
+The schema is described by SQLAlchemy models that are organized and imported within the file [dbmodel.py](https://github.com/AntaresSimulatorTeam/AntaREST/blob/master/antarest/dbmodel.py).  
+This file is then used by the Alembic [env file](https://github.com/AntaresSimulatorTeam/AntaREST/blob/master/alembic/env.py) to create the [database migration scripts](https://github.com/AntaresSimulatorTeam/AntaREST/blob/master/alembic/versions).
 
-These migration scripts are used by alembic to update a target database defined in the env file which
-uses the database url defined in an [application config]('../install/2-CONFIG.md'), whether on command line 
-(this is the method used on production deployment):
-```
-export ANTAREST_CONF=<some path to config.yaml>
+These migration scripts are used by Alembic to update a target database defined in the env file, which uses the database URL defined in an [application config]('../install/2-CONFIG.md'). This can be done either on the command line (the method used in production deployment):
+
+```shell
+export ANTAREST_CONF=/path/to/your/application.yaml
 alembic upgrade head
 ```
-or within the application launch (see [this file](https://github.com/AntaresSimulatorTeam/AntaREST/blob/master/antarest/core/persistence.py)) :
-```
-python antarest/main.py --auto-upgrade-db
-# or with the gui (default auto upgrade)
-python antarest/gui.py
+
+or within the application launch (refer to [this file](https://github.com/AntaresSimulatorTeam/AntaREST/blob/master/antarest/core/persistence.py)):
+
+```shell
+python3 antarest/main.py --auto-upgrade-db
+# or with the GUI (default auto-upgrade)
+python3 antarest/gui.py
 ```
 
-### How to update the schema
+### How to Update the Schema
 
-When developing for antarest we use a development configuration file that target
-a development database (usually sqlite but could be postgresql). After a first successful launch the database
-schema is migrated to the latest version.  
-The schema version is stored in a table named `alembic_version` that contains the revision id of the last migration file. 
-This information should match with the result of the command `alembic show head` that display the last revision id of the migration file tree.
+When developing for AntaREST, we use a development configuration file that targets a development database (usually SQLite but could be PostgreSQL).
+After a successful initial launch, the database schema is migrated to the latest version.
+The schema version is stored in a table named `alembic_version`, which contains the revision ID of the last migration file.
+This information should match the result of the command `alembic show head`, which displays the last revision ID of the migration file tree.
 
-To update the schema, there is two step.
+To update the schema, there is two steps:
 
-First we make the modification we want in the existing models (for instance in `study/model.py`, `login/model.py`, etc.)
-or create **new models in a separate file that will need to be added to the [dbmodel.py](https://github.com/AntaresSimulatorTeam/AntaREST/blob/master/antarest/dbmodel.py) file**.
-Most of the unit test that create the database from scratch using `sqlalchemy.sql.schema.MetaData.create_all` will do just fine but the integration tests (`tests/integration`) will probably
-fail since they use the alembic migration files process.
+First, we make the modifications we want in the existing models (e.g., in `study/model.py`, `login/model.py`, etc.) or create **new models in a separate file that will need to be added to the [dbmodel.py](https://github.com/AntaresSimulatorTeam/AntaREST/blob/master/antarest/dbmodel.py) file**.
+Most of the unit tests that create the database from scratch using `sqlalchemy.sql.schema.MetaData.create_all` will work fine, but the integration tests (`tests/integration`) will probably fail since they use the alembic migration files process.
 
-So second step is to create the migration file corresponding to the model change. We could create one from scratch, but most of the time,
-the script [create_db_migration.sh](https://github.com/AntaresSimulatorTeam/AntaREST/blob/master/scripts/create_db_migration.sh) (that just wraps the `alembic revision` command) will do:
-```
-export ANTAREST_CONF=<dev conf>
+So the second step is to create the migration file corresponding to the model change.
+We could create one from scratch, but most of the time, the script [create_db_migration.sh](https://github.com/AntaresSimulatorTeam/AntaREST/blob/master/scripts/create_db_migration.sh) (that just wraps the `alembic revision` command) will do:
+
+```shell
+export ANTAREST_CONF=/path/to/your/application.yaml
 ./script/create_db_migration.sh <migration_message>
 ```
-This will create a new migration file in `alembic/versions` that contains two prefilled methods `upgrade` and `downgrade`.  
-Though for a newly created model the edition of this file should be minimal or nul, edition is sometimes required, especially in these cases:
-- handling compatibility/specificity of the databases (eg. adding a sequence `alembic/versions/2ed6bf9f1690_add_tasks.py`)
-- migrating data (eg. renaming/moving a field `alembic/versions/0146b79f723c_update_study.py`)
 
-The `create_db_migration.sh` script will also update the `scripts/rollback.sh` which (as the name indicated) is used to rollback the database to a previous schema.
+This will create a new migration file in `alembic/versions` that contains two prefilled methods `upgrade` and `downgrade`.
+However, for a newly created model, the editing of this file should be minimal or null.
+Editing is sometimes required, especially in these cases:
+- handling compatibility/specificity of the databases (e.g., adding a sequence `alembic/versions/2ed6bf9f1690_add_tasks.py`)
+- migrating data (e.g., renaming/moving a field `alembic/versions/0146b79f723c_update_study.py`)
+
+The `create_db_migration.sh` script will also update the `scripts/rollback.sh` which (as the name indicates) is used to roll back the database to a previous schema.
 
-At this point the development database is not yet migrated. It is only after launching the app (or calling `alembic upgrade head`) that our
-development database will be upgraded.  
+At this point, the development database is not yet migrated.
+It is only after launching the app (or calling `alembic upgrade head`) that our development database will be upgraded.
+
 Now if we want to:
 - modify the model
-- checkout an other branch to test the application prior to this schema update
+- checkout another branch to test the application prior to this schema update
 
-we need to apply the `rollback.sh` script that will revert our local dev database to its previous schema.  
-Then we will be able to either launch the app at a previous database schema or continue modifying the model and reapply
-the migration file creation process (in that case we should delete the now obsolete migration file lastly created).
+we need to apply the `rollback.sh` script that will revert our local dev database to its previous schema.
+Then we will be able to either launch the app at a previous database schema or continue modifying the model and reapply the migration file creation process (in that case, we should delete the now obsolete migration file lastly created).
 
-/!\ Note that when deploying in production a new version with multiple database migration file, the revision id in `rollback.sh` file
-should be the last revision id of the deployed application schema. 
-
+⚠️ Note that when deploying a new version in production with multiple database migration files, the revision ID in the `rollback.sh` file should be the last revision ID of the deployed application schema.