Merge pull request #641 from AntaresSimulatorTeam/dev

Release 2.1.3

alembic/versions/a845d5eae88e_add_filetransfer_manager.py

@@ -0,0 +1,41 @@
+"""add_filetransfer_manager
+
+Revision ID: a845d5eae88e
+Revises: 9846e90c2868
+Create Date: 2021-11-24 14:42:12.269690
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a845d5eae88e'
+down_revision = 'dcbe7dbf500b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('file_download',
+    sa.Column('id', sa.String(length=36), nullable=False),
+    sa.Column('owner', sa.Integer(), nullable=True),
+    sa.Column('name', sa.String(), nullable=True),
+    sa.Column('filename', sa.String(), nullable=True),
+    sa.Column('path', sa.String(), nullable=True),
+    sa.Column('ready', sa.Boolean(), nullable=True),
+    sa.Column('expiration_date', sa.DateTime(), nullable=True),
+    sa.Column('failed', sa.Boolean(), nullable=True),
+    sa.Column('error_message', sa.String(), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('id'),
+    sa.UniqueConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('file_download')
+    # ### end Alembic commands ###

alembic/versions/dcbe7dbf500b_update_datetime.py

@@ -0,0 +1,128 @@
+"""update_datetime
+
+Revision ID: dcbe7dbf500b
+Revises: 63ed81e5ce6f
+Create Date: 2021-11-17 16:10:35.914371
+
+"""
+from typing import Optional, Union
+
+from alembic import op
+from dateutil import tz
+from sqlalchemy.engine import Connection
+from sqlalchemy import text
+from datetime import timezone, datetime, tzinfo
+
+# revision identifiers, used by Alembic.
+revision = "dcbe7dbf500b"
+down_revision = "9846e90c2868"
+branch_labels = None
+depends_on = None
+
+
+def convert_to_utc(data: Union[str, datetime]) -> Optional[str]:
+    if data is not None:
+        dt = data
+        if isinstance(data, str):
+            dt = datetime.strptime(data, "%Y-%m-%d %H:%M:%S.%f")
+        dt = dt.replace(tzinfo=tz.gettz())
+        d1 = dt.utcfromtimestamp(dt.timestamp()).strftime(
+            "%Y-%m-%d %H:%M:%S.%f"
+        )
+        return d1
+    return None
+
+
+def convert_to_local(data: str) -> Optional[str]:
+    if data is not None:
+        dt = data
+        if isinstance(data, str):
+            dt = datetime.strptime(data, "%Y-%m-%d %H:%M:%S.%f")
+        dt = dt.replace(tzinfo=timezone.utc)
+        dt = datetime.fromtimestamp(dt.timestamp())
+        d1 = dt.strftime("%Y-%m-%d %H:%M:%S.%f")
+        return d1
+    return None
+
+
+def time_convert(
+    connexion: Connection,
+    table: str,
+    completion_type: bool = False,
+    to_utc: bool = False,
+) -> None:
+
+    if completion_type:
+        column1 = "creation_date"
+        column2 = "completion_date"
+    else:
+        column1 = "created_at"
+        column2 = "updated_at"
+
+    results = connexion.execute(
+        f"SELECT id, {column1}, {column2} FROM {table}"
+    )
+    for row in results:
+        row_id = row["id"]
+        d1 = (
+            convert_to_utc(data=row[column1])
+            if to_utc
+            else convert_to_local(data=row[column1])
+        )
+        d2 = (
+            convert_to_utc(data=row[column2])
+            if to_utc
+            else convert_to_local(data=row[column2])
+        )
+        connexion.execute(
+            text(
+                f"UPDATE {table} SET {column1}= :c1, {column2}= :c2 WHERE id='{row_id}'"
+            ),
+            c1=d1,
+            c2=d2,
+        )
+
+
+def migrate_datetime(upgrade_mode: bool = True) -> None:
+    connexion: Connection = op.get_bind()
+    # DATASET
+    time_convert(
+        connexion=connexion,
+        table="dataset",
+        completion_type=False,
+        to_utc=upgrade_mode,
+    )
+
+    # STUDIES
+    time_convert(
+        connexion=connexion,
+        table="study",
+        completion_type=False,
+        to_utc=upgrade_mode,
+    )
+
+    # VARIANT STUDY SNAPSHOT
+    results = connexion.execute(
+        "SELECT id, created_at FROM variant_study_snapshot"
+    )
+    for row in results:
+        row_id = row["id"]
+        dt = (
+            convert_to_utc(data=row["created_at"])
+            if upgrade_mode
+            else convert_to_local(data=row["created_at"])
+        )
+        connexion.execute(
+            text(
+                f"UPDATE variant_study_snapshot SET created_at= :created_at WHERE id='{row_id}'"
+            ),
+            created_at=dt,
+        )
+
+
+def upgrade():
+    migrate_datetime(upgrade_mode=True)
+
+
+def downgrade():
+    migrate_datetime(upgrade_mode=False)

antarest/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.1.2"
+__version__ = "2.1.3"
 
 from pathlib import Path
 

antarest/core/config.py

@@ -88,6 +88,7 @@ class StorageConfig:
     allow_deletion: bool = False
     watcher_lock: bool = True
     watcher_lock_delay: int = 10
+    download_default_expiration_timeout_minutes: int = 1440
 
     @staticmethod
     def from_dict(data: JSON) -> "StorageConfig":
@@ -102,6 +103,9 @@ def from_dict(data: JSON) -> "StorageConfig":
             archive_dir=Path(data["archive_dir"]),
             watcher_lock=data.get("watcher_lock", True),
             watcher_lock_delay=data.get("watcher_lock_delay", 10),
+            download_default_expiration_timeout_minutes=data.get(
+                "download_default_expiration_timeout_minutes", 1440
+            ),
         )
 
 

antarest/core/filetransfer/main.py

@@ -0,0 +1,18 @@
+from fastapi import FastAPI
+
+from antarest.core.config import Config
+from antarest.core.filetransfer.repository import FileDownloadRepository
+from antarest.core.filetransfer.service import FileTransferManager
+from antarest.core.filetransfer.web import create_file_transfer_api
+from antarest.core.interfaces.eventbus import IEventBus
+
+
+def build_filetransfer_service(
+    application: FastAPI, event_bus: IEventBus, config: Config
+) -> FileTransferManager:
+    ftm = FileTransferManager(
+        repository=FileDownloadRepository(), event_bus=event_bus, config=config
+    )
+
+    application.include_router(create_file_transfer_api(ftm, config))
+    return ftm

antarest/core/filetransfer/model.py

@@ -0,0 +1,73 @@
+import uuid
+from http import HTTPStatus
+from http.client import HTTPException
+from typing import Optional
+
+from pydantic import BaseModel
+from sqlalchemy import Column, String, Integer, DateTime, Boolean  # type: ignore
+
+from antarest.core.persistence import Base
+
+
+class FileDownloadNotFound(HTTPException):
+    def __init__(self) -> None:
+        super().__init__(
+            HTTPStatus.NOT_FOUND,
+            f"Requested download file was not found. It must have expired",
+        )
+
+
+class FileDownloadNotReady(HTTPException):
+    def __init__(self) -> None:
+        super().__init__(
+            HTTPStatus.NOT_ACCEPTABLE,
+            f"Requested file is not ready for download.",
+        )
+
+
+class FileDownloadDTO(BaseModel):
+    id: str
+    name: str
+    filename: str
+    expiration_date: Optional[str]
+    ready: bool
+    failed: bool = False
+    error_message: str = ""
+
+
+class FileDownloadTaskDTO(BaseModel):
+    file: FileDownloadDTO
+    task: str
+
+
+class FileDownload(Base):  # type: ignore
+    __tablename__ = "file_download"
+
+    id = Column(
+        String(36),
+        primary_key=True,
+        default=lambda: str(uuid.uuid4()),
+        unique=True,
+    )
+    owner = Column(Integer)
+    name = Column(String)
+    filename = Column(String)
+    path = Column(String)
+    ready = Column(Boolean, default=False)
+    expiration_date = Column(DateTime)
+    failed = Column(Boolean, default=False)
+    error_message = Column(String)
+
+    def to_dto(self) -> FileDownloadDTO:
+        return FileDownloadDTO(
+            id=self.id,
+            name=self.name,
+            filename=self.filename,
+            ready=self.ready,
+            expiration_date=str(self.expiration_date),
+            failed=self.failed or False,
+            error_message=self.error_message or "",
+        )
+
+    def __repr__(self) -> str:
+        return f"(id={self.id},name={self.name},filename={self.filename},path={self.path},ready={self.ready},expiration_date={self.expiration_date})"

antarest/core/filetransfer/repository.py

@@ -0,0 +1,38 @@
+from typing import Optional, List
+
+from antarest.core.filetransfer.model import FileDownload
+from antarest.core.utils.fastapi_sqlalchemy import db
+
+
+class FileDownloadRepository:
+    def add(self, download: FileDownload) -> None:
+        db.session.add(download)
+        db.session.commit()
+
+    def get(self, download_id: str) -> Optional[FileDownload]:
+        file_download: Optional[FileDownload] = db.session.query(
+            FileDownload
+        ).get(download_id)
+        return file_download
+
+    def save(self, download: FileDownload) -> None:
+        db.session.merge(download)
+        db.session.add(download)
+        db.session.commit()
+
+    def get_all(self, owner: Optional[int] = None) -> List[FileDownload]:
+        file_download_list: List[FileDownload] = []
+        if owner:
+            file_download_list = (
+                db.session.query(FileDownload)
+                .filter(FileDownload.owner == owner)
+                .all()
+            )
+        else:
+            file_download_list = db.session.query(FileDownload).all()
+        return file_download_list
+
+    def delete(self, download_id: str) -> None:
+        download = self.get(download_id)
+        if download:
+            db.session.delete(download)