Add docker-compose prod deployment template (#952)

Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.8-slim-buster
+#FROM python:3.8-slim-buster
+FROM brunneis/python:3.8.3-ubuntu-20.04
 
 ENV ANTAREST_CONF /resources/application.yaml
 

README.md

@@ -1,10 +1,14 @@
-# AntaREST Storage
+# Antares Web
 
 [![CI](https://github.com/AntaresSimulatorTeam/AntaREST/workflows/main/badge.svg)](https://github.com/AntaresSimulatorTeam/AntaREST/actions?query=workflow%3Amain)
 [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=AntaresSimulatorTeam_api-iso-antares&metric=coverage)](https://sonarcloud.io/dashboard?id=AntaresSimulatorTeam_api-iso-antares)
 [![Licence](https://img.shields.io/github/license/AntaresSimulatorTeam/AntaREST)](https://www.apache.org/licenses/LICENSE-2.0)
 
-<!-- ![architecture api antares](./docs/images/archi-api-antares.png) -->
+![Screenshot](./docs/assets/media/img/readme_screenshot.png)
+
+## Documentation
+
+The full project documentation can be found in the [readthedocs website](https://antares-web.readthedocs.io/en/latest).
 
 ## Build the API
 

alembic/versions/9846e90c2868_fix_bot_foreign_key.py

@@ -5,14 +5,14 @@
 Create Date: 2021-11-19 11:58:11.378519
 
 """
-from sqlite3 import Connection
 
 from alembic import op
 import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
 from sqlalchemy import text
+from sqlalchemy.engine import Connection
 
 from antarest.login.model import Bot, User
 

antarest/study/storage/rawstudy/watcher.py

@@ -1,5 +1,6 @@
 import logging
 import re
+import tempfile
 import threading
 from html import escape
 from http import HTTPStatus
@@ -33,8 +34,8 @@ class Watcher:
     Files Watcher to listen raw studies changes and trigger a database update.
     """
 
-    LOCK = Path("watcher")
-    SCAN_LOCK = Path("scan.lock")
+    LOCK = Path(tempfile.gettempdir()) / "watcher"
+    SCAN_LOCK = Path(tempfile.gettempdir()) / "scan.lock"
 
     def __init__(
         self,
@@ -256,7 +257,7 @@ def scan(
             logger.info(
                 f"Waiting for FileLock to synchronize {directory_path or 'all studies'}"
             )
-            with FileLock(self.config.storage.tmp_dir / Watcher.SCAN_LOCK):
+            with FileLock(Watcher.SCAN_LOCK):
                 logger.info(
                     f"FileLock acquired to synchronize for {directory_path or 'all studies'}"
                 )

antarest/study/storage/variantstudy/business/matrix_constants_generator.py

@@ -1,3 +1,5 @@
+import tempfile
+from pathlib import Path
 from typing import Dict
 
 from filelock import FileLock
@@ -32,7 +34,9 @@ class GeneratorMatrixConstants:
     def __init__(self, matrix_service: ISimpleMatrixService) -> None:
         self.hashes: Dict[str, str] = {}
         self.matrix_service: ISimpleMatrixService = matrix_service
-        with FileLock("matrix_constant_init.lock"):
+        with FileLock(
+            str(Path(tempfile.gettempdir()) / "matrix_constant_init.lock")
+        ):
             self._init()
 
     def _init(self) -> None:

docker-compose.override.yml.example

@@ -0,0 +1,10 @@
+version: '2.1'
+services:
+  antares-antarest:
+    user: UID:GID
+  antares-antarest-watcher:
+    user: UID:GID
+  antares-antarest-matrix-gc:
+    user: UID:GID
+  postgresql:
+    user: UID:GID

docker-compose.yml

@@ -0,0 +1,65 @@
+version: '2.1'
+services:
+  antares-antarest:
+    image: antarest:latest
+    container_name : antarest
+    environment :
+      - UVICORN_ROOT_PATH=/api
+    depends_on:
+      - redis
+      - postgresql
+    volumes:
+      - ./resources/deploy/examples:/workspaces
+      - ./resources/deploy/tmp:/antarest_tmp_dir
+      - ./resources/deploy/matrices:/matrixstore
+      - ./resources/deploy/config.prod.yaml:/resources/application.yaml
+      - ./resources/deploy/logs:/logs
+      - ./resources/deploy/gunicorn.py:/conf/gunicorn.py
+      - ./antares-8.2.2-Ubuntu-20.04/bin:/antares_simulator
+  antares-antarest-watcher:
+    image: antarest:latest
+    container_name: antarest-watcher
+    volumes:
+      - ./resources/deploy/examples:/workspaces
+      - ./resources/deploy/tmp:/antarest_tmp_dir
+      - ./resources/deploy/matrices:/matrixstore
+      - ./resources/deploy/config.prod.yaml:/resources/application.yaml
+      - ./resources/deploy/logs:/logs
+    depends_on:
+      - antares-antarest
+    command: watcher
+  antares-antarest-matrix-gc:
+    image: antarest:latest
+    container_name : antarest-matrix-gc
+    volumes:
+      - ./resources/deploy/examples:/workspaces
+      - ./resources/deploy/tmp:/antarest_tmp_dir
+      - ./resources/deploy/matrices:/matrixstore
+      - ./resources/deploy/config.prod.yaml:/resources/application.yaml
+      - ./resources/deploy/logs:/logs
+    depends_on:
+      - antares-antarest
+    command: matrix_gc
+  postgresql:
+    image: postgres:latest
+    container_name: postgres
+    environment:
+      - POSTGRES_PASSWORD=somepass
+      - PG_DATA=/var/lib/postgresql/data/pgdata
+    volumes:
+      - ./resources/deploy/db:/var/lib/postgresql/data
+    command: [ "postgres", "-c", "log_statement=all", "-c", "log_destination=stderr" ]
+  redis:
+    image: redis:latest
+    container_name : redis
+  nginx:
+    image: nginx:latest
+    container_name: nginx
+    depends_on:
+      - antares-antarest
+    ports:
+      - 80:80
+    volumes:
+      - ./resources/deploy/nginx.conf:/etc/nginx/conf.d/default.conf:ro
+      - ./webapp/build:/www
+      - ./resources/deploy/web.config.json:/www/config.json:ro

docs/install/0-INSTALL.md

@@ -5,7 +5,11 @@ The front end is a [React](https://reactjs.org/) web application. A local build
 
 ## Quick start
 
-First clone the projet:
+Requirements : 
+- python : 3.8.x
+- node : 14.x
+
+1. First clone the projet:
 
 ```
 git clone https://github.com/AntaresSimulatorTeam/AntaREST.git
@@ -14,24 +18,23 @@ git submodule init
 git submodule update
 ```
 
-Install back dependencies
+2. Install back dependencies
 
 ```
 python -m pip install --upgrade pip
 pip install pydantic --no-binary pydantic
 pip install -r requirements.txt  # use requirements-dev.txt if building a single binary with pyinstaller 
 ```
 
-Build front
+3. Build front (for local mode use `cd ..; ./scripts/build-front.sh` instead of `npm run build`)
 
 ```
 cd webapp
 npm install
-cd ..
-./scripts/build-front.sh
+npm run build 
 ```
 
-Run the application
+4. Run the application
 
 ```
 export PYTHONPATH=$(pwd)
@@ -40,3 +43,6 @@ python antarest/main.py -c resources/application.yaml --auto-upgrade-db
 
 ## Deploy
 
+There are 2 ways to use and/or deploy the application :
+- As [a server application](./2-DEPLOY.md#production-server-deployment)
+- As [a desktop systray application](./2-DEPLOY.md#local-application-build)

docs/install/2-DEPLOY.md

@@ -2,15 +2,69 @@
 
 This application can be used in two modes:
 - a production dockerized environment
-- as a local desktop application
+- a local desktop application
 
 ## Production server deployment
 
-Requires :
+The production server deployment uses `docker` and `docker-compose` to run the following containers :
+- antarest : the web application workers
+- antarest-watcher : the workspace scanner worker
+- antarest-matrix-gc: the matrices garbage collector worker 
+- redis : the cache that allows the multiple application server workers to synchronize
+- postgresql : the database
+- nginx : the web server front end (can be used to set up ssl)
+
+The following example shows how to deploy this simple base environment.
+
+### Example deployment steps
+
+Requirements :
+- a linux host
 - docker
-- redis
-- postgresql
-- slurm cluster
+- docker-compose
+
+These steps should work on any linux system with docker and docker-compose installed.
+
+0. First, the steps 1 and 3 of the [quick start build](0-INSTALL.md#quick-start) must have been done. So this guide will assume that you have previously cloned the [code repository](https://github.com/AntaresSimulatorTeam/AntaREST)
+   (don't forget the git submodule), the frontend built and that your working directory is at the root of the project.
+
+1. Then download and unzip AntaresSimulator binaries:
+```
+wget https://github.com/AntaresSimulatorTeam/Antares_Simulator/releases/download/v8.2.2/antares-8.2.2-Ubuntu-20.04.tar.gz
+tar xzf antares-8.2.2-Ubuntu-20.04.tar.gz
+```
+
+2. Build the docker image
+```
+docker build -t antarest:latest .
+```
+
+3. Prepare the environment (This is important, in order to prevent docker containers to write files into your file system with root permissions.)  
+   a. Copy `docker-compose.override.yml.example` to `docker-compose.override.yml` and replace the UID and GUI values with your user's one.  
+You can get these values by running the following commands:
+   - UID: `id -u`
+   - GID: `id -g`
+
+   b. Create the directory `resources/deploy/db`
+
+
+4. Run the following command to spin up the application containers :  
+`docker-compose up`
+
+5. You can then access the application at http://localhost
+
+6. To stop the application you can juste hit "CTRL-C" to end the containers
+
+This is an example deployment.  
+You'll have to edit your own `docker-compose.yml` file and [`application.yaml` configuration](./1-CONFIG.md) to customize it to your needs.
+
+## Local application build
 
+The local application is a bundled build of the web server to ease its launch as a kind of desktop application.  
+When started, the application will be shown as a systray application (icon in the bottom right corner of the Windows bar). The menu will allow to go
+to the local address where the interface is available.
 
-## Local application build
+The build is directly available in the [release](https://github.com/AntaresSimulatorTeam/AntaREST/releases) files for each version.
+You can download the latest version here :
+- [For Windows](https://github.com/AntaresSimulatorTeam/AntaREST/releases/download/v2.5.0/AntaresWeb-windows-latest.zip)
+- [For Ubuntu](https://github.com/AntaresSimulatorTeam/AntaREST/releases/download/v2.5.0/AntaresWeb-ubuntu-latest.zip)

resources/deploy/config.prod.yaml

@@ -0,0 +1,80 @@
+security:
+  disabled: false
+  jwt:
+    key: secretkeytochange
+  login:
+    admin:
+      pwd: admin
+  external_auth:
+    url: ""
+    default_group_role: 10
+
+db:
+  url: "postgresql://postgres:somepass@postgresql:5432/postgres"
+  admin_url: "postgresql://postgres:somepass@postgresql:5432/postgres"
+  pool_recycle: 3600
+
+storage:
+  tmp_dir: /antarest_tmp_dir
+  archive_dir: /studies/archives
+  matrixstore: /matrixstore
+  matrix_gc_dry_run: true
+  workspaces:
+    default: # required, no filters applied, this folder is not watched
+      path: /workspaces/internal_studies/
+    # other workspaces can be added
+    # if a directory is to be ignored by the watcher, place a file named AW_NO_SCAN inside
+    tmp:
+      path: /workspaces/studies/
+      # filter_in: ['.*'] # default to '.*'
+      # filter_out: [] # default to empty
+      # groups: [] # default empty
+
+launcher:
+  default: local
+  local:
+    binaries:
+      800: /antares_simulator/antares-8.2-solver
+#  slurm:
+#    local_workspace: path/to/workspace
+#    username: username
+#    hostname: 0.0.0.0
+#    port: 22
+#    private_key_file: path/to/key
+#    key_password: key_password
+#    password: password_is_optional_but_necessary_if_key_is_absent
+#    default_wait_time: 900
+#    default_time_limit: 172800
+#    default_n_cpu: 12
+#    default_json_db_name: launcher_db.json
+#    slurm_script_path: /path/to/launchantares_v1.1.3.sh
+#    db_primary_key: name
+#    antares_versions_on_remote_server :
+#      - "610"
+#      - "700"
+#      - "710"
+#      - "720"
+#      - "800"
+
+
+debug: false
+
+root_path: "/api"
+
+#tasks:
+#  max_workers: 5
+server:
+  worker_threadpool_size: 12
+#  services:
+#    - watcher
+
+logging:
+  level: INFO
+#  logfile: /logs/antarest.log
+#  json: true
+
+# Uncomment these lines to use redis as a backend for the eventbus
+# It is required to use redis when using this application on multiple workers in a preforked model like gunicorn for instance
+redis:
+  host: redis
+  port: 6379

resources/deploy/config.yaml

@@ -72,7 +72,6 @@ logging:
 
 # Uncomment these lines to use redis as a backend for the eventbus
 # It is required to use redis when using this application on multiple workers in a preforked model like gunicorn for instance
-#eventbus:
-#  redis:
-#    host: localhost
-#    port: 6379
+#redis:
+#  host: localhost
+#  port: 6379