diff --git a/Dockerfile b/Dockerfile index ee2825af17..dccc2ff4fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.8-slim-buster +#FROM python:3.8-slim-buster +FROM brunneis/python:3.8.3-ubuntu-20.04 ENV ANTAREST_CONF /resources/application.yaml diff --git a/README.md b/README.md index 3d6df51e0c..a9a3db1298 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,14 @@ -# AntaREST Storage +# Antares Web [![CI](https://github.com/AntaresSimulatorTeam/AntaREST/workflows/main/badge.svg)](https://github.com/AntaresSimulatorTeam/AntaREST/actions?query=workflow%3Amain) [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=AntaresSimulatorTeam_api-iso-antares&metric=coverage)](https://sonarcloud.io/dashboard?id=AntaresSimulatorTeam_api-iso-antares) [![Licence](https://img.shields.io/github/license/AntaresSimulatorTeam/AntaREST)](https://www.apache.org/licenses/LICENSE-2.0) - +![Screenshot](./docs/assets/media/img/readme_screenshot.png) + +## Documentation + +The full project documentation can be found in the [readthedocs website](https://antares-web.readthedocs.io/en/latest). ## Build the API diff --git a/alembic/versions/9846e90c2868_fix_bot_foreign_key.py b/alembic/versions/9846e90c2868_fix_bot_foreign_key.py index 9fa93fc913..c79b428405 100644 --- a/alembic/versions/9846e90c2868_fix_bot_foreign_key.py +++ b/alembic/versions/9846e90c2868_fix_bot_foreign_key.py @@ -5,7 +5,6 @@ Create Date: 2021-11-19 11:58:11.378519 """ -from sqlite3 import Connection from alembic import op import sqlalchemy as sa @@ -13,6 +12,7 @@ # revision identifiers, used by Alembic. from sqlalchemy import text +from sqlalchemy.engine import Connection from antarest.login.model import Bot, User diff --git a/antarest/study/storage/rawstudy/watcher.py b/antarest/study/storage/rawstudy/watcher.py index 3f6539f093..0648b0fd15 100644 --- a/antarest/study/storage/rawstudy/watcher.py +++ b/antarest/study/storage/rawstudy/watcher.py @@ -1,5 +1,6 @@ import logging import re +import tempfile import threading from html import escape from http import HTTPStatus @@ -33,8 +34,8 @@ class Watcher: Files Watcher to listen raw studies changes and trigger a database update. """ - LOCK = Path("watcher") - SCAN_LOCK = Path("scan.lock") + LOCK = Path(tempfile.gettempdir()) / "watcher" + SCAN_LOCK = Path(tempfile.gettempdir()) / "scan.lock" def __init__( self, @@ -256,7 +257,7 @@ def scan( logger.info( f"Waiting for FileLock to synchronize {directory_path or 'all studies'}" ) - with FileLock(self.config.storage.tmp_dir / Watcher.SCAN_LOCK): + with FileLock(Watcher.SCAN_LOCK): logger.info( f"FileLock acquired to synchronize for {directory_path or 'all studies'}" ) diff --git a/antarest/study/storage/variantstudy/business/matrix_constants_generator.py b/antarest/study/storage/variantstudy/business/matrix_constants_generator.py index dd47538dd2..6069acebb8 100644 --- a/antarest/study/storage/variantstudy/business/matrix_constants_generator.py +++ b/antarest/study/storage/variantstudy/business/matrix_constants_generator.py @@ -1,3 +1,5 @@ +import tempfile +from pathlib import Path from typing import Dict from filelock import FileLock @@ -32,7 +34,9 @@ class GeneratorMatrixConstants: def __init__(self, matrix_service: ISimpleMatrixService) -> None: self.hashes: Dict[str, str] = {} self.matrix_service: ISimpleMatrixService = matrix_service - with FileLock("matrix_constant_init.lock"): + with FileLock( + str(Path(tempfile.gettempdir()) / "matrix_constant_init.lock") + ): self._init() def _init(self) -> None: diff --git a/docker-compose.override.yml.example b/docker-compose.override.yml.example new file mode 100644 index 0000000000..478988454a --- /dev/null +++ b/docker-compose.override.yml.example @@ -0,0 +1,10 @@ +version: '2.1' +services: + antares-antarest: + user: UID:GID + antares-antarest-watcher: + user: UID:GID + antares-antarest-matrix-gc: + user: UID:GID + postgresql: + user: UID:GID diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..a18e3bc817 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,65 @@ +version: '2.1' +services: + antares-antarest: + image: antarest:latest + container_name : antarest + environment : + - UVICORN_ROOT_PATH=/api + depends_on: + - redis + - postgresql + volumes: + - ./resources/deploy/examples:/workspaces + - ./resources/deploy/tmp:/antarest_tmp_dir + - ./resources/deploy/matrices:/matrixstore + - ./resources/deploy/config.prod.yaml:/resources/application.yaml + - ./resources/deploy/logs:/logs + - ./resources/deploy/gunicorn.py:/conf/gunicorn.py + - ./antares-8.2.2-Ubuntu-20.04/bin:/antares_simulator + antares-antarest-watcher: + image: antarest:latest + container_name: antarest-watcher + volumes: + - ./resources/deploy/examples:/workspaces + - ./resources/deploy/tmp:/antarest_tmp_dir + - ./resources/deploy/matrices:/matrixstore + - ./resources/deploy/config.prod.yaml:/resources/application.yaml + - ./resources/deploy/logs:/logs + depends_on: + - antares-antarest + command: watcher + antares-antarest-matrix-gc: + image: antarest:latest + container_name : antarest-matrix-gc + volumes: + - ./resources/deploy/examples:/workspaces + - ./resources/deploy/tmp:/antarest_tmp_dir + - ./resources/deploy/matrices:/matrixstore + - ./resources/deploy/config.prod.yaml:/resources/application.yaml + - ./resources/deploy/logs:/logs + depends_on: + - antares-antarest + command: matrix_gc + postgresql: + image: postgres:latest + container_name: postgres + environment: + - POSTGRES_PASSWORD=somepass + - PG_DATA=/var/lib/postgresql/data/pgdata + volumes: + - ./resources/deploy/db:/var/lib/postgresql/data + command: [ "postgres", "-c", "log_statement=all", "-c", "log_destination=stderr" ] + redis: + image: redis:latest + container_name : redis + nginx: + image: nginx:latest + container_name: nginx + depends_on: + - antares-antarest + ports: + - 80:80 + volumes: + - ./resources/deploy/nginx.conf:/etc/nginx/conf.d/default.conf:ro + - ./webapp/build:/www + - ./resources/deploy/web.config.json:/www/config.json:ro \ No newline at end of file diff --git a/docs/assets/media/img/readme_screenshot.png b/docs/assets/media/img/readme_screenshot.png new file mode 100644 index 0000000000..c555306ecd Binary files /dev/null and b/docs/assets/media/img/readme_screenshot.png differ diff --git a/docs/install/0-INSTALL.md b/docs/install/0-INSTALL.md index e26d939a9a..6703b7e3d8 100644 --- a/docs/install/0-INSTALL.md +++ b/docs/install/0-INSTALL.md @@ -5,7 +5,11 @@ The front end is a [React](https://reactjs.org/) web application. A local build ## Quick start -First clone the projet: +Requirements : +- python : 3.8.x +- node : 14.x + +1. First clone the projet: ``` git clone https://github.com/AntaresSimulatorTeam/AntaREST.git @@ -14,7 +18,7 @@ git submodule init git submodule update ``` -Install back dependencies +2. Install back dependencies ``` python -m pip install --upgrade pip @@ -22,16 +26,15 @@ pip install pydantic --no-binary pydantic pip install -r requirements.txt # use requirements-dev.txt if building a single binary with pyinstaller ``` -Build front +3. Build front (for local mode use `cd ..; ./scripts/build-front.sh` instead of `npm run build`) ``` cd webapp npm install -cd .. -./scripts/build-front.sh +npm run build ``` -Run the application +4. Run the application ``` export PYTHONPATH=$(pwd) @@ -40,3 +43,6 @@ python antarest/main.py -c resources/application.yaml --auto-upgrade-db ## Deploy +There are 2 ways to use and/or deploy the application : +- As [a server application](./2-DEPLOY.md#production-server-deployment) +- As [a desktop systray application](./2-DEPLOY.md#local-application-build) \ No newline at end of file diff --git a/docs/install/2-DEPLOY.md b/docs/install/2-DEPLOY.md index 20deb0f35b..be727fdcd7 100644 --- a/docs/install/2-DEPLOY.md +++ b/docs/install/2-DEPLOY.md @@ -2,15 +2,69 @@ This application can be used in two modes: - a production dockerized environment -- as a local desktop application +- a local desktop application ## Production server deployment -Requires : +The production server deployment uses `docker` and `docker-compose` to run the following containers : +- antarest : the web application workers +- antarest-watcher : the workspace scanner worker +- antarest-matrix-gc: the matrices garbage collector worker +- redis : the cache that allows the multiple application server workers to synchronize +- postgresql : the database +- nginx : the web server front end (can be used to set up ssl) + +The following example shows how to deploy this simple base environment. + +### Example deployment steps + +Requirements : +- a linux host - docker -- redis -- postgresql -- slurm cluster +- docker-compose + +These steps should work on any linux system with docker and docker-compose installed. + +0. First, the steps 1 and 3 of the [quick start build](0-INSTALL.md#quick-start) must have been done. So this guide will assume that you have previously cloned the [code repository](https://github.com/AntaresSimulatorTeam/AntaREST) + (don't forget the git submodule), the frontend built and that your working directory is at the root of the project. + +1. Then download and unzip AntaresSimulator binaries: +``` +wget https://github.com/AntaresSimulatorTeam/Antares_Simulator/releases/download/v8.2.2/antares-8.2.2-Ubuntu-20.04.tar.gz +tar xzf antares-8.2.2-Ubuntu-20.04.tar.gz +``` + +2. Build the docker image +``` +docker build -t antarest:latest . +``` + +3. Prepare the environment (This is important, in order to prevent docker containers to write files into your file system with root permissions.) + a. Copy `docker-compose.override.yml.example` to `docker-compose.override.yml` and replace the UID and GUI values with your user's one. +You can get these values by running the following commands: + - UID: `id -u` + - GID: `id -g` + + b. Create the directory `resources/deploy/db` + + +4. Run the following command to spin up the application containers : +`docker-compose up` + +5. You can then access the application at http://localhost + +6. To stop the application you can juste hit "CTRL-C" to end the containers + +This is an example deployment. +You'll have to edit your own `docker-compose.yml` file and [`application.yaml` configuration](./1-CONFIG.md) to customize it to your needs. + +## Local application build +The local application is a bundled build of the web server to ease its launch as a kind of desktop application. +When started, the application will be shown as a systray application (icon in the bottom right corner of the Windows bar). The menu will allow to go +to the local address where the interface is available. -## Local application build \ No newline at end of file +The build is directly available in the [release](https://github.com/AntaresSimulatorTeam/AntaREST/releases) files for each version. +You can download the latest version here : +- [For Windows](https://github.com/AntaresSimulatorTeam/AntaREST/releases/download/v2.5.0/AntaresWeb-windows-latest.zip) +- [For Ubuntu](https://github.com/AntaresSimulatorTeam/AntaREST/releases/download/v2.5.0/AntaresWeb-ubuntu-latest.zip) diff --git a/resources/deploy/config.prod.yaml b/resources/deploy/config.prod.yaml new file mode 100644 index 0000000000..1bb5e30878 --- /dev/null +++ b/resources/deploy/config.prod.yaml @@ -0,0 +1,80 @@ +security: + disabled: false + jwt: + key: secretkeytochange + login: + admin: + pwd: admin + external_auth: + url: "" + default_group_role: 10 + +db: + url: "postgresql://postgres:somepass@postgresql:5432/postgres" + admin_url: "postgresql://postgres:somepass@postgresql:5432/postgres" + pool_recycle: 3600 + +storage: + tmp_dir: /antarest_tmp_dir + archive_dir: /studies/archives + matrixstore: /matrixstore + matrix_gc_dry_run: true + workspaces: + default: # required, no filters applied, this folder is not watched + path: /workspaces/internal_studies/ + # other workspaces can be added + # if a directory is to be ignored by the watcher, place a file named AW_NO_SCAN inside + tmp: + path: /workspaces/studies/ + # filter_in: ['.*'] # default to '.*' + # filter_out: [] # default to empty + # groups: [] # default empty + +launcher: + default: local + local: + binaries: + 800: /antares_simulator/antares-8.2-solver +# slurm: +# local_workspace: path/to/workspace +# username: username +# hostname: 0.0.0.0 +# port: 22 +# private_key_file: path/to/key +# key_password: key_password +# password: password_is_optional_but_necessary_if_key_is_absent +# default_wait_time: 900 +# default_time_limit: 172800 +# default_n_cpu: 12 +# default_json_db_name: launcher_db.json +# slurm_script_path: /path/to/launchantares_v1.1.3.sh +# db_primary_key: name +# antares_versions_on_remote_server : +# - "610" +# - "700" +# - "710" +# - "720" +# - "800" + + +debug: false + +root_path: "/api" + +#tasks: +# max_workers: 5 +server: + worker_threadpool_size: 12 +# services: +# - watcher + +logging: + level: INFO +# logfile: /logs/antarest.log +# json: true + +# Uncomment these lines to use redis as a backend for the eventbus +# It is required to use redis when using this application on multiple workers in a preforked model like gunicorn for instance +redis: + host: redis + port: 6379 diff --git a/resources/deploy/config.yaml b/resources/deploy/config.yaml index 82c210ce75..e839384ebc 100644 --- a/resources/deploy/config.yaml +++ b/resources/deploy/config.yaml @@ -72,7 +72,6 @@ logging: # Uncomment these lines to use redis as a backend for the eventbus # It is required to use redis when using this application on multiple workers in a preforked model like gunicorn for instance -#eventbus: -# redis: -# host: localhost -# port: 6379 +#redis: +# host: localhost +# port: 6379 diff --git a/resources/deploy/gunicorn.py b/resources/deploy/gunicorn.py new file mode 100644 index 0000000000..26f063db13 --- /dev/null +++ b/resources/deploy/gunicorn.py @@ -0,0 +1,30 @@ +# Reference: https://github.com/benoitc/gunicorn/blob/master/examples/example_config.py + +import multiprocessing +import os + +bind = "0.0.0.0:5000" + +""" +Gunicorn relies on the operating system to provide all of the load balancing +when handling requests. Generally we recommend (2 x $num_cores) + 1 +as the number of workers to start off with. While not overly scientific, +the formula is based on the assumption that for a given core, +one worker will be reading or writing from the socket +while the other worker is processing a request. +https://docs.gunicorn.org/en/stable/design.html#how-many-workers +""" + +workers = os.getenv("GUNICORN_WORKERS") +if workers == "ALL_AVAILABLE" or workers is None: + workers = multiprocessing.cpu_count() * 2 + 1 + +timeout = 10 * 120 # 10 minutes +keepalive = 24 * 60 * 60 # 1 day + +capture_output = True + +loglevel = "info" +errorlog = "/logs/gunicorn.error.log" +accesslog = "/logs/gunicorn.access.log" +preload_app = False diff --git a/resources/deploy/logs/.placeholder b/resources/deploy/logs/.placeholder new file mode 100644 index 0000000000..e69de29bb2 diff --git a/resources/deploy/nginx.conf b/resources/deploy/nginx.conf new file mode 100644 index 0000000000..393e258292 --- /dev/null +++ b/resources/deploy/nginx.conf @@ -0,0 +1,24 @@ +server { + listen 80; + client_max_body_size 1G; + + # serve static webapp files + location / { + root /www; + try_files $uri $uri/ /index.html = 404; + } + + location /api/ { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Host $host:$server_port; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_redirect off; + proxy_pass http://antarest:5000/; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 1200; + } +} \ No newline at end of file diff --git a/resources/deploy/web.config.json b/resources/deploy/web.config.json new file mode 100644 index 0000000000..9c89d31d6a --- /dev/null +++ b/resources/deploy/web.config.json @@ -0,0 +1,4 @@ +{ + "restEndpoint": "/api", + "wsEndpoint": "/api/ws" +} \ No newline at end of file