diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18a07d75..4b00143a 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,19 +10,25 @@ on: jobs: test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - uses: FNNDSC/cube-integration-action@master - with: - repository: ${{ github.workspace }} - which: all - + - uses: taiki-e/install-action@just + - name: Build development image + run: just build + - name: Pull other images + run: just pull + - name: Start ancillary services + run: just start-ancillary + - name: Run unit tests + run: just test-unit + - name: Run integration tests + run: just test-integration build: needs: [ test ] if: github.event_name == 'push' || github.event_name == 'release' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: repo_name: fnndsc/cube steps: diff --git a/OLD_DEVELOP.md b/OLD_DEVELOP.md new file mode 100644 index 00000000..1d05bc8b --- /dev/null +++ b/OLD_DEVELOP.md @@ -0,0 +1,206 @@ +# ChRIS\_ultron\_backEnd Old Development Scripts + +### Preconditions + +#### Operating system support -- please read + +##### Linux + +Linux is the first class host platform for all things _CUBE_ related. Linux distributions used by various core developers include Ubuntu, Arch, and Fedora. The development team is happy to provide help to folks trying / struggling to run _CUBE_ on most any Linux distribution. + +##### macOS + +macOS is fully supported as a host platform for _CUBE_. Please note that you **must update/tweak some things first**. Most importantly, macOS is distributed with a deprecated version of the `bash` shell **that will not work with our Makefile**. If you want to host _CUBE_ on macOS, you **must** update `bash` to a current version. Instructions are out of scope of this document, but we recommend [homebrew](https://brew.sh) as your friend here. + +##### Windows + +In a word, **don't** (ok, that's technically two words). _CUBE_ is ideally meant to be deployed on Linux/*nix systems. **Windows is not officially supported nor recommended as the host environment**. If you insist on trying on Windows you can consult some unmaintained documentation on attempts to deploy _CUBE_ using the Windows Subsystem for Linux (WSL) [here](https://github.com/FNNDSC/CHRIS_docs/blob/master/workflows/ChRIS_on_WSL.asciidoc). This probably will break. Note that currently no one on the core development uses Windows in much of any capacity so interest or knowledge to help questions about Windows support is low. Nonetheless, we would welcome any brave soul though who has the time and inclination to fully investigate _CUBE_ on Windows deployment. + + +#### Install latest Docker and Docker Compose. + +Currently tested platforms: +* ``Docker 18.06.0+`` +* ``Docker Compose 1.27.0+`` +* ``Ubuntu 18.04+ and MAC OS X 10.14+`` + +#### On a Linux machine make sure to add your computer user to the ``docker`` group + +Consult this page https://docs.docker.com/engine/install/linux-postinstall/ + +### TL;DR + +#### If you read nothing else on this page, and just want to get an instance of the ChRIS backend services up and running with no mess, no fuss: + +##### The real TL;DR + +The all in one copy/paste line to drop into your terminal (assuming of course you are in the repo directory and have the preconditions met): + +```bash +docker swarm leave --force && docker swarm init --advertise-addr 127.0.0.1 && \ +./unmake.sh && sudo rm -fr CHRIS_REMOTE_FS && rm -fr CHRIS_REMOTE_FS && \ +./make.sh -U -I -i +``` + +This will start a **bare bones** _CUBE_. This _CUBE_ will **NOT** have any plugins installed. To install a set of plugins, do + +```bash +./postscript.sh +``` + +##### Slightly longer but still short TL;DR + +Start a local Docker Swarm cluster if not already started: + +```bash +docker swarm init --advertise-addr 127.0.0.1 +``` + +Get the source code from CUBE repo: + +```bash +git clone https://github.com/FNNDSC/ChRIS_ultron_backend +cd ChRIS_ultron_backend +``` + +Run full CUBE instantiation with tests: +```bash +./unmake.sh ; sudo rm -fr CHRIS_REMOTE_FS; rm -fr CHRIS_REMOTE_FS; ./make.sh +``` + +Or skip unit and integration tests and the intro: +```bash +./unmake.sh ; sudo rm -fr CHRIS_REMOTE_FS; rm -fr CHRIS_REMOTE_FS; ./make.sh -U -I -s +``` + +Once the system is "up" you can add more compute plugins to the ecosystem: + +```bash +./postscript.sh +``` + +The resulting CUBE instance uses the default Django development server and therefore is not suitable for production. + + +#### Docker Swarm-based development environment: + +Start a local Docker Swarm cluster if not already started: + +```bash +docker swarm init --advertise-addr 127.0.0.1 +``` + +Start CUBE from the repository source directory by running the make bash script + +```bash +git clone https://github.com/FNNDSC/ChRIS_ultron_backEnd.git +cd ChRIS_ultron_backEnd +./make.sh +``` +All the steps performed by the above script are properly documented in the script itself. +After running this script all the automated tests should have successfully run and a Django development server should be running in interactive mode in this terminal. + +Later you can stop and remove CUBE services and storage space by running the following bash script from the repository source directory: + +```bash +./unmake.sh +``` + +Then remove the local Docker Swarm cluster if desired: + +```bash +docker swarm leave --force +``` + +#### Kubernetes-based development environment: + +Install single-node Kubernetes cluster. +On MAC OS Docker Desktop includes a standalone Kubernetes server and client. +Consult this page https://docs.docker.com/desktop/kubernetes/. +On Linux there is a simple MicroK8s installation. Consult this page https://microk8s.io. +Then create the required alias: + +```bash +snap alias microk8s.kubectl kubectl +microk8s.kubectl config view --raw > $HOME/.kube/config +``` + +Start the Kubernetes cluster: + +```bash +microk8s start +``` + +Start CUBE from the repository source directory by running the make bash script + +```bash +git clone https://github.com/FNNDSC/ChRIS_ultron_backEnd.git +cd ChRIS_ultron_backEnd +export HOSTIP= +./make.sh -O kubernetes +``` + +Later you can stop and remove CUBE services and storage space by running the following bash script from the repository source directory: + +```bash +./unmake.sh -O kubernetes +``` + +Stop the Kubernetes cluster if desired: + +```bash +microk8s stop +``` + +#### Rerun automated tests after modifying source code + +Open another terminal and run the Unit and Integration tests within the container running the Django server: + +To run only the Unit tests: + +```bash +cd ChRIS_ultron_backEnd +docker compose -f docker-compose_dev.yml exec chris_dev python manage.py test --exclude-tag integration +``` + +To run only the Integration tests: + +```bash +docker compose -f docker-compose_dev.yml exec chris_dev python manage.py test --tag integration +``` + +To run all the tests: + +```bash +docker compose -f docker-compose_dev.yml exec chris_dev python manage.py test +``` + +After running the Integration tests the ``./CHRIS_REMOTE_FS`` directory **must** be empty otherwise it means some error has occurred and you should manually empty it. + + +#### Check code coverage of the automated tests +Make sure the ``chris_backend/`` dir is world writable. Then type: + +```bash +docker compose -f docker-compose_dev.yml exec chris_dev coverage run --source=feeds,plugins,userfiles,users manage.py test +docker compose -f docker-compose_dev.yml exec chris_dev coverage report +``` + +#### Using [HTTPie](https://httpie.org/) client to play with the REST API +A simple GET request to retrieve the user-specific list of feeds: +```bash +http -a cube:cube1234 http://localhost:8000/api/v1/ +``` +A simple POST request to run the plugin with id 1: +```bash +http -a cube:cube1234 POST http://localhost:8000/api/v1/plugins/1/instances/ Content-Type:application/vnd.collection+json Accept:application/vnd.collection+json template:='{"data":[{"name":"dir","value":"cube/"}]}' +``` +Then keep making the following GET request until the ``"status"`` descriptor in the response becomes ``"finishedSuccessfully"``: +```bash +http -a cube:cube1234 http://localhost:8000/api/v1/plugins/instances/1/ +``` + +#### Using swift client to list files in the users bucket +```bash +swift -A http://127.0.0.1:8080/auth/v1.0 -U chris:chris1234 -K testing list users +``` diff --git a/README.md b/README.md index a6e25166..4721cad4 100755 --- a/README.md +++ b/README.md @@ -1,241 +1,153 @@ -# ![ChRIS logo](https://github.com/FNNDSC/ChRIS_ultron_backEnd/blob/master/docs/assets/logo_chris.png) ChRIS\_ultron\_backEnd +# ![ChRIS logo](./docs/assets/logo_chris.png) ChRIS\_ultron\_backEnd [![Build](https://github.com/FNNDSC/ChRIS_ultron_backEnd/actions/workflows/ci.yml/badge.svg)](https://github.com/FNNDSC/ChRIS_ultron_backEnd/actions/workflows/ci.yml) [![License](https://img.shields.io/github/license/fnndsc/ChRIS_ultron_backEnd.svg)](./LICENSE) _ChRIS_ is an open-source platform for containerized medical compute. -The _ChRIS_ backend, a.k.a. _ChRIS Ultron Backend_ or _**CUBE**_ for short, -is a component of the _ChRIS_ system. - -![Architecture Diagram](https://chrisproject.org/img/figures/ChRIS_architecture.svg#gh-light-mode-only) -![Architecture Diagram](https://chrisproject.org/img/figures/ChRIS_architecture_dark.svg#gh-dark-mode-only) - -The core backend service for the ChRIS distributed software platform, also known by the anacronym _CUBE_. Internally the service is implemented as a Django-PostgreSQL project offering a [collection+json](http://amundsen.com/media-types/collection/) REST API. Important ancillary components include the ``pfcon`` and ``pman`` file transfer and remote process management microservices. - - -## ChRIS development, testing and deployment - -### Abstract - -_ChRIS Ultron Back End_ (sometimes also _ChRIS Underlying Back End_) or simply _CUBE_ is the main core of the ChRIS system. _CUBE_ provides the main REST API to the ChRIS system, as well as maintaining an internal database of users, files, pipelines, and plugins. Currently _CUBE_ has two separate compute paradigms depending on deployment context. In the case of _development_ all components of _CUBE_ use `docker` and `docker swarm` technologies. In the case of _production_ technologies such as `openshift` and `kubernetes` are also supported. - -Consult this page for instructions on starting _CUBE_ in either _development_ or _production_ contexts. For documentation/overview/background, please see the [documention](https://github.com/FNNDSC/ChRIS_docs). - - -### Preconditions - -#### Operating system support -- please read - -##### Linux - -Linux is the first class host platform for all things _CUBE_ related. Linux distributions used by various core developers include Ubuntu, Arch, and Fedora. The development team is happy to provide help to folks trying / struggling to run _CUBE_ on most any Linux distribution. - -##### macOS - -macOS is fully supported as a host platform for _CUBE_. Please note that you **must update/tweak some things first**. Most importantly, macOS is distributed with a deprecated version of the `bash` shell **that will not work with our Makefile**. If you want to host _CUBE_ on macOS, you **must** update `bash` to a current version. Instructions are out of scope of this document, but we recommend [homebrew](https://brew.sh) as your friend here. -##### Windows +https://chrisproject.org/ -In a word, **don't** (ok, that's technically two words). _CUBE_ is ideally meant to be deployed on Linux/*nix systems. **Windows is not officially supported nor recommended as the host environment**. If you insist on trying on Windows you can consult some unmaintained documentation on attempts to deploy _CUBE_ using the Windows Subsystem for Linux (WSL) [here](https://github.com/FNNDSC/CHRIS_docs/blob/master/workflows/ChRIS_on_WSL.asciidoc). This probably will break. Note that currently no one on the core development uses Windows in much of any capacity so interest or knowledge to help questions about Windows support is low. Nonetheless, we would welcome any brave soul though who has the time and inclination to fully investigate _CUBE_ on Windows deployment. +## TL;DR - -#### Install latest Docker and Docker Compose. +With [Docker Compose](https://docs.docker.com/compose/) and [just](https://just.systems/) installed, run -Currently tested platforms: -* ``Docker 18.06.0+`` -* ``Docker Compose 1.27.0+`` -* ``Ubuntu 18.04+ and MAC OS X 10.14+`` - -#### On a Linux machine make sure to add your computer user to the ``docker`` group - -Consult this page https://docs.docker.com/engine/install/linux-postinstall/ - -### TL;DR - -#### If you read nothing else on this page, and just want to get an instance of the ChRIS backend services up and running with no mess, no fuss: - -##### The real TL;DR - -The all in one copy/paste line to drop into your terminal (assuming of course you are in the repo directory and have the preconditions met): - -```bash -docker swarm leave --force && docker swarm init --advertise-addr 127.0.0.1 && \ -./unmake.sh && sudo rm -fr CHRIS_REMOTE_FS && rm -fr CHRIS_REMOTE_FS && \ -./make.sh -U -I -i +```shell +git clone https://github.com/FNNDSC/ChRIS_ultron_backEnd.git +cd ChRIS_ultron_backEnd +just ``` -This will start a **bare bones** _CUBE_. This _CUBE_ will **NOT** have any plugins installed. To install a set of plugins, do - -```bash -./postscript.sh -``` +## Introduction -##### Slightly longer but still short TL;DR +The _ChRIS_ backend, a.k.a. _ChRIS Ultron Backend_ or _**CUBE**_ for short, +is a component of the _ChRIS_ system. It is responsible for maintaining the database +of users, files, plugins, and pipelines. -Start a local Docker Swarm cluster if not already started: + + + + Architecture diagram + -```bash -docker swarm init --advertise-addr 127.0.0.1 -``` +Here lives the code of _CUBE_. It is a Django project using PostgreSQL and Celery. +The HTTP API primarily supports the [collection+json](http://amundsen.com/media-types/collection/) content-type. -Get the source code from CUBE repo: +## Development -```bash -git clone https://github.com/FNNDSC/ChRIS_ultron_backend -cd ChRIS_ultron_backend -``` +Development is mainly supported on Linux. MacOS and WSL on Windows also work (because Docker Desktop is a Linux VM). You will need at least 8GM RAM, 20GB disk space, and a good internet connection. -Run full CUBE instantiation with tests: -```bash -./unmake.sh ; sudo rm -fr CHRIS_REMOTE_FS; rm -fr CHRIS_REMOTE_FS; ./make.sh -``` +Install Docker (version 27 or above), Docker Compose, and [just](https://github.com/casey/just?tab=readme-ov-file#installation). -Or skip unit and integration tests and the intro: -```bash -./unmake.sh ; sudo rm -fr CHRIS_REMOTE_FS; rm -fr CHRIS_REMOTE_FS; ./make.sh -U -I -s -``` +
+ +Docker Installation Instructions + -Once the system is "up" you can add more compute plugins to the ecosystem: +- For MacOS and Windows, see: https://docs.docker.com/get-started/get-docker/ +- For Linux, see: https://docs.docker.com/engine/install/ -```bash -./postscript.sh -``` +> [!CAUTION] +> On **Linux**, the official Docker Documentation will try to trick you into installing "Docker Desktop." Do not install "Docker Desktop." Look for "Docker Engine" instead. -The resulting CUBE instance uses the default Django development server and therefore is not suitable for production. +> [!CAUTION] +> On **Ubuntu**, make sure you follow the instructions here: https://docs.docker.com/engine/install/ubuntu/. If you do not follow the instructions, Ubuntu will try to install Docker using snap, which will cause many problems. +
-### Production deployments +### Just Commands -Please refer to https://github.com/FNNDSC/khris-helm +Development is handled by [`just`](https://just.systems). +Running _CUBE_ in development mode is as-simple-as running the command +```shell +just +``` -### Development +The first run of `just` will take 5-20 minutes because it needs to pull and build container images. Subsequent runs should only take 1-5 minutes. -#### Docker Swarm-based development environment: +_CUBE_ is now running at http://localhost:8000/api/v1/. You can click around in the web browser. Alternatively, check out [chrs](https://chrisproject.org/docs/chrs) and/or [ChRIS\_ui](https://github.com/FNNDSC/ChRIS_ui). -Start a local Docker Swarm cluster if not already started: +Run tests: -```bash -docker swarm init --advertise-addr 127.0.0.1 +```shell +just test-all # run all tests +just test-unit # run unit tests +just test-integration # run integration tests +just test feeds.tests.test_views # run chris_backend/feeds/tests/test_views.py ``` -Start CUBE from the repository source directory by running the make bash script +Shut down and clean up: -```bash -git clone https://github.com/FNNDSC/ChRIS_ultron_backEnd.git -cd ChRIS_ultron_backEnd -./make.sh +```shell +just nuke ``` -All the steps performed by the above script are properly documented in the script itself. -After running this script all the automated tests should have successfully run and a Django development server should be running in interactive mode in this terminal. -Later you can stop and remove CUBE services and storage space by running the following bash script from the repository source directory: +List all `just` commands: -```bash -./unmake.sh +```shell +just --list --unsorted ``` -Then remove the local Docker Swarm cluster if desired: - -```bash -docker swarm leave --force -``` +### Development Tips and Tricks -#### Kubernetes-based development environment: +#### Recreate containers after changing `docker-compose_just.yml` -Install single-node Kubernetes cluster. -On MAC OS Docker Desktop includes a standalone Kubernetes server and client. -Consult this page https://docs.docker.com/desktop/kubernetes/. -On Linux there is a simple MicroK8s installation. Consult this page https://microk8s.io. -Then create the required alias: +If you modify `docker-compose_just.yml`, you need to recreate/restart the affected services. -```bash -snap alias microk8s.kubectl kubectl -microk8s.kubectl config view --raw > $HOME/.kube/config +```shell +just up ``` -Start the Kubernetes cluster: +#### Rebuild the image after changing package dependencies -```bash -microk8s start -``` - -Start CUBE from the repository source directory by running the make bash script +If you modify `Dockerfile` or `requirements/*.txt`, you need to rebuild the image and recreate your containers. -```bash -git clone https://github.com/FNNDSC/ChRIS_ultron_backEnd.git -cd ChRIS_ultron_backEnd -export HOSTIP= -./make.sh -O kubernetes +```shell +just rebuild && just up ``` -Later you can stop and remove CUBE services and storage space by running the following bash script from the repository source directory: +#### Trying HTTP requests from the CLI -```bash -./unmake.sh -O kubernetes -``` - -Stop the Kubernetes cluster if desired: +For CLI tools, I recommend [xh](https://github.com/ducaale/xh) and [jnv](https://github.com/ynqa/jnv) or [jq](https://jqlang.github.io/jq/). Example: -```bash -microk8s stop +```shell +xh -a chris:chris1234 :8000/api/v1/ | jnv ``` -#### Rerun automated tests after modifying source code - -Open another terminal and run the Unit and Integration tests within the container running the Django server: +#### Interactive shell -To run only the Unit tests: +It is often easiest to debug things using a shell. -```bash -cd ChRIS_ultron_backEnd -docker compose -f docker-compose_dev.yml exec chris_dev python manage.py test --exclude-tag integration +```shell +just bash # run bash in a container +# -- or -- +just shell # run a Python REPL ``` -To run only the Integration tests: - -```bash -docker compose -f docker-compose_dev.yml exec chris_dev python manage.py test --tag integration -``` +In the Python REPL, you can import models and interact with them. Here is some common starter code: -To run all the tests: +```python +from django.conf import settings +from django.contrib.auth.models import User, Group +from plugins.models import * +from plugininstances.models import * +from core.storage import connect_storage -```bash -docker compose -f docker-compose_dev.yml exec chris_dev python manage.py test +storage = connect_storage(settings) ``` -After running the Integration tests the ``./CHRIS_REMOTE_FS`` directory **must** be empty otherwise it means some error has occurred and you should manually empty it. - +### Alternative Development Script -#### Check code coverage of the automated tests -Make sure the ``chris_backend/`` dir is world writable. Then type: +Old development scripts usage is described in [OLD_DEVELOP.md](./OLD_DEVELOP.md). -```bash -docker compose -f docker-compose_dev.yml exec chris_dev coverage run --source=feeds,plugins,userfiles,users manage.py test -docker compose -f docker-compose_dev.yml exec chris_dev coverage report -``` - -#### Using [HTTPie](https://httpie.org/) client to play with the REST API -A simple GET request to retrieve the user-specific list of feeds: -```bash -http -a cube:cube1234 http://localhost:8000/api/v1/ -``` -A simple POST request to run the plugin with id 1: -```bash -http -a cube:cube1234 POST http://localhost:8000/api/v1/plugins/1/instances/ Content-Type:application/vnd.collection+json Accept:application/vnd.collection+json template:='{"data":[{"name":"dir","value":"cube/"}]}' -``` -Then keep making the following GET request until the ``"status"`` descriptor in the response becomes ``"finishedSuccessfully"``: -```bash -http -a cube:cube1234 http://localhost:8000/api/v1/plugins/instances/1/ -``` +## Production Deployment -#### Using swift client to list files in the users bucket -```bash -swift -A http://127.0.0.1:8080/auth/v1.0 -U chris:chris1234 -K testing list users -``` +See https://chrisproject.org/docs/run/helm +## Documentation -### Documentation +> [!CAUTION] +> Everything below in this section is outdated. #### REST API reference @@ -260,11 +172,3 @@ Available [here](https://github.com/FNNDSC/ChRIS_ultron_backEnd/wiki/ChRIS-REST- #### ChRIS backend database design. Available [here](https://github.com/FNNDSC/ChRIS_ultron_backEnd/wiki/ChRIS-backend-database-design). - -#### Wiki. - -Available [here](https://github.com/FNNDSC/ChRIS_ultron_backEnd/wiki). - -### Learn More - -If you are interested in contributing or joining us, Check [here](http://chrisproject.org/join-us). diff --git a/chris_backend/config/settings/local.py b/chris_backend/config/settings/local.py index 29ce2c00..296cd494 100755 --- a/chris_backend/config/settings/local.py +++ b/chris_backend/config/settings/local.py @@ -101,7 +101,7 @@ MEDIA_ROOT = None if STORAGE_ENV in ('fslink', 'filesystem'): STORAGES['default'] = {'BACKEND': 'django.core.files.storage.FileSystemStorage'} - MEDIA_ROOT = '/var/chris' # local filesystem storage settings + MEDIA_ROOT = '/data' # local filesystem storage settings try: verify_storage_connection( diff --git a/docker-compose_just.yml b/docker-compose_just.yml new file mode 100644 index 00000000..f6aea634 --- /dev/null +++ b/docker-compose_just.yml @@ -0,0 +1,181 @@ +# Docker Compose file for ChRIS backend development. +# See justfile for usage. *DO NOT* use the `docker compose` command directly. + +services: + chrisomatic: + image: ghcr.io/fnndsc/chrisomatic:1.0.0 + profiles: + - tools + volumes: + - "./chrisomatic:/etc/chrisomatic:ro" + - "/var/run/docker.sock:/var/run/docker.sock" + working_dir: /etc/chrisomatic + userns_mode: host + depends_on: + - chris + networks: + - local + + chris: + image: ${CUBE_IMAGE:-localhost/fnndsc/cube:dev} + build: + context: . + args: + ENVIRONMENT: local + command: python manage.py runserver 0.0.0.0:8000 + ports: + - "8000:8000" + volumes: &CHRIS_VOLUMES + - chris_files:/data + - ./chris_backend:/opt/app-root/src + environment: &CHRIS_ENV + DJANGO_SETTINGS_MODULE: "config.settings.local" + STORAGE_ENV: "fslink" + user: ${UID:?Please run me using just.}:${GID:?Please run me using just.} + profiles: + - cube + depends_on: &CHRIS_DEPENDS + db: + condition: service_healthy + rabbitmq: + condition: service_started + cube-nonroot-user-volume-fix: + condition: service_completed_successfully + networks: + local: + aliases: + - chrisdev.local # hard-coded in chrisomatic/*.yml + worker-mains: + image: ${CUBE_IMAGE:-localhost/fnndsc/cube:dev} + command: celery -A core worker -c 4 -l info -Q main1,main2 + volumes: *CHRIS_VOLUMES + environment: *CHRIS_ENV + user: ${UID}:${GID} + profiles: + - cube + depends_on: *CHRIS_DEPENDS + networks: + - local + worker-periodic: + image: ${CUBE_IMAGE:-localhost/fnndsc/cube:dev} + command: celery -A core worker -c 2 -l info -Q periodic + volumes: *CHRIS_VOLUMES + environment: *CHRIS_ENV + user: ${UID}:${GID} + profiles: + - cube + depends_on: *CHRIS_DEPENDS + networks: + - local + celery-scheduler: + image: ${CUBE_IMAGE:-localhost/fnndsc/cube:dev} + command: celery -A core beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler + volumes: *CHRIS_VOLUMES + environment: *CHRIS_ENV + user: ${UID}:${GID} + profiles: + - cube + depends_on: *CHRIS_DEPENDS + networks: + - local + db: + image: docker.io/library/postgres:16 + volumes: + - db:/var/lib/postgresql/data + environment: + - POSTGRES_DB=chris_dev + - POSTGRES_USER=chris + - POSTGRES_PASSWORD=Chris1234 + healthcheck: + test: ["CMD", "pg_isready"] + interval: 2s + timeout: 4s + retries: 3 + start_period: 60s + networks: + local: + aliases: + - chris_dev_db # hard-coded in chris_backend/config/settings/local.py + rabbitmq: + image: docker.io/library/rabbitmq:3 + volumes: + - rabbitmq:/var/lib/rabbitmq + networks: + local: + aliases: + - queue # hard-coded in chris_backend/config/settings/local.py + + pfcon: + image: ${PFCON_IMAGE:-ghcr.io/fnndsc/pfcon:latest} + environment: + COMPUTE_SERVICE_URL: http://pman:5010/api/v1/ + SECRET_KEY: secret + PFCON_USER: pfcon + PFCON_PASSWORD: pfcon1234 + PFCON_INNETWORK: "true" + STORAGE_ENV: filesystem + STOREBASE_MOUNT: /var/local/storeBase + # port is hard-coded in chris_backend/config/settings/local.py + command: gunicorn -b 0.0.0.0:30005 -w 8 -t 120 pfcon.wsgi:application + ports: + - "30005:30005" + volumes: + - chris_files:/var/local/storeBase + user: ${UID}:${GID} + networks: + local: + aliases: + - pfcon.remote # hard-coded in chris_backend/config/settings/local.py + labels: + org.chrisproject.role: "pfcon" + + pman: + image: ${PMAN_IMAGE:-ghcr.io/fnndsc/pman:latest} + environment: + CONTAINER_ENV: docker + CONTAINER_USER: "${UID}:${GID}" + ENABLE_HOME_WORKAROUND: "yes" + JOB_LABELS: "org.chrisproject.miniChRIS=plugininstance" + SECRET_KEY: secret + REMOVE_JOBS: "yes" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + depends_on: + - pfcon + ports: + - "5010:5010" + networks: + - local + userns_mode: "host" + + lldap: + image: docker.io/nitnelave/lldap:stable + ports: + - "3890:3890" + - "17170:17170" + volumes: + - "lldap:/data" + environment: + TZ: America/New_York + LLDAP_JWT_SECRET: super_secret_random_string + LLDAP_LDAP_USER_PASS: chris1234 + LLDAP_LDAP_BASE_DN: dc=example,dc=org + networks: + local: + + cube-nonroot-user-volume-fix: + image: docker.io/library/alpine:latest + volumes: + - chris_files:/data + user: root + command: chmod 777 /data + restart: "no" + +volumes: + chris_files: + db: + lldap: + rabbitmq: + +networks: + local: diff --git a/justfile b/justfile new file mode 100644 index 00000000..987c636b --- /dev/null +++ b/justfile @@ -0,0 +1,81 @@ +compose_file := 'docker-compose_just.yml' + +# Start the ChRIS backend in development mode, and attach to the live-reloading server. +dev: chrisomatic attach + +# Start the ChRIS backend in development mode. +start: start-ancillary migrate up + +# Start services (without running database migrations). +up: (docker-compose '--profile=cube up -d') + +# Attach to the chris container. +attach: (docker-compose '--profile=cube attach chris') + +# Open a Python shell. +shell: (run 'python manage.py shell') + +# Open a Bash shell. +bash: (run 'bash') + +# Run chrisomatic, a tool which adds plugins and users to CUBE. +chrisomatic *args: start + @just docker-compose --profile=cube run --rm chrisomatic chrisomatic {{args}} + +# Run chrisomatic with the contents of chrisomatic/postscript.yml +postscript: (chrisomatic 'postscript.yml') + +# Perform database migrations. +migrate: (run 'python manage.py migrate --noinput') + +# Run tests, e.g. `just test pacsfiles` +test *args: + @just run python manage.py test --force-color {{args}} + +# Run all tests. +test-all: test-unit test-integration + +# Run unit tests. +test-unit: (run 'python manage.py test --force-color --exclude-tag integration') + +# Run integration tests. +test-integration: start-ancillary (run 'python manage.py test --force-color --tag integration') + +# Start dependency services. +start-ancillary: (docker-compose 'up -d') + +# Stop services. +down: (docker-compose '--profile=cube --profile=tools down') + +# Stop services and remove all data. +nuke: reap-plugin-instances (docker-compose '--profile=cube --profile=tools down -v --remove-orphans') + +# Remove all plugin instance containers. +reap-plugin-instances: ( + docker-compose 'run --rm pman python -c' ''' + ' + import os + import docker + d = docker.from_env() + filters = {"label": os.getenv("JOB_LABELS")} + containers = d.containers.list(all=True, filters=filters) + for container in containers: + print(f"Removing container: {container.name} ({container.image})", flush=True) + container.remove(force=True) + ' + ''' +) + +# (Re-)build the container image. +build: (docker-compose '--profile=cube build') + +# Pull container images. +pull: (docker-compose 'pull') + +# docker-compose ... run helper function. +run +command: + @just docker-compose --profile=cube run --rm chris {{command}} + +# docker-compose ... helper function. +docker-compose +command: + env UID=$(id -u) GID=$(id -g) docker compose -f '{{ compose_file }}' {{command}}