Merge pull request #309 from unicef/fix-compose-and-docs

fix compose and docs

Author: datamik

.gitignore

@@ -1,4 +1,5 @@
 .*
+.env
 ~*
 __pycache__
 !.dockerignore
@@ -15,6 +16,8 @@ __pycache__
 !tests/.coveragerc
 !docs/**/.pages
 
+var/
+
 junit.xml
 coverage.xml
 

compose.yml

@@ -1,62 +1,94 @@
+# Requires a .env file next to this compose.yml (gitignored):
+#   cp env.sample .env   # then fill in the values — a working development
+# configuration is in docs/src/development/getting-started.md.
+
 x-common: &common
   build:
     context: .
     dockerfile: docker/Dockerfile
     target: tests
   platform: linux/amd64
-  environment:
-    - ADMIN_EMAIL=adm@hde.org
-    - ADMIN_PASSWORD=123
-    - ALLOWED_HOSTS=localhost,127.0.0.1
-    - CACHE_URL=redis://redis:6379/1
-    - CELERY_BROKER_URL=redis://redis:6379/9
-    - CELERY_TASK_ALWAYS_EAGER=False
-    - CSRF_COOKIE_SECURE=False
-    - DATABASE_URL=postgres://hde:password@db:5432/hope_dedupe_engine
-    - DEFAULT_ROOT=/var/hope_dedupe_engine/default
-    - DJANGO_SETTINGS_MODULE=hope_dedup_engine.config.settings
-    - FILE_STORAGE_DEFAULT=django.core.files.storage.FileSystemStorage
-    - FILE_STORAGE_DNN=storages.backends.azure_storage.AzureStorage?azure_container=dnn&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;
-    - FILE_STORAGE_HOPE=storages.backends.azure_storage.AzureStorage?azure_container=hope&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;
-    - FILE_STORAGE_MEDIA=storages.backends.azure_storage.AzureStorage?azure_container=media&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;
-    - FILE_STORAGE_STATIC=storages.backends.azure_storage.AzureStorage?azure_container=static&overwrite_files=True&custom_domain=localhost:10000/&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;
-    - MEDIA_ROOT=/var/hope_dedupe_engine/media
-    - PYTHONPATH=/code/src/:/code/__pypackages__/3.12/lib/
-    - SECRET_KEY=very-secret-key
-    - SECURE_SSL_REDIRECT=False
-    - SESSION_COOKIE_DOMAIN=
-    - SESSION_COOKIE_SECURE=False
-    - SOCIAL_AUTH_REDIRECT_IS_HTTPS=False
-    - STATIC_ROOT=/var/hope_dedupe_engine/static
+  env_file:
+    - .env
+  extra_hosts:
+    - "host.docker.internal:host-gateway"
   volumes:
-    - .:/code
+    - .:/app
     - /var/run/docker.sock:/var/run/docker.sock
+    - ./weights:/var/run/app/deepface/.deepface/weights
+    - ./ofiq_omdels:/root/.ofiq/data
+    - ${IMAGES_HOST_PATH:-./var/data}:/var/data
   restart: always
   depends_on:
     db:
       condition: service_healthy
     redis:
       condition: service_healthy
+    azurite:
+      condition: service_started
 
 
 services:
   backend:
     <<: *common
     ports:
       - 8000:8000
+      - 5689:5680
     # command: ["tail", "-f", "/dev/null"]
     command: >
       /bin/sh -c "
       django-admin demo --skip-checks &&
       django-admin upgrade &&
       django-admin runserver 0.0.0.0:8000
       "
+    # Debug variant (debugpy on host port 5689):
+    # command: >
+    #   /bin/sh -c "
+    #   django-admin demo --skip-checks &&
+    #   django-admin upgrade --no-sync-models &&
+    #   python -Xfrozen_modules=off -u -m debugpy --listen 0.0.0.0:5680 manage.py runserver 0.0.0.0:8000
+    #   "
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8000/healthcheck"]
       interval: 10s
       timeout: 5s
       retries: 5
 
+  celery-worker:
+    <<: *common
+    command: >
+      sh -c '
+      mkdir -p /var/hope_dedupe_engine/default &&
+      export C_FORCE_ROOT=1 &&
+      celery -A hope_dedup_engine.config.celery worker -E --loglevel=WARNING --pool=prefork --concurrency=${CELERY_CONCURRENCY:-2}
+      '
+    # Debug variant:
+    # command: >
+    #   sh -c '
+    #   mkdir -p /var/hope_dedupe_engine/default &&
+    #   export C_FORCE_ROOT=1 &&
+    #   python -Xfrozen_modules=off -u -m debugpy --listen 0.0.0.0:5680 -m celery -A hope_dedup_engine.config.celery worker -E --loglevel=WARNING --concurrency=1
+    #   '
+
+  celery-beat:
+    <<: *common
+    entrypoint: ["sh", "-c", "exec docker-entrypoint.sh \"$0\" \"$@\""]
+    command: >
+      sh -c '
+      mkdir -p /var/hope_dedupe_engine/default &&
+      export C_FORCE_ROOT=1 &&
+      celery -A hope_dedup_engine.config.celery beat --loglevel=INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler
+      '
+
+  celery-flower:
+    <<: *common
+    ports:
+      - 5555:5555
+    command: >
+      sh -c "
+      exec celery -A hope_dedup_engine.config.celery flower --address=0.0.0.0
+      "
+
   db:
     image: postgres:16
     environment:
@@ -66,7 +98,7 @@ services:
     volumes:
       - postgres_data:/var/lib/postgresql/data/
     ports:
-      - 5432:5432
+      - ${DB_PORT:-5432}:5432
     restart: always
     healthcheck:
       test: ["CMD", "pg_isready", "-U", "hde", "-d", "hope_dedupe_engine"]
@@ -79,7 +111,7 @@ services:
   redis:
     image: redis:7.2
     ports:
-      - 6379:6379
+      - ${REDIS_PORT:-6379}:6379
     restart: always
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
@@ -94,35 +126,16 @@ services:
     command: "azurite -l /workspace -d /workspace/debug.log --blobPort 10000 --blobHost 0.0.0.0 --loose --silent --skipApiVersionCheck"
     restart: always
     ports:
-      - "10000:10000" # Blob service
+      - "10000:10000"
     volumes:
       - azurite_data:/workspace
 
-  celery-worker:
-    <<: *common
-    # entrypoint: ["sh", "-c", "exec docker-entrypoint.sh \"$0\" \"$@\""]
-    # command: worker
-    command: >
-      sh -c '
-      mkdir -p /var/hope_dedupe_engine/default &&
-      chown -R user:app /var/hope_dedupe_engine &&
-      gosu user:app django-admin syncdnn &&
-      gosu user:app celery -A hope_dedup_engine.config.celery worker -E --loglevel=WARNING --concurrency=4
-      '
-
-  celery-beat:
-    <<: *common
-    entrypoint: ["sh", "-c", "exec docker-entrypoint.sh \"$0\" \"$@\""]
-    command: beat
-
-  celery-flower:
+  docs:
     <<: *common
+    profiles: ["docs"]
     ports:
-      - 5555:5555
-    command: >
-      sh -c "
-      exec celery -A hope_dedup_engine.config.celery flower --address=0.0.0.0
-      "
+      - 8012:8012
+    command: ["uv", "run", "mkdocs", "serve", "-a", "0.0.0.0:8012", "--no-strict"]
 
 
 volumes:

docs/src/admin/environment.md

@@ -7,7 +7,10 @@ $ docker run -it -t unicef/hope-dedup-engine:<tag> django-admin env          # c
 $ docker run -it -t unicef/hope-dedup-engine:<tag> django-admin env --check  # verify required ones
 ```
 
-An auto-generated reference of every known variable is on the [Settings](../settings.md) page; below are the ones that matter operationally, grouped by purpose.
+An auto-generated reference of every known variable is on the [Settings](settings.md) page; below are the ones that matter operationally, grouped by purpose.
+
+!!! note "Local development"
+    When running via the repository's `compose.yml`, all of these variables are read from a gitignored `.env` file (template: `env.sample`) — see [Getting Started](../development/getting-started.md#configuration) for a working development configuration.
 
 ## Required
 

docs/src/development/demo.md

@@ -1,26 +1,28 @@
 # Demo App and API Scripts
 
-A self-contained demo environment with sample images and helper scripts lets you exercise the whole system — including the REST API — without configuring anything.
+A self-contained demo environment with sample images and helper scripts lets you exercise the whole system — including the REST API.
 
 ## Running the demo server
 
+Use the root `compose.yml` — on startup the backend seeds the demo data via the `demo` management command. It needs a `.env` file first (see [Getting Started](getting-started.md#configuration) for a working development configuration):
+
 ```console
-$ docker compose -f tests/extras/demoapp/compose.yml up --build
+$ cp env.sample .env    # fill in the values from Getting Started
+$ docker compose up --build
 ```
 
-(The root `compose.yml` also seeds the same demo data via the `demo` management command.)
+(To demo a published release instead of building locally, follow the [deployment guide](../admin/deployment.md) with the desired image tag — but note the API scripts below track the current code, so they may not work against older releases.)
 
 Admin panel access:
 
 - URL: [http://localhost:8000/admin](http://localhost:8000/admin)
-- Username: `adm@hde.org`
-- Password: `123`
+- Username / password: the `ADMIN_EMAIL` / `ADMIN_PASSWORD` values from your `.env`
 
 The demo setup creates an external system and an API token you can find under `Home › Api › Tokens`.
 
 ## API interaction scripts
 
-Shell scripts in `tests/extras/demoapp/scripts/` drive the API end to end. They require [httpie](https://httpie.io/) and [jq](https://jqlang.github.io/jq/).
+Shell scripts in `tests/extras/demoapp/scripts/` drive the API end to end. They require [httpie](https://httpie.io/), [jq](https://jqlang.github.io/jq/), and the `mimetype` command (Debian/Ubuntu package `libfile-mimeinfo-perl`).
 
 ### Configuration
 

docs/src/development/getting-started.md

@@ -11,18 +11,76 @@ Two ways to develop locally: **Docker Compose** (recommended — everything incl
 
 ## Option A: Docker Compose (recommended)
 
-The repository's `compose.yml` starts the full stack — backend, Celery worker, Celery beat, Flower, PostgreSQL, and Redis:
+The repository's `compose.yml` starts the full stack — backend, Celery worker, Celery beat, Flower, PostgreSQL, Redis, and Azurite (local Azure blob emulator). It reads all environment configuration from a `.env` file (gitignored) next to `compose.yml`:
 
 ```console
+$ cp env.sample .env    # then fill in the values (see below)
 $ docker compose up --build
 ```
 
-On startup the backend seeds demo data, applies migrations, and serves at [http://localhost:8000](http://localhost:8000). The admin panel is at `/admin/` (credentials from the compose environment: `adm@hde.org` / `123`), the API docs at `/api/rest/swagger/`.
+On startup the backend seeds demo data, applies migrations, and serves at [http://localhost:8000](http://localhost:8000). The admin panel is at `/admin/` (credentials `ADMIN_EMAIL` / `ADMIN_PASSWORD` from your `.env`), the API docs at `/api/rest/swagger/`.
+
+### Configuration
+
+`env.sample` lists every required key; `.env` is never committed, so real tokens and connection strings are safe there. This configuration works with the compose stack as-is (Azurite for blob storages, local filesystem for uploaded images):
+
+```bash
+# ── Django core ──────────────────────────────────────────────
+ADMIN_EMAIL=adm@hde.org
+ADMIN_PASSWORD=123
+ALLOWED_HOSTS=localhost,127.0.0.1
+SECRET_KEY=very-secret-key
+DJANGO_SETTINGS_MODULE=hope_dedup_engine.config.settings
+PYTHONPATH=/app/src
+
+# ── Database / cache / broker ────────────────────────────────
+DATABASE_URL=postgres://hde:password@db:5432/hope_dedupe_engine
+CACHE_URL=redis://redis:6379/1
+CELERY_BROKER_URL=redis://redis:6379/9
+CELERY_TASK_ALWAYS_EAGER=False
+
+# ── Security (local dev — all disabled) ──────────────────────
+CSRF_COOKIE_SECURE=False
+SECURE_SSL_REDIRECT=False
+SESSION_COOKIE_DOMAIN=
+SESSION_COOKIE_SECURE=False
+SOCIAL_AUTH_REDIRECT_IS_HTTPS=False
+
+# ── File storage backends ────────────────────────────────────
+FILE_STORAGE_DEFAULT=django.core.files.storage.FileSystemStorage
+FILE_STORAGE_IMAGES=django.core.files.storage.FileSystemStorage
+FILE_STORAGE_DNN=storages.backends.azure_storage.AzureStorage?azure_container=dnn&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;
+FILE_STORAGE_MEDIA=storages.backends.azure_storage.AzureStorage?azure_container=media&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;
+FILE_STORAGE_STATIC=storages.backends.azure_storage.AzureStorage?azure_container=static&overwrite_files=True&custom_domain=localhost:10000/&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;
+
+# ── Filesystem roots ─────────────────────────────────────────
+DEFAULT_ROOT=/var/hope_dedupe_engine/default
+IMAGES_ROOT=/var/data
+MEDIA_ROOT=/var/hope_dedupe_engine/media
+STATIC_ROOT=/var/hope_dedupe_engine/static
+
+# ── ML models ────────────────────────────────────────────────
+DEEPFACE_HOME=/var/run/app/deepface
+OFIQ_DATA_DIR=/root/.ofiq/data
+
+# ── Performance tuning ───────────────────────────────────────
+OMP_NUM_THREADS=2
+TF_NUM_INTRA_OP_THREADS=2
+TF_NUM_INTER_OP_THREADS=2
+
+# ── Integration ──────────────────────────────────────────────
+HOPE_API_TOKEN=
+```
+
+(The `AccountKey` above is [Azurite's public well-known development key](https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite#well-known-storage-account-and-key), not a secret.)
 
-Things to know:
+Compose-level knobs (volume paths and port bindings) can also be set in `.env`: `IMAGES_HOST_PATH` (host directory for uploaded images, default `./var/data`), `DB_PORT`, `REDIS_PORT`, `CELERY_CONCURRENCY`.
+
+### Things to know
 
 - The repo is bind-mounted into the containers, so code changes are picked up by `runserver` automatically. The Celery worker does **not** auto-reload — restart it after changing task code.
 - Model weight files are mounted from `./weights` (DeepFace) and `./ofiq_omdels` (OFIQ). See the comments in `compose.yml` for debugging variants of the service commands (debugpy).
+- Uploaded images are stored under `./var/data` on the host — the same directory is mounted into the backend and the workers, mirroring the shared storage mount used in production.
 - Run any management command inside the stack with `docker compose run --rm backend django-admin <command>`.
 
 ## Option B: Native virtualenv
@@ -73,6 +131,8 @@ $ uv sync --group docs
 $ uv run mkdocs serve   # http://127.0.0.1:8001
 ```
 
+Alternatively, serve them from the compose stack (port 8012): `docker compose --profile docs up docs`.
+
 ## Typical feature workflow
 
 1. Create a branch from `develop`.

env.sample

@@ -0,0 +1,58 @@
+# Template for the .env file used by compose.yml (gitignored).
+# Copy it (`cp env.sample .env`) and fill in the values — a complete working
+# development configuration is available in the documentation:
+# docs/src/development/getting-started.md ("Configuration").
+# See docs/src/admin/environment.md for the description of every variable.
+
+# ── Django core ──────────────────────────────────────────────
+ADMIN_EMAIL=
+ADMIN_PASSWORD=
+ALLOWED_HOSTS=
+SECRET_KEY=
+DJANGO_SETTINGS_MODULE=
+PYTHONPATH=
+
+# ── Database / cache / broker ────────────────────────────────
+DATABASE_URL=
+CACHE_URL=
+CELERY_BROKER_URL=
+CELERY_TASK_ALWAYS_EAGER=
+
+# ── Security ─────────────────────────────────────────────────
+CSRF_COOKIE_SECURE=
+SECURE_SSL_REDIRECT=
+SESSION_COOKIE_DOMAIN=
+SESSION_COOKIE_SECURE=
+SOCIAL_AUTH_REDIRECT_IS_HTTPS=
+
+# ── File storage backends ────────────────────────────────────
+FILE_STORAGE_DEFAULT=
+FILE_STORAGE_IMAGES=
+FILE_STORAGE_DNN=
+FILE_STORAGE_MEDIA=
+FILE_STORAGE_STATIC=
+
+# ── Filesystem roots ─────────────────────────────────────────
+DEFAULT_ROOT=
+IMAGES_ROOT=
+MEDIA_ROOT=
+STATIC_ROOT=
+
+# ── ML models ────────────────────────────────────────────────
+DEEPFACE_HOME=
+OFIQ_DATA_DIR=
+
+# ── Performance tuning ───────────────────────────────────────
+OMP_NUM_THREADS=
+TF_NUM_INTRA_OP_THREADS=
+TF_NUM_INTER_OP_THREADS=
+
+# ── Integration ──────────────────────────────────────────────
+HOPE_API_TOKEN=
+
+# ── Compose-level (volume paths, port bindings) ──────────────
+# Only used for ${VAR} interpolation in compose.yml, not passed to Django.
+# IMAGES_HOST_PATH=
+# DB_PORT=
+# REDIS_PORT=
+# CELERY_CONCURRENCY=