From 6af3701666e2691be1cfe164fd6746a02b8da340 Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Fri, 26 May 2023 13:34:25 +0200 Subject: [PATCH 01/43] Update openimis.json Added missing core-mis modules --- openimis.json | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/openimis.json b/openimis.json index 8c01098e..64d9277a 100644 --- a/openimis.json +++ b/openimis.json @@ -4,6 +4,14 @@ "name": "core", "pip": "git+https://github.com/openimis/openimis-be-core_py.git@develop#egg=openimis-be-core" }, + { + "name": "individual", + "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@develop#egg=openimis-be-individual" + }, + { + "name": "workflow", + "pip": "git+https://github.com/openimis/openimis-be-workflow_py.git@develop#egg=openimis-be-workflow" + }, { "name": "report", "pip": "git+https://github.com/openimis/openimis-be-report_py.git@develop#egg=openimis-be-report" @@ -115,6 +123,10 @@ { "name":"dhis2_etl", "pip":"git+https://github.com/openimis/openimis-be-dhis2_etl_py.git@develop#egg=openimis-be-dhis2_etl" - } + }, + { + "name": "social_protection", + "pip": "git+https://github.com/openimis/openimis-be-social_protection_py.git@develop#egg=openimis-be-social_protection" + }, ] } From 5935c04e8496d5902805cb8da17f5672684abcf4 Mon Sep 17 00:00:00 2001 From: wzglinieckisoldevelo <98958634+wzglinieckisoldevelo@users.noreply.github.com> Date: Mon, 29 May 2023 09:32:29 +0200 Subject: [PATCH 02/43] Update openimis.json (#142) Fix - removed redundant comma to fix CI. --- openimis.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openimis.json b/openimis.json index 64d9277a..b1efe91b 100644 --- a/openimis.json +++ b/openimis.json @@ -127,6 +127,6 @@ { "name": "social_protection", "pip": "git+https://github.com/openimis/openimis-be-social_protection_py.git@develop#egg=openimis-be-social_protection" - }, + } ] } From 557d6047ad7579cd68cfdb7b07f6412980bb3030 Mon Sep 17 00:00:00 2001 From: sniedzielski Date: Mon, 29 May 2023 16:51:02 +0200 Subject: [PATCH 03/43] CM-97: fixed exception handler to process 401 Unathorized --- openIMIS/openIMIS/ExceptionHandlerDispatcher.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/openIMIS/openIMIS/ExceptionHandlerDispatcher.py b/openIMIS/openIMIS/ExceptionHandlerDispatcher.py index 4dde35df..fabad6e9 100644 --- a/openIMIS/openIMIS/ExceptionHandlerDispatcher.py +++ b/openIMIS/openIMIS/ExceptionHandlerDispatcher.py @@ -1,7 +1,6 @@ from .ExceptionHandlerRegistry import ExceptionHandlerRegistry from rest_framework.views import exception_handler -from rest_framework import exceptions, status - +from rest_framework import exceptions, status, views def dispatcher(exc, context): """ @@ -10,6 +9,10 @@ def dispatcher(exc, context): module_name = _extract_module_name(context['request']) handler = ExceptionHandlerRegistry.get_exception_handler(module_name) + response = _process_exception_handler(exc, context) + if response: + return response + if handler is None: # Fallback to default DRF exception handler if no handler is defined for the module handler = exception_handler @@ -21,9 +24,16 @@ def dispatcher(exc, context): return response +def _process_exception_handler(exc, context): + response = views.exception_handler(exc, context) + + if isinstance(exc, (exceptions.AuthenticationFailed, exceptions.NotAuthenticated)): + response.status_code = status.HTTP_401_UNAUTHORIZED + return response + + def _extract_module_name(request): """ Extracts the module name from the request URL. """ return request.path.split('/')[2] - From c850615f50d91ace6c1012a0e96f0de847da5beb Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 21 Jun 2023 14:07:47 +0200 Subject: [PATCH 04/43] CM-146: add tasks_management module (#155) * CM-147: update openimis.json * CM-146: move tasks_management --- openimis.json | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/openimis.json b/openimis.json index b1efe91b..62364ef0 100644 --- a/openimis.json +++ b/openimis.json @@ -5,13 +5,17 @@ "pip": "git+https://github.com/openimis/openimis-be-core_py.git@develop#egg=openimis-be-core" }, { - "name": "individual", - "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@develop#egg=openimis-be-individual" + "name": "individual", + "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@develop#egg=openimis-be-individual" }, { - "name": "workflow", - "pip": "git+https://github.com/openimis/openimis-be-workflow_py.git@develop#egg=openimis-be-workflow" + "name": "workflow", + "pip": "git+https://github.com/openimis/openimis-be-workflow_py.git@develop#egg=openimis-be-workflow" }, + { + "name": "tasks_management", + "pip": "git+https://github.com/openimis/openimis-be-tasks_management_py.git@develop#egg=openimis-be-tasks_management" + }, { "name": "report", "pip": "git+https://github.com/openimis/openimis-be-report_py.git@develop#egg=openimis-be-report" @@ -125,8 +129,8 @@ "pip":"git+https://github.com/openimis/openimis-be-dhis2_etl_py.git@develop#egg=openimis-be-dhis2_etl" }, { - "name": "social_protection", - "pip": "git+https://github.com/openimis/openimis-be-social_protection_py.git@develop#egg=openimis-be-social_protection" + "name": "social_protection", + "pip": "git+https://github.com/openimis/openimis-be-social_protection_py.git@develop#egg=openimis-be-social_protection" } ] } From ba5d92027b291272699d21e54e5b6379a8a1512b Mon Sep 17 00:00:00 2001 From: sniedzielski <52816247+sniedzielski@users.noreply.github.com> Date: Sun, 25 Jun 2023 17:50:49 +0200 Subject: [PATCH 05/43] CM-152: added backend module for reporting in openSearch to coreMIS (#157) --- openimis.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openimis.json b/openimis.json index 62364ef0..f7adbad8 100644 --- a/openimis.json +++ b/openimis.json @@ -131,6 +131,10 @@ { "name": "social_protection", "pip": "git+https://github.com/openimis/openimis-be-social_protection_py.git@develop#egg=openimis-be-social_protection" + }, + { + "name": "opensearch_reports", + "pip": "git+https://github.com/openimis/openimis-be-opensearch_reports_py.git@develop#egg=openimis-be-opensearch_reports" } ] } From c7b52c7942292904d9e39ab6766c2ed1cbd3f697 Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Fri, 26 May 2023 13:34:25 +0200 Subject: [PATCH 06/43] Update openimis.json Added missing core-mis modules --- openimis.json | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/openimis.json b/openimis.json index 8c01098e..64d9277a 100644 --- a/openimis.json +++ b/openimis.json @@ -4,6 +4,14 @@ "name": "core", "pip": "git+https://github.com/openimis/openimis-be-core_py.git@develop#egg=openimis-be-core" }, + { + "name": "individual", + "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@develop#egg=openimis-be-individual" + }, + { + "name": "workflow", + "pip": "git+https://github.com/openimis/openimis-be-workflow_py.git@develop#egg=openimis-be-workflow" + }, { "name": "report", "pip": "git+https://github.com/openimis/openimis-be-report_py.git@develop#egg=openimis-be-report" @@ -115,6 +123,10 @@ { "name":"dhis2_etl", "pip":"git+https://github.com/openimis/openimis-be-dhis2_etl_py.git@develop#egg=openimis-be-dhis2_etl" - } + }, + { + "name": "social_protection", + "pip": "git+https://github.com/openimis/openimis-be-social_protection_py.git@develop#egg=openimis-be-social_protection" + }, ] } From 597d143b1c362faa980c018ec41dcbcf4a24ddf5 Mon Sep 17 00:00:00 2001 From: wzglinieckisoldevelo <98958634+wzglinieckisoldevelo@users.noreply.github.com> Date: Mon, 29 May 2023 09:32:29 +0200 Subject: [PATCH 07/43] Update openimis.json (#142) Fix - removed redundant comma to fix CI. --- openimis.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openimis.json b/openimis.json index 64d9277a..b1efe91b 100644 --- a/openimis.json +++ b/openimis.json @@ -127,6 +127,6 @@ { "name": "social_protection", "pip": "git+https://github.com/openimis/openimis-be-social_protection_py.git@develop#egg=openimis-be-social_protection" - }, + } ] } From 5b78168cc4549d74e154401a842d219ae37402fb Mon Sep 17 00:00:00 2001 From: sniedzielski Date: Mon, 29 May 2023 16:51:02 +0200 Subject: [PATCH 08/43] CM-97: fixed exception handler to process 401 Unathorized --- openIMIS/openIMIS/ExceptionHandlerDispatcher.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/openIMIS/openIMIS/ExceptionHandlerDispatcher.py b/openIMIS/openIMIS/ExceptionHandlerDispatcher.py index 4dde35df..fabad6e9 100644 --- a/openIMIS/openIMIS/ExceptionHandlerDispatcher.py +++ b/openIMIS/openIMIS/ExceptionHandlerDispatcher.py @@ -1,7 +1,6 @@ from .ExceptionHandlerRegistry import ExceptionHandlerRegistry from rest_framework.views import exception_handler -from rest_framework import exceptions, status - +from rest_framework import exceptions, status, views def dispatcher(exc, context): """ @@ -10,6 +9,10 @@ def dispatcher(exc, context): module_name = _extract_module_name(context['request']) handler = ExceptionHandlerRegistry.get_exception_handler(module_name) + response = _process_exception_handler(exc, context) + if response: + return response + if handler is None: # Fallback to default DRF exception handler if no handler is defined for the module handler = exception_handler @@ -21,9 +24,16 @@ def dispatcher(exc, context): return response +def _process_exception_handler(exc, context): + response = views.exception_handler(exc, context) + + if isinstance(exc, (exceptions.AuthenticationFailed, exceptions.NotAuthenticated)): + response.status_code = status.HTTP_401_UNAUTHORIZED + return response + + def _extract_module_name(request): """ Extracts the module name from the request URL. """ return request.path.split('/')[2] - From 374df58173069289a70eb3edaeab62e1fda5d7f0 Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 21 Jun 2023 14:07:47 +0200 Subject: [PATCH 09/43] CM-146: add tasks_management module (#155) * CM-147: update openimis.json * CM-146: move tasks_management --- openimis.json | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/openimis.json b/openimis.json index b1efe91b..62364ef0 100644 --- a/openimis.json +++ b/openimis.json @@ -5,13 +5,17 @@ "pip": "git+https://github.com/openimis/openimis-be-core_py.git@develop#egg=openimis-be-core" }, { - "name": "individual", - "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@develop#egg=openimis-be-individual" + "name": "individual", + "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@develop#egg=openimis-be-individual" }, { - "name": "workflow", - "pip": "git+https://github.com/openimis/openimis-be-workflow_py.git@develop#egg=openimis-be-workflow" + "name": "workflow", + "pip": "git+https://github.com/openimis/openimis-be-workflow_py.git@develop#egg=openimis-be-workflow" }, + { + "name": "tasks_management", + "pip": "git+https://github.com/openimis/openimis-be-tasks_management_py.git@develop#egg=openimis-be-tasks_management" + }, { "name": "report", "pip": "git+https://github.com/openimis/openimis-be-report_py.git@develop#egg=openimis-be-report" @@ -125,8 +129,8 @@ "pip":"git+https://github.com/openimis/openimis-be-dhis2_etl_py.git@develop#egg=openimis-be-dhis2_etl" }, { - "name": "social_protection", - "pip": "git+https://github.com/openimis/openimis-be-social_protection_py.git@develop#egg=openimis-be-social_protection" + "name": "social_protection", + "pip": "git+https://github.com/openimis/openimis-be-social_protection_py.git@develop#egg=openimis-be-social_protection" } ] } From faa2a188e99f49fbbafba1380b7f0ce9b059e1b2 Mon Sep 17 00:00:00 2001 From: sniedzielski <52816247+sniedzielski@users.noreply.github.com> Date: Sun, 25 Jun 2023 17:50:49 +0200 Subject: [PATCH 10/43] CM-152: added backend module for reporting in openSearch to coreMIS (#157) --- openimis.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openimis.json b/openimis.json index 62364ef0..f7adbad8 100644 --- a/openimis.json +++ b/openimis.json @@ -131,6 +131,10 @@ { "name": "social_protection", "pip": "git+https://github.com/openimis/openimis-be-social_protection_py.git@develop#egg=openimis-be-social_protection" + }, + { + "name": "opensearch_reports", + "pip": "git+https://github.com/openimis/openimis-be-opensearch_reports_py.git@develop#egg=openimis-be-opensearch_reports" } ] } From 36861a9f12509b03ec92797633ecff7b3693990e Mon Sep 17 00:00:00 2001 From: Kamil Malinowski Date: Tue, 25 Jul 2023 09:51:37 +0200 Subject: [PATCH 11/43] Added payment_cycle and calcrule_social_protection to openimis.josn (#164) --- openimis.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openimis.json b/openimis.json index f7adbad8..a9c5196f 100644 --- a/openimis.json +++ b/openimis.json @@ -135,6 +135,14 @@ { "name": "opensearch_reports", "pip": "git+https://github.com/openimis/openimis-be-opensearch_reports_py.git@develop#egg=openimis-be-opensearch_reports" + }, + { + "name": "payment_cycle", + "pip": "git+https://github.com/openimis/openimis-be-payment_cycle_py.git@develop#egg=openimis-be-payment_cycle" + }, + { + "name": "calcrule_social_protection", + "pip": "git+https://github.com/openimis/openimis-be-calcrule_social_protection_py.git@develop#egg=openimis-be-calcrule_social_protection" } ] } From 8a03a821f23aa2be8793cd213cd8e666bd52fa2f Mon Sep 17 00:00:00 2001 From: sniedzielski <52816247+sniedzielski@users.noreply.github.com> Date: Mon, 7 Aug 2023 13:35:36 +0200 Subject: [PATCH 12/43] added backend 'payroll' module (#165) --- openimis.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openimis.json b/openimis.json index a9c5196f..adf806b0 100644 --- a/openimis.json +++ b/openimis.json @@ -143,6 +143,10 @@ { "name": "calcrule_social_protection", "pip": "git+https://github.com/openimis/openimis-be-calcrule_social_protection_py.git@develop#egg=openimis-be-calcrule_social_protection" + }, + { + "name": "payroll", + "pip": "git+https://github.com/openimis/openimis-be-payroll_py.git@develop#egg=openimis-be-payroll" } ] } From c01cb6b0389366d0e2fc6dcd2886181eefc71e59 Mon Sep 17 00:00:00 2001 From: Kamil Malinowski Date: Thu, 17 Aug 2023 11:52:49 +0200 Subject: [PATCH 13/43] Merge develop into coreMIS (#169) * ONI-62: Add Nepali calendar dependency. (#163) * Activate policy renewal task (#159) * Rename pgsql to pgsql.yml (#136) * OP-1512 Specified version of apscheduler (#167) --------- Co-authored-by: wzglinieckisoldevelo <98958634+wzglinieckisoldevelo@users.noreply.github.com> Co-authored-by: Dragos DOBRE Co-authored-by: Patrick Delcroix --- .github/workflows/pgsql | 66 ------------------------- .github/workflows/pgsql.yml | 92 +++++++++++++++++++++++++++++++++++ openIMIS/openIMIS/settings.py | 12 ++--- requirements.txt | 4 +- 4 files changed, 101 insertions(+), 73 deletions(-) delete mode 100644 .github/workflows/pgsql create mode 100644 .github/workflows/pgsql.yml diff --git a/.github/workflows/pgsql b/.github/workflows/pgsql deleted file mode 100644 index 99eeb7f8..00000000 --- a/.github/workflows/pgsql +++ /dev/null @@ -1,66 +0,0 @@ -name: Automated CI testing -# This workflow run automatically for every commit on github it checks the syntax and launch the tests. -# | grep . | uniq -c filters out empty lines and then groups consecutive lines together with the number of occurrences -on: - push: - workflow_dispatch: - inputs: - comment: - description: Just a simple comment to know the purpose of the manual build - required: false - -jobs: - build: - runs-on: ubuntu-20.04 - services: - pgsql: - build: - context: https://github.com/openimis/database_postgresql.git#${DB_BRANCH:-develop} - args: - - POSTGRES_PASSWORD=${DB_PASSWORD} - env: - DB_HOST: localhost - DB_PORT: 1433 - POSTGRES_DB: imis - POSTGRES_USER: sa - POSTGRES_PASSWORD: GitHub999 - ports: - - 5432:5432 - # needed because the mssql container does not provide a health check - options: --health-interval=10s --health-timeout=3s --health-start-period=10s --health-retries=10 --health-cmd="pg_isready -U ${POSTGRES_USER} -d ${imis}" - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - python modules-requirements.py openimis.json > modules-requirements.txt - pip install -r modules-requirements.txt - - - name: Django tests - run: | - python -V - ls -l - cd openIMIS - ls -l - python manage.py test --keepdb $(jq -r '(.modules[]|.name)' ../openimis.json) - env: - SECRET_KEY: secret - DEBUG: true - DB_ENGINE: django.db.backends.postgresql - #DJANGO_SETTINGS_MODULE: hat.settings - DB_HOST: localhost - DB_PORT: 5432 - DB_NAME: imis - DB_USER: sa - DB_PASSWORD: GitHub999 - #DEV_SERVER: true - SITE_ROOT: api - REMOTE_USER_AUTHENTICATION: False diff --git a/.github/workflows/pgsql.yml b/.github/workflows/pgsql.yml new file mode 100644 index 00000000..1ca06a11 --- /dev/null +++ b/.github/workflows/pgsql.yml @@ -0,0 +1,92 @@ +name: Automated CI PSQL testing +# This workflow run automatically for every commit on github it checks the syntax and launch the tests. +# | grep . | uniq -c filters out empty lines and then groups consecutive lines together with the number of occurrences +on: + push: + workflow_dispatch: + inputs: + comment: + description: Just a simple comment to know the purpose of the manual build + required: false + +jobs: + build: + runs-on: ubuntu-20.04 + services: + pgsql: + image: postgres + env: + DB_HOST: localhost + DB_PORT: 5432 + POSTGRES_DB: imis + POSTGRES_USER: postgres + POSTGRES_PASSWORD: GitHub999 + ports: + - 5432:5432 + # needed because the mssql container does not provide a health check + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + python modules-requirements.py openimis.json > modules-requirements.txt + pip install -r modules-requirements.txt + + + - name: Environment info + run: | + pip list + + export DBBRANCH="$([ $GITHUB_REF == 'main' ] && echo "main" || echo "develop")" + + if [ ${GITHUB_REF##*/} = "develop" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi + echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database" + + git clone --depth 1 --branch $DBBRANCH https://github.com/openimis/database_postgresql.git ./sql + + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/postgresql-pgdg.list > /dev/null + sudo apt-get install postgresql-client + + - name: Initialize DB + run: | + echo 'set search_path to public' >> ~/.psqlrc + PGPASSWORD=GitHub999 psql -U postgres -h localhost -d imis -U postgres -f ./sql/database\ scripts/00_dump.sql | grep . | uniq -c + PGPASSWORD=GitHub999 psql -U postgres -h localhost -d imis -U postgres -f ./sql/database\ scripts/02_aux_functions.sql | grep . | uniq -c + PGPASSWORD=GitHub999 psql -U postgres -h localhost -d imis -U postgres -f ./sql/database\ scripts/03_views.sql | grep . | uniq -c + PGPASSWORD=GitHub999 psql -U postgres -h localhost -d imis -U postgres -f ./sql/database\ scripts/04_functions.sql | grep . | uniq -c + PGPASSWORD=GitHub999 psql -U postgres -h localhost -d imis -U postgres -f ./sql/database\ scripts/05_stored_procs.sql | grep . | uniq -c + PGPASSWORD=GitHub999 psql -U postgres -h localhost -d imis -U postgres -f ./sql/database\ scripts/demo_db.sql | grep . | uniq -c + + - name: Django tests + run: | + python -V + ls -l + cd openIMIS + ls -l + python manage.py test --keepdb $(jq -r '(.modules[]|.name)' ../openimis.json) + env: + SECRET_KEY: secret + DEBUG: true + DB_ENGINE: django.db.backends.postgresql + #DJANGO_SETTINGS_MODULE: hat.settings + DB_HOST: localhost + DB_PORT: 5432 + DB_NAME: imis + DB_USER: postgres + DB_PASSWORD: GitHub999 + #DEV_SERVER: true + SITE_ROOT: api diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index 24b5c3b8..a976e419 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -304,7 +304,7 @@ def SITE_URL(): "unicode_results": True, } else: - DATABASE_OPTIONS = {} + DATABASE_OPTIONS = {'options': '-c search_path=django,public'} if not os.environ.get("NO_DATABASE_ENGINE", "False") == "True": DATABASES = { @@ -344,11 +344,11 @@ def SITE_URL(): "args": ["cron"], "kwargs": {"id": "openimis_test_batch", "minute": 16, "replace_existing": True}, }, - # { - # "method": "policy.tasks.get_policies_for_renewal", - # "args": ["cron"], - # "kwargs": {"id": "openimis_renewal_batch", "hour": 8, "minute": 30, "replace_existing": True}, - # }, + { + "method": "policy.tasks.get_policies_for_renewal", + "args": ["cron"], + "kwargs": {"id": "openimis_renewal_batch", "hour": 8, "minute": 30, "replace_existing": True}, + }, # { # "method": "policy_notification.tasks.send_notification_messages", # "args": ["cron"], diff --git a/requirements.txt b/requirements.txt index 589d6599..acd5bdc0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ wheel whitenoise django-health-check requests~=2.28.1 -apscheduler +apscheduler==3.10.1 # As from v0.4, Django-apscheduler has a migration that is incompatible with SQL Server # (autoincrement int => bigint) so we are using our own fork with a squashed migration git+https://github.com/openimis/django-apscheduler.git#egg=django-apscheduler @@ -45,4 +45,6 @@ drf-spectacular==0.25.1 django-cprofile-middleware==1.0.5 # flake8 and django-nose are needed for sonar scan in backend modules flake8 +# TODO remove to a separate file for implementations and change test workflows to reflect that +nepali_datetime From c67e91f5d3f747251f1ff88f8821dfbf055e0513 Mon Sep 17 00:00:00 2001 From: sniedzielski <52816247+sniedzielski@users.noreply.github.com> Date: Mon, 2 Oct 2023 14:35:28 +0200 Subject: [PATCH 14/43] CM-269: Added opensearch dsl library (#181) * CM-269: added for coreMIS opensearch dsl library and configuration * CM-269: changed opensearch DSL env * CM-269: fixed typo --- openIMIS/openIMIS/settings.py | 14 +++++++++++++- requirements.txt | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index a976e419..31d0fb1f 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -159,7 +159,8 @@ def SITE_URL(): "django_apscheduler", "channels", # Websocket support "developer_tools", - "drf_spectacular" # Swagger UI for FHIR API + "drf_spectacular", # Swagger UI for FHIR API + "django_opensearch_dsl" ] INSTALLED_APPS += OPENIMIS_APPS INSTALLED_APPS += ["apscheduler_runner", "signal_binding"] # Signal binding should be last installed module @@ -472,3 +473,14 @@ def SITE_URL(): USE_X_FORWARDED_HOST = True SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') + +OPENSEARCH_DSL = { + 'default': { + 'hosts': os.environ.get("OPENSEARCH_HOST", '0.0.0.0:9200'), + 'http_auth': ( + os.environ.get("OPENSEARCH_ADMIN"), + os.environ.get("OPENSEARCH_PASSWORD") + ), + 'timeout': 120, + } +} diff --git a/requirements.txt b/requirements.txt index acd5bdc0..ef6d6f60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,6 +43,7 @@ daphne==3.0.1 GitPython~=3.1.27 drf-spectacular==0.25.1 django-cprofile-middleware==1.0.5 +django-opensearch-dsl==0.5.1 # flake8 and django-nose are needed for sonar scan in backend modules flake8 # TODO remove to a separate file for implementations and change test workflows to reflect that From ad981c432c87bf3123f8b1779cab4b9ddc0594d5 Mon Sep 17 00:00:00 2001 From: sniedzielski <52816247+sniedzielski@users.noreply.github.com> Date: Mon, 2 Oct 2023 16:36:56 +0200 Subject: [PATCH 15/43] CM-269: added information to README about openSearch (#182) * CM-269: added for coreMIS opensearch dsl library and configuration * CM-269: changed opensearch DSL env * CM-269: fixed typo * CM-269: added openSearch information how to deploy and configure * CM-269: fixed readme section regarding opensearch * CM-269: fixed link to opensearch module * CM-269: fixed more details about opensearch section * CM-269: fixed more details about opensearch section * CM-269: fixed more details about opensearch section --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 7ca53b22..1e9176e1 100644 --- a/README.md +++ b/README.md @@ -328,3 +328,22 @@ You can find more informations about seeting up db docker [here](https://github. ### How to report another issues? If you face another issues not described in that section you could use our [ticketing site](https://openimis.atlassian.net/servicedesk/customer/portal/1). Here you can report any bugs/problems you faced during setting up openIMIS app. + + +## OpenSearch + +### OpenSearch - Adding Environmental Variables to Your Build +To configure environmental variables for your build, include the following: +* `OPENSEARCH_HOST` - For the non-dockerized instance in a local context, set it to 0.0.0.0:9200. +For the dockerized instance, use opensearch:9200. +* `OPENSEARCH_ADMIN` This variable is used for the admin username. (default value: admin) +* `OPENSEARCH_PASSWORD` This variable is used for the admin password. (default value: admin) + +### OpenSearch - How to initialize data after deployment +* If you have initialized the application but still have some data to be transferred, you can effortlessly +achieve this by using the commands available in the business module: `python manage.py add__data_to_opensearch`. +This command loads existing data into OpenSearch. + +### OpenSearch - more details +* For more comprehensive details on OpenSearch configuration, please consult the [resource](https://github.com/openimis/openimis-be-opensearch_reports_py/tree/develop) +provided in the README section. From 55e0c7ccade2800155f291a126f3defb9f6dd222 Mon Sep 17 00:00:00 2001 From: sniedzielski <52816247+sniedzielski@users.noreply.github.com> Date: Mon, 2 Oct 2023 19:12:14 +0200 Subject: [PATCH 16/43] CM-269: fixing openSearch auth settings (#183) * CM-269: added for coreMIS opensearch dsl library and configuration * CM-269: changed opensearch DSL env * CM-269: fixed typo * CM-269: added openSearch information how to deploy and configure * CM-269: fixed readme section regarding opensearch * CM-269: fixed link to opensearch module * CM-269: fixed more details about opensearch section * CM-269: fixed more details about opensearch section * CM-269: fixed more details about opensearch section * CM-269: fixed http_auth --- openIMIS/openIMIS/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index 31d0fb1f..a4136bab 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -478,8 +478,8 @@ def SITE_URL(): 'default': { 'hosts': os.environ.get("OPENSEARCH_HOST", '0.0.0.0:9200'), 'http_auth': ( - os.environ.get("OPENSEARCH_ADMIN"), - os.environ.get("OPENSEARCH_PASSWORD") + f"{os.environ.get('OPENSEARCH_ADMIN')}", + f"{os.environ.get('OPENSEARCH_PASSWORD')}" ), 'timeout': 120, } From e22ac9afee816a68588307d077b88ddedd5b23ad Mon Sep 17 00:00:00 2001 From: sniedzielski <52816247+sniedzielski@users.noreply.github.com> Date: Thu, 11 Jan 2024 11:07:24 +0100 Subject: [PATCH 17/43] CM-446: added deduplication to openIMIS module list in json (#196) --- openimis.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openimis.json b/openimis.json index adf806b0..9e0a6082 100644 --- a/openimis.json +++ b/openimis.json @@ -147,6 +147,10 @@ { "name": "payroll", "pip": "git+https://github.com/openimis/openimis-be-payroll_py.git@develop#egg=openimis-be-payroll" + }, + { + "name": "deduplication", + "pip": "git+https://github.com/openimis/openimis-be-deduplication_py.git@develop#egg=openimis-be-deduplication" } ] } From 049821ff7f87213fbf3b671f0850fcbb220e85a5 Mon Sep 17 00:00:00 2001 From: sniedzielski <52816247+sniedzielski@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:23:07 +0100 Subject: [PATCH 18/43] added validation rule module (#197) --- openimis.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openimis.json b/openimis.json index 9e0a6082..bfeceda1 100644 --- a/openimis.json +++ b/openimis.json @@ -151,6 +151,10 @@ { "name": "deduplication", "pip": "git+https://github.com/openimis/openimis-be-deduplication_py.git@develop#egg=openimis-be-deduplication" - } + }, + { + "name": "calcrule_validations", + "pip": "git+https://github.com/openimis/openimis-be-calcrule_validations_py.git@develop#egg=openimis-be-calcrule_validations" + } ] } From e2eb9617bd281e6ef74c1c6bc721d262b01e8410 Mon Sep 17 00:00:00 2001 From: Kamil Malinowski Date: Thu, 25 Jan 2024 15:48:10 +0100 Subject: [PATCH 19/43] enabled individual import demo --- openimis.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openimis.json b/openimis.json index bfeceda1..2dbac834 100644 --- a/openimis.json +++ b/openimis.json @@ -6,7 +6,7 @@ }, { "name": "individual", - "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@develop#egg=openimis-be-individual" + "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@feature/individual_upload_prototype#egg=openimis-be-individual" }, { "name": "workflow", From acea8e803abc07ba69ea8084e9f2064ce1b7be1b Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 30 Jan 2024 11:03:03 +0100 Subject: [PATCH 20/43] change-individual-branch: change to develop (#200) Co-authored-by: Jan --- openimis.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openimis.json b/openimis.json index 2dbac834..bfeceda1 100644 --- a/openimis.json +++ b/openimis.json @@ -6,7 +6,7 @@ }, { "name": "individual", - "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@feature/individual_upload_prototype#egg=openimis-be-individual" + "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@develop#egg=openimis-be-individual" }, { "name": "workflow", From 0fff41de13aee0d2bedbc18a40e6fcac5f2b3269 Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 30 Jan 2024 11:46:42 +0100 Subject: [PATCH 21/43] Change individual branch (#201) * change-individual-branch: change to develop * Revert "change-individual-branch: change to develop" This reverts commit 85769ed08ad60de75aa87f0ca1341f32891282a5. --------- Co-authored-by: Jan From 1147d76540de4aaab42c61e6c1e783a5d3dfac2b Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 30 Jan 2024 12:15:59 +0100 Subject: [PATCH 22/43] Change individual branch (#202) * change-individual-branch: change to develop * Revert "change-individual-branch: change to develop" This reverts commit 85769ed08ad60de75aa87f0ca1341f32891282a5. --------- Co-authored-by: Jan From c288a37e220af25396664d0a1a1ea581198c5d7a Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 30 Jan 2024 12:18:27 +0100 Subject: [PATCH 23/43] change branch name (#203) Co-authored-by: Jan --- openimis.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openimis.json b/openimis.json index bfeceda1..2dbac834 100644 --- a/openimis.json +++ b/openimis.json @@ -6,7 +6,7 @@ }, { "name": "individual", - "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@develop#egg=openimis-be-individual" + "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@feature/individual_upload_prototype#egg=openimis-be-individual" }, { "name": "workflow", From fe247e8d713382519d88fb9b9079a4dd5f50d503 Mon Sep 17 00:00:00 2001 From: Kamil Malinowski Date: Fri, 9 Feb 2024 09:20:49 +0100 Subject: [PATCH 24/43] OM-503 Added global storage --- .gitignore | 1 + openIMIS/openIMIS/settings.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/.gitignore b/.gitignore index ffb4c264..f8446a70 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ modules-tests.sh openIMIS/locale/en/LC_MESSAGES/django.mo **/staticfiles extracted_translations_fe +openIMIS/file_storage diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index a4136bab..105118b3 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -484,3 +484,16 @@ def SITE_URL(): 'timeout': 120, } } + +MEDIA_URL = "/file_storage/" +MEDIA_ROOT = os.path.join(BASE_DIR, "file_storage/") + +STORAGES = { + "default": { + "class": "core.filesystem.storage.FileSystemStorage", + "options": { + "location": MEDIA_URL, + "base_url": MEDIA_URL + }, + } +} From ba13c1c9b816e957b7a5cd043b4a6ba0e4c0c4cb Mon Sep 17 00:00:00 2001 From: sniedzielski Date: Mon, 19 Feb 2024 14:00:09 +0100 Subject: [PATCH 25/43] changed branch individual to develop --- openimis.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openimis.json b/openimis.json index 2dbac834..bfeceda1 100644 --- a/openimis.json +++ b/openimis.json @@ -6,7 +6,7 @@ }, { "name": "individual", - "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@feature/individual_upload_prototype#egg=openimis-be-individual" + "pip": "git+https://github.com/openimis/openimis-be-individual_py.git@develop#egg=openimis-be-individual" }, { "name": "workflow", From a230e09b819d6eca2b6fa3bebb2c536f3dd19985 Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 27 Feb 2024 09:34:54 +0100 Subject: [PATCH 26/43] hotfix: create MEDIAR_ROOT directory (#210) Co-authored-by: Jan --- openIMIS/openIMIS/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index 105118b3..33fae3e4 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -488,6 +488,9 @@ def SITE_URL(): MEDIA_URL = "/file_storage/" MEDIA_ROOT = os.path.join(BASE_DIR, "file_storage/") +if not os.path.exists(MEDIA_ROOT): + os.makedirs(MEDIA_ROOT) + STORAGES = { "default": { "class": "core.filesystem.storage.FileSystemStorage", From 2638b9e32f1d8df1e02010031686b0d549de238c Mon Sep 17 00:00:00 2001 From: Jan Date: Fri, 29 Mar 2024 09:07:06 +0100 Subject: [PATCH 27/43] CM-855: add grievance_social_protection (#219) Co-authored-by: Jan --- openimis.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openimis.json b/openimis.json index bfeceda1..7160fb9a 100644 --- a/openimis.json +++ b/openimis.json @@ -155,6 +155,10 @@ { "name": "calcrule_validations", "pip": "git+https://github.com/openimis/openimis-be-calcrule_validations_py.git@develop#egg=openimis-be-calcrule_validations" + }, + { + "name": "grievance_social_protection", + "pip": "git+https://github.com/openimis/openimis-be-grievance_social_protection_py.git@develop#egg=openimis-be-grievance_social_protection" } ] } From 1c2bee223387bc006493567fff95f87c8ff22e8a Mon Sep 17 00:00:00 2001 From: lruzicki <56487722+lruzicki@users.noreply.github.com> Date: Thu, 16 May 2024 12:12:18 +0200 Subject: [PATCH 28/43] CQI-146: config for password policy (#246) * CQI-146: config for password policy * CQI-146: axes config --- openIMIS/openIMIS/settings.py | 13 +++++++++++++ requirements.txt | 3 +++ 2 files changed, 16 insertions(+) diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index 33fae3e4..32eb2bb8 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -160,6 +160,7 @@ def SITE_URL(): "channels", # Websocket support "developer_tools", "drf_spectacular", # Swagger UI for FHIR API + "axes", "django_opensearch_dsl" ] INSTALLED_APPS += OPENIMIS_APPS @@ -212,8 +213,14 @@ def SITE_URL(): "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", + "axes.middleware.AxesMiddleware", + "core.middleware.DefaultAxesAttributesMiddleware", ] +AXES_ENABLED = True if os.environ.get("MODE", "DEV") == "PROD" else False +AXES_FAILURE_LIMIT = int(os.getenv("LOGIN_LOCKOUT_FAILURE_LIMIT", 5)) +AXES_COOLOFF_TIME = timedelta(minutes=int(os.getenv("LOGIN_LOCKOUT_COOLOFF_TIME", 5))) + if DEBUG: # Attach profiler middleware MIDDLEWARE.append( @@ -500,3 +507,9 @@ def SITE_URL(): }, } } + +PASSWORD_MIN_LENGTH = int(os.getenv('PASSWORD_MIN_LENGTH', 8)) +PASSWORD_UPPERCASE = int(os.getenv('PASSWORD_UPPERCASE', 1)) +PASSWORD_LOWERCASE = int(os.getenv('PASSWORD_LOWERCASE', 1)) +PASSWORD_DIGITS = int(os.getenv('PASSWORD_DIGITS', 1)) +PASSWORD_SYMBOLS = int(os.getenv('PASSWORD_SYMBOLS', 1)) diff --git a/requirements.txt b/requirements.txt index ef6d6f60..2216b017 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,3 +49,6 @@ flake8 # TODO remove to a separate file for implementations and change test workflows to reflect that nepali_datetime +zxcvbn~=4.4.28 +password-validator==1.0 +django-axes==6.4.0 From c48b5fb5a46c7fd06ed2211b18dbd30c7d4607c2 Mon Sep 17 00:00:00 2001 From: lruzicki <56487722+lruzicki@users.noreply.github.com> Date: Tue, 4 Jun 2024 09:24:16 +0200 Subject: [PATCH 29/43] CQI: security fixes for password, headers, rate limiter, csrf and lockout mechanism (#259) * CQI: security fixes for password, headers, rate limiter, csrf and lockout mechanism * Resovled issue with missing modules - error handling --- .env.example | 23 ++++++++++ README.md | 76 +++++++++++++++++++++++++++++++ openIMIS/openIMIS/openimisapps.py | 7 ++- openIMIS/openIMIS/settings.py | 70 ++++++++++++++++++++++++---- requirements.txt | 1 - 5 files changed, 165 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index 952786bf..4278f7a9 100644 --- a/.env.example +++ b/.env.example @@ -22,3 +22,26 @@ PHOTO_ROOT_PATH= DJANGO_MIGRATE=True # Should the modules be searched for scheduled tasks. Comment out for false # SCHEDULER_AUTOSTART=True + +CSRF_TRUSTED_ORIGINS=http://localhost:3000,http://localhost:8000 # Define the trusted origins for CSRF protection, separated by commas + +# Lockout mechanism +LOGIN_LOCKOUT_FAILURE_LIMIT=5 # Allowed login failures before lockout +LOGIN_LOCKOUT_COOLOFF_TIME=5 # Lockout duration in minutes + +PASSWORD_MIN_LENGTH=8 +PASSWORD_UPPERCASE=1 # Minimum number of uppercase letters +PASSWORD_LOWERCASE=1 # Minimum number of lowercase letters +PASSWORD_DIGITS=1 # Minimum number of digits +PASSWORD_SYMBOLS=1 # Minimum number of symbols +PASSWORD_SPACES=1 # Maximum number of spaces allowed + +# Rate limiting settings +RATELIMIT_CACHE=default # The cache alias to use for rate limiting +RATELIMIT_KEY=ip # Key to identify the client; 'ip' means it will use the client's IP address +RATELIMIT_RATE=150/m # Rate limit (150 requests per minute) +RATELIMIT_METHOD=ALL # HTTP methods to rate limit; 'ALL' means all methods +RATELIMIT_GROUP=graphql # Group name for the rate limit +RATELIMIT_SKIP_TIMEOUT=False # Whether to skip rate limiting + + diff --git a/README.md b/README.md index 1e9176e1..d11cbb4b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,14 @@ +# Environment Variables +| ENV | Values | Description | +| --------------------------- | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| RATELIMIT_CACHE | String | The cache alias to use for rate limiting. Defaults to `default`. | +| RATELIMIT_KEY | String | Key to identify the client for rate limiting; `ip` means it will use the client's IP address. Defaults to `ip`. | +| RATELIMIT_RATE | String | Rate limit value (e.g., `150/m` for 150 requests per minute). Defaults to `150/m`. | +| RATELIMIT_METHOD | String | HTTP methods to rate limit; `ALL` means all methods. Defaults to `ALL`. | +| RATELIMIT_GROUP | String | Group name for the rate limit. Defaults to `graphql`. | +| RATELIMIT_SKIP_TIMEOUT | Boolean | Whether to skip rate limiting during cache timeout. Defaults to `False`. | +| CSRF_TRUSTED_ORIGINS | String | Define the trusted origins for CSRF protection, separated by commas. Defaults to `http://localhost:3000,http://localhost:8000`. | + # openIMIS Backend Reference Implementation : Windows Docker This repository holds the configuration files for the openIMIS Frontend Reference Implementation: @@ -267,6 +278,71 @@ Notes: to extract frontend translations of all modules present in `openimis.json`. * those translations will be copied into 'extracted_translations_fe' folder in assembly backend module +### JWT Security Configuration + +To enhance JWT token security, you can configure the system to use RSA keys for signing and verifying tokens. + +1. **Generate RSA Keys**: + ```bash + # Generate a private key + openssl genpkey -algorithm RSA -out jwt_private_key.pem -aes256 + + # Generate a public key + openssl rsa -pubout -in jwt_private_key.pem -out jwt_public_key.pem + +2. **Store RSA Keys**: + Place jwt_private_key.pem and jwt_public_key.pem in a secure directory within your project, e.g., keys/. + +3. **Django Configuration**: + Ensure that the settings.py file is configured to read these keys. If RSA keys are found, the system will use RS256. Otherwise, it will fallback to HS256 using DJANGO_SECRET_KEY. + +Note: If RSA keys are not provided, the system defaults to HS256. Using RS256 with RSA keys is recommended for enhanced security. + +## CSRF Setup Guide + +CSRF (Cross-Site Request Forgery) protection ensures that unauthorized commands are not performed on behalf of authenticated users without their consent. It achieves this by including a unique token in each form submission or AJAX request, which is then validated by the server. +When using JWT (JSON Web Token) for authentication, CSRF protection is not executed because the server does not rely on cookies for authentication. Instead, the JWT is included in the request headers, making CSRF attacks less likely. + +### Development Environment + +In the development environment, CSRF protection is configured to allow requests from `localhost:3000` and `localhost:8000` by default in .env.example file. + +### Production Environment + +In the production environment, you need to specify the trusted origins in your `.env` file. + +1. **Trusted Origins Setup**: + - Define the trusted origins in your `.env` file to allow cross-origin requests from specific domains. + - Use a comma-separated list to specify multiple origins. + - Example of setting trusted origins in `.env`: + ```env + CSRF_TRUSTED_ORIGINS=https://example.com,https://api.example.com + ``` + + +## Security Headers + +This section describes the security headers used in the application, based on OWASP recommendations, to enhance the security of your Django application. + +### Security Headers in Production + +In the production environment, several security headers are set to protect the application from common vulnerabilities: + +- **Strict-Transport-Security**: `max-age=63072000; includeSubDomains` - Enforces secure (HTTP over SSL/TLS) connections to the server and ensures all subdomains also follow this rule. +- **Content-Security-Policy**: `default-src 'self';` - Prevents a wide range of attacks, including Cross-Site Scripting (XSS), by restricting sources of content to the same origin. +- **X-Frame-Options**: `DENY` - Protects against clickjacking attacks by preventing the page from being framed. +- **X-Content-Type-Options**: `nosniff` - Prevents the browser from MIME-sniffing the content type, ensuring that the browser uses the declared content type. +- **Referrer-Policy**: `no-referrer` - Controls how much referrer information is included with requests by not sending any referrer information with requests. +- **Permissions-Policy**: `geolocation=(), microphone=()` - Controls access to browser features by disabling access to geolocation and microphone features. + +In production, additional security settings are applied to cookies used for CSRF and JWT: + +- **CSRF_COOKIE_SECURE**: Ensures the CSRF cookie is only sent over HTTPS. +- **CSRF_COOKIE_HTTPONLY**: Prevents JavaScript from accessing the CSRF cookie. +- **CSRF_COOKIE_SAMESITE**: Sets the `SameSite` attribute to 'Lax', which allows the cookie to be sent with top-level navigations and gets rid of the risk of CSRF attacks. +- **JWT_COOKIE_SECURE**: Ensures the JWT cookie is only sent over HTTPS. +- **JWT_COOKIE_SAMESITE**: Sets the `SameSite` attribute to 'Lax' for the JWT cookie. + ## Custom exception handler for new modules REST-based modules If the module you want to add to the openIMIS uses its own REST exception handler you have to register it in the main module. To do this, you can use following code snippet in the class diff --git a/openIMIS/openIMIS/openimisapps.py b/openIMIS/openIMIS/openimisapps.py index 202bb6fa..02003735 100644 --- a/openIMIS/openIMIS/openimisapps.py +++ b/openIMIS/openIMIS/openimisapps.py @@ -21,8 +21,11 @@ def get_locale_folders(): basedirs = [] for mod in load_openimis_conf()["modules"]: mod_name = mod["name"] - with resources.path(mod_name, "__init__.py") as path: - apps.append(path.parent.parent) # This might need to be more restrictive + try: + with resources.path(mod_name, "__init__.py") as path: + apps.append(path.parent.parent) + except ModuleNotFoundError: + raise Exception(f"Module \"{mod_name}\" not found.") for topdir in ["."] + apps: for dirpath, dirnames, filenames in os.walk(topdir, topdown=True): diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index 32eb2bb8..c5da7246 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -8,6 +8,7 @@ from dotenv import load_dotenv from .openimisapps import openimis_apps, get_locale_folders from datetime import timedelta +from cryptography.hazmat.primitives import serialization load_dotenv() @@ -172,6 +173,7 @@ def SITE_URL(): AUTHENTICATION_BACKENDS += ["django.contrib.auth.backends.RemoteUserBackend"] AUTHENTICATION_BACKENDS += [ + "axes.backends.AxesStandaloneBackend", "rules.permissions.ObjectPermissionBackend", "graphql_jwt.backends.JSONWebTokenBackend", "django.contrib.auth.backends.ModelBackend", @@ -208,19 +210,31 @@ def SITE_URL(): MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", "whitenoise.middleware.WhiteNoiseMiddleware", + 'core.middleware.GraphQLRateLimitMiddleware', + "axes.middleware.AxesMiddleware", + "core.middleware.DefaultAxesAttributesMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.locale.LocaleMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", - "axes.middleware.AxesMiddleware", - "core.middleware.DefaultAxesAttributesMiddleware", + "core.middleware.SecurityHeadersMiddleware", ] AXES_ENABLED = True if os.environ.get("MODE", "DEV") == "PROD" else False AXES_FAILURE_LIMIT = int(os.getenv("LOGIN_LOCKOUT_FAILURE_LIMIT", 5)) AXES_COOLOFF_TIME = timedelta(minutes=int(os.getenv("LOGIN_LOCKOUT_COOLOFF_TIME", 5))) + +MODE = os.environ.get("MODE") + +RATELIMIT_CACHE = os.getenv('RATELIMIT_CACHE', 'default') +RATELIMIT_KEY = os.getenv('RATELIMIT_KEY', 'ip') +RATELIMIT_RATE = os.getenv('RATELIMIT_RATE', '150/m') +RATELIMIT_METHOD = os.getenv('RATELIMIT_METHOD', 'ALL') +RATELIMIT_GROUP = os.getenv('RATELIMIT_GROUP', 'graphql') +RATELIMIT_SKIP_TIMEOUT = os.getenv('RATELIMIT_SKIP_TIMEOUT', 'False') + if DEBUG: # Attach profiler middleware MIDDLEWARE.append( @@ -286,6 +300,50 @@ def SITE_URL(): ], } +# Load RSA keys +private_key_path = os.path.join(BASE_DIR, 'keys', 'jwt_private_key.pem') +public_key_path = os.path.join(BASE_DIR, 'keys', 'jwt_public_key.pem') + +if os.path.exists(private_key_path) and os.path.exists(public_key_path): + with open(private_key_path, 'rb') as f: + private_key = serialization.load_pem_private_key( + f.read(), + password=None, + ) + + with open(public_key_path, 'rb') as f: + public_key = serialization.load_pem_public_key( + f.read(), + ) + + # If RSA keys exist, update the algorithm and add keys to GRAPHQL_JWT settings + GRAPHQL_JWT.update({ + "JWT_ALGORITHM": "RS256", + "JWT_PRIVATE_KEY": private_key, + "JWT_PUBLIC_KEY": public_key, + }) + + SECURE_BROWSER_XSS_FILTER = True + SECURE_CONTENT_TYPE_NOSNIFF = True + SECURE_HSTS_SECONDS = 63072000 + SECURE_HSTS_INCLUDE_SUBDOMAINS = True + SECURE_HSTS_PRELOAD = True + SECURE_SSL_REDIRECT = True + +if MODE == "PROD": + # Enhance security in production + GRAPHQL_JWT.update({ + "JWT_COOKIE_SECURE": True, + "JWT_COOKIE_SAMESITE": "Lax", + }) + + CSRF_COOKIE_SECURE = True + CSRF_COOKIE_HTTPONLY = True + CSRF_COOKIE_SAMESITE = 'Lax' + +csrf_trusted_origins = os.environ.get('CSRF_TRUSTED_ORIGINS', default='') +CSRF_TRUSTED_ORIGINS = csrf_trusted_origins.split(',') if csrf_trusted_origins else [] + # Database # https://docs.djangoproject.com/en/2.1/ref/settings/#databases @@ -392,13 +450,7 @@ def SITE_URL(): "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", - }, - { - "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", - }, - { - "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + "NAME": "core.utils.CustomPasswordValidator", }, ] diff --git a/requirements.txt b/requirements.txt index 2216b017..ee42dda6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,7 +48,6 @@ django-opensearch-dsl==0.5.1 flake8 # TODO remove to a separate file for implementations and change test workflows to reflect that nepali_datetime - zxcvbn~=4.4.28 password-validator==1.0 django-axes==6.4.0 From 6e9aefaa140293edb0f5db80c3963d4ac84984f9 Mon Sep 17 00:00:00 2001 From: Wei Lu Date: Fri, 14 Jun 2024 15:42:51 -0400 Subject: [PATCH 30/43] Remove modules unneeded for a basic social protection MIS setup FHIR and DHIS are both healthcare specific modules --- openimis.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/openimis.json b/openimis.json index 7160fb9a..3acd5b84 100644 --- a/openimis.json +++ b/openimis.json @@ -68,10 +68,6 @@ "name": "tools", "pip": "git+https://github.com/openimis/openimis-be-tools_py.git@develop#egg=openimis-be-tools" }, - { - "name": "api_fhir_r4", - "pip": "git+https://github.com/openimis/openimis-be-api_fhir_r4_py.git@develop#egg=openimis-be-api_fhir_r4" - }, { "name": "calculation", "pip": "git+https://github.com/openimis/openimis-be-calculation_py.git@develop#egg=openimis-be-calculation" @@ -124,10 +120,6 @@ "name":"im_export", "pip":"git+https://github.com/openimis/openimis-be-im_export_py.git@develop#egg=openimis-be-im_export" }, - { - "name":"dhis2_etl", - "pip":"git+https://github.com/openimis/openimis-be-dhis2_etl_py.git@develop#egg=openimis-be-dhis2_etl" - }, { "name": "social_protection", "pip": "git+https://github.com/openimis/openimis-be-social_protection_py.git@develop#egg=openimis-be-social_protection" From be690af6b03e1bc3bb741016756c5cb9359d3634 Mon Sep 17 00:00:00 2001 From: Wei Lu Date: Tue, 18 Jun 2024 07:54:47 -0400 Subject: [PATCH 31/43] Update README with instructions to work with psql (#261) As the dev database --- .env.example | 2 ++ README.md | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 4278f7a9..6ad142b3 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,5 @@ +# Database engine, if not specified will default to mssql +DB_ENGINE= # Database host DB_HOST= # Database port diff --git a/README.md b/README.md index d11cbb4b..ae95ae55 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ In case of troubles, please consult/contact our service desk via our [ticketing * start openIMIS from within `openimis-be_py/openIMIS`: `python manage.py runserver` At this stage, you may (depends on the database you connect to) need to: -* apply django migrations, from `openimis-be_py/openIMIS`: `python manage.py migrate` +* apply django migrations, from `openimis-be_py/openIMIS`: `python manage.py migrate`. See [PostgresQL section](#postgresql) if you are using postgresql for dev DB. * create a superuser for django admin console, from `openimis-be_py/openIMIS`: `python manage.py createsuperuser` (will not prompt for a password) and then `python manage.py changepassword @@ -172,6 +172,39 @@ Notes: * default 'options' in openIMIS are `{'driver': 'ODBC Driver 17 for SQL Server','unicode_results': True}` If you need to provide other options, use the `DB_OPTIONS` entry in the `.env` file (be complete: the new json string will entirely replace the default one) +### PostgresQL + +**Requirement:** `postgres-json-schema`. +Follow the README at https://github.com/gavinwahl/postgres-json-schema to install. + +To use postgresql as the dev database, specify `DB_ENGINE` in `.env` alongside other DB config vars: +``` +DB_ENGINE=django.db.backends.postgresql +DB_HOST=localhost +DB_PORT=5432 +DB_NAME=coremis +DB_USER=postgres +``` + +Create the database, named `coremis` using the example `DB_NAME` above: +```bash +psql postgres -c 'create database coremis' +``` + +Before applying django migrations, the database needs to be prepared using scripts from https://github.com/openimis/database_postgresql: + +```bash +git clone https://github.com/openimis/database_postgresql +cd database_postgresql + +# generate concatenated .sql for easy execution +bash concatenate_files.sh + +# prepare the database - replace fullDemoDatabase.sql with EmptyDatabase.sql if you don't need demo data +psql -d coremis -a -f output/fullDemoDatabase.sql +``` + +From here on django's `python manage.py migrate` should execute succesfully. ## Developer tools From e77de52280b4d4cbd3f44588cab252cac5d2ff25 Mon Sep 17 00:00:00 2001 From: Wei Lu Date: Thu, 11 Jul 2024 14:55:44 +0800 Subject: [PATCH 32/43] Fix django.core.exceptions.ImproperlyConfigured: STATICFILES_STORAGE/STORAGES are mutually exclusive. --- openIMIS/openIMIS/settings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index 35e0fa3f..61b3f45a 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -547,7 +547,6 @@ def SITE_URL(): # https://docs.djangoproject.com/en/2.1/howto/static-files/ STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles") -STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" STATIC_URL = "/%sstatic/" % SITE_ROOT() @@ -629,7 +628,10 @@ def SITE_URL(): "location": MEDIA_URL, "base_url": MEDIA_URL }, - } + }, + 'staticfiles': { + 'BACKEND': "whitenoise.storage.CompressedManifestStaticFilesStorage", + }, } PASSWORD_MIN_LENGTH = int(os.getenv('PASSWORD_MIN_LENGTH', 8)) From f19a284df94a2f7e3d16c9fd4aee3e40fbe65500 Mon Sep 17 00:00:00 2001 From: Patrick Delcroix Date: Tue, 16 Jul 2024 15:46:04 +0200 Subject: [PATCH 33/43] removing useless test batch --- openIMIS/openIMIS/settings.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index 61b3f45a..ad191b7c 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -474,11 +474,6 @@ def SITE_URL(): # This list will be called with scheduler.add_job() as specified: # Note that the document implies that the time is local and follows DST but that seems false and in UTC regardless SCHEDULER_JOBS = [ - { - "method": "core.tasks.openimis_test_batch", - "args": ["cron"], - "kwargs": {"id": "openimis_test_batch", "minute": 16, "replace_existing": True}, - }, { "method": "policy.tasks.get_policies_for_renewal", "args": ["cron"], From 21413b441d08f02cd6118b0bdf905e3937ab0c07 Mon Sep 17 00:00:00 2001 From: Wei Lu Date: Wed, 17 Jul 2024 17:19:24 +0800 Subject: [PATCH 34/43] Fix default storage backend config (#275) --- openIMIS/openIMIS/settings.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index ad191b7c..73edcd3c 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -618,11 +618,7 @@ def SITE_URL(): STORAGES = { "default": { - "class": "core.filesystem.storage.FileSystemStorage", - "options": { - "location": MEDIA_URL, - "base_url": MEDIA_URL - }, + "BACKEND": "django.core.files.storage.FileSystemStorage", }, 'staticfiles': { 'BACKEND': "whitenoise.storage.CompressedManifestStaticFilesStorage", From c587587933292770179aeb4653a0988118376bbd Mon Sep 17 00:00:00 2001 From: Wei Lu Date: Mon, 12 Aug 2024 16:38:01 +0800 Subject: [PATCH 35/43] Flag if the current env is unit testing For switching off OpenSearch in Django tests --- openIMIS/openIMIS/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index 73edcd3c..308bcbdd 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -4,6 +4,7 @@ import json import logging import os +import sys from dotenv import load_dotenv from .openimisapps import openimis_apps, get_locale_folders @@ -630,3 +631,5 @@ def SITE_URL(): PASSWORD_LOWERCASE = int(os.getenv('PASSWORD_LOWERCASE', 1)) PASSWORD_DIGITS = int(os.getenv('PASSWORD_DIGITS', 1)) PASSWORD_SYMBOLS = int(os.getenv('PASSWORD_SYMBOLS', 1)) + +IS_UNIT_TEST_ENV = 'test' in sys.argv From fe0571e629624a7835ea00b7193e440bb729e258 Mon Sep 17 00:00:00 2001 From: sniedzielski <52816247+sniedzielski@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:28:53 +0200 Subject: [PATCH 36/43] hotfix: fixed CI - bumped upload articact to v4 (#285) --- .github/workflows/ci_module.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_module.yml b/.github/workflows/ci_module.yml index b4fce97a..53ec4ef3 100755 --- a/.github/workflows/ci_module.yml +++ b/.github/workflows/ci_module.yml @@ -109,13 +109,13 @@ jobs: tar -czf openimis.tar.gz ./openimis ./current-module - name: Upload compressed site-packages as artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: site-packages path: site-packages.tar.gz - name: Upload build as artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: code-artifacts path: openimis.tar.gz @@ -450,7 +450,7 @@ jobs: cat coverage.xml - name: Coverage results if: success() || failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage.xml path: ./openimis/openIMIS/coverage @@ -498,7 +498,7 @@ jobs: python -m flake8 $MOD_DIR - name: Flake8 results upload if: success() || failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: flake8-report.txt path: ./openimis/flake8-report.txt From 88cc99d94351ba0376d51a4bfc410821b8064742 Mon Sep 17 00:00:00 2001 From: sniedzielski <52816247+sniedzielski@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:54:10 +0200 Subject: [PATCH 37/43] hotfix: fixed CI - bumped download articact to v4 (#288) --- .github/workflows/ci_module.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_module.yml b/.github/workflows/ci_module.yml index 53ec4ef3..b13567c3 100755 --- a/.github/workflows/ci_module.yml +++ b/.github/workflows/ci_module.yml @@ -145,7 +145,7 @@ jobs: python-version: '3.10' - name: Download site-packages artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: site-packages path: /tmp/ @@ -156,7 +156,7 @@ jobs: echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV - name: Download site-packages artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: code-artifacts path: /tmp/ @@ -255,7 +255,7 @@ jobs: python-version: '3.10' - name: Download site-packages artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: site-packages path: /tmp/ @@ -266,7 +266,7 @@ jobs: echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV - name: Download site-packages artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: code-artifacts path: /tmp/ @@ -359,7 +359,7 @@ jobs: python-version: '3.10' - name: Download site-packages artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: site-packages path: /tmp/ @@ -370,7 +370,7 @@ jobs: echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV - name: Download site-packages artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: code-artifacts path: /tmp/ @@ -464,7 +464,7 @@ jobs: with: python-version: '3.10' - name: Download site-packages artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: site-packages path: /tmp/ @@ -475,7 +475,7 @@ jobs: echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV - name: Download code artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: code-artifacts path: /tmp/ @@ -530,17 +530,17 @@ jobs: echo "SONAR_EXCLUSIONS=${{ inputs.SONAR_EXCLUSIONS }}" >> $GITHUB_ENV fi - name: Download coverage artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: coverage.xml path: report - name: Download flake8 artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: flake8-report.txt path: report - name: Download code artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: code-artifacts path: /tmp/ From 457715d9c7c24cb6fc76b0cbb33176e4dc52ade2 Mon Sep 17 00:00:00 2001 From: sniedzielski <52816247+sniedzielski@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:25:39 +0200 Subject: [PATCH 38/43] CM-963: added etl api backend module (#296) --- openimis.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openimis.json b/openimis.json index 12016042..ab934385 100644 --- a/openimis.json +++ b/openimis.json @@ -163,6 +163,10 @@ { "name": "claim_sampling", "pip": "git+https://github.com/openimis/openimis-be-claim_sampling_py.git@develop#egg=openimis-be-claim_sampling" + }, + { + "name": "api_etl", + "pip": "git+https://github.com/openimis/openimis-be-api_etl_py.git@develop#egg=openimis-be-api_etl" } ] } From 444645ff18ec8639cde8b3247528ccb800abdd59 Mon Sep 17 00:00:00 2001 From: sniedzielski Date: Thu, 24 Oct 2024 11:54:34 +0200 Subject: [PATCH 39/43] hotfix: fixed cache issue --- openIMIS/openIMIS/settings.py | 38 +++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index 308bcbdd..74a94be1 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -450,13 +450,39 @@ def SITE_URL(): CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND") if 'CACHE_BACKEND' in os.environ and 'CACHE_URL' in os.environ: - CACHES = { - 'default': { - 'BACKEND': os.environ.get('CACHE_BACKEND'), - 'LOCATION': os.environ.get("CACHE_URL"), - 'OPTIONS': json.loads(os.environ.get("CACHE_OPTIONS", "")) - } + CACHE_BACKEND = os.environ.get('CACHE_BACKEND') + CACHE_URL = os.environ.get("CACHE_URL") + CACHE_OPTIONS = os.environ.get("CACHE_OPTIONS", None) + if CACHE_OPTIONS: + CACHE_OPTIONS = json.loads(CACHE_OPTIONS) +else: + CACHE_BACKEND = 'django.core.cache.backends.locmem.LocMemCache' + CACHE_URL = None + CACHE_OPTIONS = None + +CACHE_PARAM = {} +CACHE_PARAM['BACKEND'] = CACHE_BACKEND +if CACHE_URL: + CACHE_PARAM['LOCATION'] = CACHE_URL + +if CACHE_OPTIONS: + CACHE_PARAM['OPTIONS'] = CACHE_OPTIONS + +CACHES = { + 'default': { + **CACHE_PARAM, + 'KEY_PREFIX': "oi" + }, + 'location': { + **CACHE_PARAM, + 'KEY_PREFIX': "loc" + }, + 'coverage': { + **CACHE_PARAM, + 'KEY_PREFIX': "cov" + } +} # This scheduler config will: # - Store jobs in the project database From 59e58b8e6b8b034f6cd2148273c78b63acea2a29 Mon Sep 17 00:00:00 2001 From: sniedzielski Date: Fri, 7 Feb 2025 13:59:00 +0100 Subject: [PATCH 40/43] OSB-12: added command to load fixture with unknown foreign key --- .../commands/load_fixture_foreign_key.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 openIMIS/developer_tools/management/commands/load_fixture_foreign_key.py diff --git a/openIMIS/developer_tools/management/commands/load_fixture_foreign_key.py b/openIMIS/developer_tools/management/commands/load_fixture_foreign_key.py new file mode 100644 index 00000000..1317b7c6 --- /dev/null +++ b/openIMIS/developer_tools/management/commands/load_fixture_foreign_key.py @@ -0,0 +1,100 @@ +from django.core.management.base import BaseCommand +from django.core.exceptions import ObjectDoesNotExist +import json +from django.apps import apps +from django.core.management import call_command +import os + + +class Command(BaseCommand): + help = 'Load a fixture and replace foreign keys using a natural key (like uuid, name, location, etc.) with corresponding model IDs' + + def add_arguments(self, parser): + # Argument for the fixture file + parser.add_argument( + 'fixture_file', + type=str, + help='Path to the fixture file (JSON format)' + ) + # Argument for the field name that will be used as the natural key (e.g., "uuid", "name", "location", etc.) + parser.add_argument( + '--field', + type=str, + help="The unique field to use for resolving foreign keys (e.g., 'uuid', 'name', 'location', etc.)", + required=True + ) + + def handle(self, *args, **kwargs): + fixture_file = kwargs['fixture_file'] + field_name = kwargs['field'] + + # Load the fixture data + try: + with open(fixture_file, 'r') as f: + data = json.load(f) + except FileNotFoundError: + self.stdout.write(self.style.ERROR(f"Fixture file '{fixture_file}' not found")) + return + except json.JSONDecodeError: + self.stdout.write(self.style.ERROR(f"Fixture file '{fixture_file}' is not valid JSON")) + return + + # Process the fixture data + for obj in data: + model = obj['model'] + if model: + # Get the model class dynamically using the app label and model name + app_label, model_name = model.split('.') + try: + model_class = apps.get_model(app_label, model_name) + except LookupError: + self.stdout.write(self.style.ERROR(f"Model '{model}' not found")) + continue + + # Loop through fields in the fixture and process foreign keys + for field, field_value in obj['fields'].items(): + # If the field value is a list (e.g., ["uuid_value"]), handle it as a list + if isinstance(field_value, list): + if len(field_value) == 1: # If there is only one element (like with the 'role' field) + related_field = model_class._meta.get_field(field) + + if related_field.is_relation: # Check if it's a foreign key + # Look up the related model (e.g., Role) using the field_name (e.g., uuid, name) + related_model = related_field.related_model + try: + # We fetch the related object by the unique field (e.g., uuid, name, location, etc.) + related_object = related_model.objects.get(**{field_name: field_value[0]}) + # Replace the field value with the primary key (ID) - not a list anymore + obj['fields'][field] = related_object.id + except ObjectDoesNotExist: + self.stdout.write(self.style.ERROR( + f"{related_model} with {field_name} '{field_value[0]}' not found")) + continue + # If it's not a list, process as usual (no change needed) + elif isinstance(field_value, str): # Checking if the value is a string (uuid, name, etc.) + related_field = model_class._meta.get_field(field) + if related_field.is_relation: + related_model = related_field.related_model + try: + # Fetch the related object + related_object = related_model.objects.get(**{field_name: field_value}) + # Replace the value with the ID + obj['fields'][field] = related_object.id + except ObjectDoesNotExist: + self.stdout.write( + self.style.ERROR(f"{related_model} with {field_name} '{field_value}' not found")) + continue + + # Save the modified fixture to a new file + output_file = fixture_file.replace('.json', '_modified.json') + with open(output_file, 'w') as f: + json.dump(data, f, indent=4) + + self.stdout.write(self.style.SUCCESS(f'Successfully transformed the fixture and saved it as {output_file}')) + + # Now that unique field values are replaced with IDs, load this fixture into the database + try: + call_command('loaddata', output_file) # This will load the modified fixture + self.stdout.write(self.style.SUCCESS(f'Successfully loaded the modified fixture into the database')) + except Exception as e: + self.stdout.write(self.style.ERROR(f'Error loading fixture: {e}')) From 9362f7da4908866627dd90584f27fd633cbbf802 Mon Sep 17 00:00:00 2001 From: sniedzielski <52816247+sniedzielski@users.noreply.github.com> Date: Wed, 19 Mar 2025 15:29:44 +0100 Subject: [PATCH 41/43] changes from security audit 2024 November (#336) --- openIMIS/openIMIS/settings.py | 31 +++++++++++++++++++++++++++---- requirements.txt | 3 ++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/openIMIS/openIMIS/settings.py b/openIMIS/openIMIS/settings.py index 74a94be1..516f2c9c 100644 --- a/openIMIS/openIMIS/settings.py +++ b/openIMIS/openIMIS/settings.py @@ -224,6 +224,7 @@ def SITE_URL(): "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "core.middleware.SecurityHeadersMiddleware", + "csp.middleware.CSPMiddleware", ] MODE = os.environ.get("MODE") @@ -334,10 +335,6 @@ def SITE_URL(): "JWT_COOKIE_SAMESITE": "Lax", }) - CSRF_COOKIE_SECURE = True - CSRF_COOKIE_HTTPONLY = True - CSRF_COOKIE_SAMESITE = 'Lax' - SECURE_BROWSER_XSS_FILTER = True SECURE_CONTENT_TYPE_NOSNIFF = True SECURE_HSTS_SECONDS = 63072000 @@ -659,3 +656,29 @@ def SITE_URL(): PASSWORD_SYMBOLS = int(os.getenv('PASSWORD_SYMBOLS', 1)) IS_UNIT_TEST_ENV = 'test' in sys.argv + +# CSRF settings +CSRF_COOKIE_SECURE = True +SESSION_COOKIE_SECURE = True +# session cookie validity = 8 hours +SESSION_COOKIE_AGE = 28800 +SESSION_COOKIE_NAME = "openimis_session" + +# CORS settings +CORS_ALLOW_CREDENTIALS = True + +# Cookie settings +CSRF_COOKIE_NAME = 'csrftoken' +CSRF_USE_SESSIONS = True +SESSION_COOKIE_SAMESITE = 'Lax' # or 'None' if cross-site +CSRF_COOKIE_SAMESITE = 'Lax' # or 'None' if cross-site +CSRF_COOKIE_HTTPONLY = False # False if you need to access it from JavaScript + +USER_AGENT_CSRF_BYPASS = [] + +CSP_DEFAULT_SRC = ["'self'"] +CSP_SCRIPT_SRC = ["'self'"] +CSP_STYLE_SRC = ["'self'"] +CSP_IMG_SRC = ["'self'", "data:"] # Allows images from the same origin and base64 encoded images +CSP_FRAME_ANCESTORS = ["'self'"] + diff --git a/requirements.txt b/requirements.txt index 5bc8b0b2..55111bdf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ waitress wheel whitenoise django-health-check -requests~=2.32.0 +requests~=2.32.0 apscheduler==3.10.1 # As from v0.4, Django-apscheduler has a migration that is incompatible with SQL Server # (autoincrement int => bigint) so we are using our own fork with a squashed migration @@ -57,3 +57,4 @@ django-opensearch-dsl==0.5.1 zxcvbn~=4.4.28 password-validator==1.0 django-axes==6.4.0 +django-csp From 9950417363902823a8538227aa674d4338701efc Mon Sep 17 00:00:00 2001 From: Wei Lu Date: Wed, 14 May 2025 11:52:37 -0400 Subject: [PATCH 42/43] Merge branch 'develop' into coreMIS --- .env.example | 3 + .env.openSearch | 8 + .github/workflows/ci_assembly.yml | 198 +- .github/workflows/ci_module.yml | 275 +- .github/workflows/docker-manual.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/pgsql.yml | 92 - .travis.yml | 4 +- .vscode/launch.json | 41 +- .vscode/settings.json | 1 + Dockerfile | 4 +- README.md | 399 +- docker-compose.yml | 20 +- .../commands/extract_translations.py | 2 +- .../skeletons/calculation_rule.py.template | 31 +- .../skeletons/openmis-module-test.yml | 111 +- .../skeletons/python-publish.yml | 4 +- openIMIS/locale/en/LC_MESSAGES/django.po | 5579 +++++++++++++++- openIMIS/locale/fr/LC_MESSAGES/django.po | 5675 ++++++++++++++++- openIMIS/openIMIS/asgi.py | 11 +- openIMIS/openIMIS/openimisapps.py | 130 +- openIMIS/openIMIS/openimisconf.py | 14 +- openIMIS/openIMIS/schema.py | 4 +- openIMIS/openIMIS/settings.py | 684 -- openIMIS/openIMIS/settings/__init__.py | 38 + openIMIS/openIMIS/settings/base.py | 212 + openIMIS/openIMIS/settings/common.py | 10 + openIMIS/openIMIS/settings/database.py | 95 + openIMIS/openIMIS/settings/dev.py | 17 + openIMIS/openIMIS/settings/logging.py | 52 + openIMIS/openIMIS/settings/opensearch.py | 15 + openIMIS/openIMIS/settings/prod.py | 44 + openIMIS/openIMIS/settings/queue_cache.py | 54 + openIMIS/openIMIS/settings/scheduler.py | 41 + openIMIS/openIMIS/settings/security.py | 130 + openIMIS/openIMIS/settings/sentry.py | 32 + openIMIS/openIMIS/settings/trad.py | 18 + openIMIS/openIMIS/tracer.py | 22 +- openIMIS/openIMIS/views.py | 8 +- openIMIS/receiver_binding/__init__.py | 1 + openIMIS/receiver_binding/apps.py | 27 + openimis.json | 344 +- requirements.txt | 4 +- script/all_requirements.py | 25 + script/launch-mssql-in-docker.sh | 2 +- script/modules-requirements.py | 4 +- script/setup-local-dev.py | 23 +- script/utils.py | 2 +- 48 files changed, 12857 insertions(+), 1657 deletions(-) create mode 100644 .env.openSearch delete mode 100644 .github/workflows/pgsql.yml create mode 100644 .vscode/settings.json delete mode 100644 openIMIS/openIMIS/settings.py create mode 100644 openIMIS/openIMIS/settings/__init__.py create mode 100644 openIMIS/openIMIS/settings/base.py create mode 100644 openIMIS/openIMIS/settings/common.py create mode 100644 openIMIS/openIMIS/settings/database.py create mode 100644 openIMIS/openIMIS/settings/dev.py create mode 100644 openIMIS/openIMIS/settings/logging.py create mode 100644 openIMIS/openIMIS/settings/opensearch.py create mode 100644 openIMIS/openIMIS/settings/prod.py create mode 100644 openIMIS/openIMIS/settings/queue_cache.py create mode 100644 openIMIS/openIMIS/settings/scheduler.py create mode 100644 openIMIS/openIMIS/settings/security.py create mode 100644 openIMIS/openIMIS/settings/sentry.py create mode 100644 openIMIS/openIMIS/settings/trad.py create mode 100644 openIMIS/receiver_binding/__init__.py create mode 100644 openIMIS/receiver_binding/apps.py create mode 100644 script/all_requirements.py diff --git a/.env.example b/.env.example index 79b6b6b2..71e382ff 100644 --- a/.env.example +++ b/.env.example @@ -62,3 +62,6 @@ RATELIMIT_RATE=150/m # Rate limit (150 requests per minute) RATELIMIT_METHOD=ALL # HTTP methods to rate limit; 'ALL' means all methods RATELIMIT_GROUP=graphql # Group name for the rate limit RATELIMIT_SKIP_TIMEOUT=False # Whether to skip rate limiting + +OPENSEARCH_ADMIN=admin +OPENSEARCH_PASSWORD=B9wc9VrqX7pY diff --git a/.env.openSearch b/.env.openSearch new file mode 100644 index 00000000..63a7deae --- /dev/null +++ b/.env.openSearch @@ -0,0 +1,8 @@ +DISCOVERY_TYPE=single-node +CLUSTER_NAME=my_opensearch_cluster +OPEN_SEARCH_HTTP_PORT=9200 +SLL_HTTP_ENABLED=false +OPENSEARCH_ADMIN=admin +OPENSEARCH_PASSWORD=B9wc9VrqX7pY +OPENSEARCH_HOSTS='opensearch:9200' +OPENSEARCH_DSL_AUTOSYNC=True \ No newline at end of file diff --git a/.github/workflows/ci_assembly.yml b/.github/workflows/ci_assembly.yml index 1e60d52f..daddb7e7 100755 --- a/.github/workflows/ci_assembly.yml +++ b/.github/workflows/ci_assembly.yml @@ -11,32 +11,32 @@ on: jobs: ci_assembly: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest services: - mssql: - image: mcr.microsoft.com/mssql/server:2017-latest - env: - ACCEPT_EULA: Y - SA_PASSWORD: GitHub999 - ports: - - 1433:1433 - # this health-cmd needed because the mssql container does not provide a health check - options: >- - --health-interval=10s - --health-timeout=5s - --health-start-period=10s - --health-retries=10 - --health-cmd="/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P ${SA_PASSWORD} -Q 'SELECT 1' || exit 1" - + # mssql: + # image: mcr.microsoft.com/mssql/server:2022-CU14-ubuntu-22.04 + # env: + # ACCEPT_EULA: Y + # SA_PASSWORD: YourStrong!Passw0rd + # ports: + # - 1433:1433 + # options: + # --health-interval=10s + # --health-interval=20s + # --health-timeout=3s + # --health-start-period=10s + # --health-retries=10 + # --health-cmd="sqlcmd -S localhost -U sa -P ${SA_PASSWORD} -Q 'SELECT 1' || exit 1" + pgsql: - image: postgres + image: ghcr.io/openimis/openimis-pgsql:develop env: DB_HOST: localhost DB_PORT: 5432 - POSTGRES_DB: imis + POSTGRES_DB: test_imis POSTGRES_USER: postgres - POSTGRES_PASSWORD: GitHub999 + POSTGRES_PASSWORD: YourStrong!Passw0rd ports: - 5432:5432 options: >- @@ -44,16 +44,31 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 10 + opensearch: + image: opensearchproject/opensearch:latest + ports: + - 9200:9200 + env: + discovery.type: single-node + cluster.name: my_opensearch_local + http.port: 9200 + plugins.security.ssl.http.enabled: false + OPENSEARCH_INITIAL_ADMIN_PASSWORD: B9wc9VrqX7pY + options: >- + --health-cmd "curl -f -u admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD} http://localhost:9200/_cluster/health || exit 1" + --health-interval 10s + --health-timeout 5s + --health-retries 10 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Cache dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} @@ -81,56 +96,52 @@ jobs: - name: Initialize PSQL run: | export DB_NAME_TEST="test_$DB_NAME" - if [ ${GITHUB_REF##*/} = "main" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database" git clone --depth=1 --branch=$DBBRANCH https://github.com/openimis/database_postgresql.git ./sql_psql - + sudo bash ./sql_psql/install_postgres_json_schema_extension.sh echo 'set search_path to public' >> ~/.psqlrc - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -c "DROP DATABASE IF EXISTS \"$DB_NAME_TEST\";" - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -c "CREATE DATABASE \"$DB_NAME_TEST\";" - - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/00_dump.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/02_aux_functions.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/03_views.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/04_functions.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/05_stored_procs.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/demo_db.sql | grep . | uniq -c - + PGPASSWORD=YourStrong!Passw0rd psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/json_schema_extension.sql | grep . | uniq -c env: DB_HOST: localhost DB_PORT: 5432 DB_NAME: imis DB_USER: postgres - DB_PASSWORD: GitHub999 - - - name: Initialize MSSQL - run: | - export DB_NAME_TEST="test_$DB_NAME" - - if [ ${GITHUB_REF##*/} = "main" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi - echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database" - git clone --depth=1 --branch=$DBBRANCH https://github.com/openimis/database_ms_sqlserver.git ./sql_mssql - cd sql_mssql/ && bash concatenate_files.sh && cd .. + DB_PASSWORD: YourStrong!Passw0rd - curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list - sudo apt-get update - sudo apt-get install mssql-tools unixodbc-dev + # - name: Initialize MSSQL + # run: | + # export DB_NAME_TEST="test_$DB_NAME" - #Main db has to exists for tests to run correctly, psql container creates empty main db by default - /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "DROP DATABASE IF EXISTS $DB_NAME" - /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "CREATE DATABASE $DB_NAME" - - /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "DROP DATABASE IF EXISTS $DB_NAME_TEST" - /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "CREATE DATABASE $DB_NAME_TEST" - /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -d $DB_NAME_TEST -i ./sql_mssql/output/fullDemoDatabase.sql | grep . | uniq -c - env: - DB_HOST: localhost - DB_PORT: 1433 - DB_NAME: imis - DB_USER: sa - DB_PASSWORD: GitHub999 + # if [ ${GITHUB_REF##*/} = "main" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi + # echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database" + # git clone --depth=1 --branch=$DBBRANCH https://github.com/openimis/database_ms_sqlserver.git ./sql_mssql + # cd sql_mssql/ && bash concatenate_files.sh && cd .. + + # curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - + # curl https://packages.microsoft.com/config/ubuntu/24.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list + # sudo apt-get update + # sudo apt-get install -y --allow-downgrades --allow-remove-essential --allow-change-held-packages mssql-tools18 unixodbc-dev + # echo "Waiting for SQL Server to be fully operational..." + # until /opt/mssql-tools18/bin/sqlcmd -S localhost -U $DB_USER -P $DB_PASSWORD -Q "SELECT @@VERSION" > /dev/null 2>&1 + # do + # echo "SQL Server is still initializing. Waiting..." + # sleep 5 + # done + # echo "SQL Server is now operational." + # #Main db has to exists for tests to run correctly, psql container creates empty main db by default + # /opt/mssql-tools18/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "DROP DATABASE IF EXISTS $DB_NAME" + # /opt/mssql-tools18/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "CREATE DATABASE $DB_NAME" + + # /opt/mssql-tools18/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "DROP DATABASE IF EXISTS $DB_NAME_TEST" + # /opt/mssql-tools18/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "CREATE DATABASE $DB_NAME_TEST" + # /opt/mssql-tools18/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -d $DB_NAME_TEST -i ./sql_mssql/output/fullDemoDatabase.sql | grep . | uniq -c + # env: + # DB_HOST: localhost + # DB_PORT: 1433 + # DB_NAME: imis + # DB_USER: sa + # DB_PASSWORD: YourStrong!Passw0rd - name: Django tests PSQL working-directory: ./openIMIS @@ -145,38 +156,49 @@ jobs: env: SECRET_KEY: secret MODE: DEV - DB_ENGINE: django.db.backends.postgresql + DB_DEFAULT: postgresql + CELERY_BROKER_URL: "memory://openIMIS-test//" + CELERY_RESULT_BACKEND: "cache+memory://openIMIS-test//" #DJANGO_SETTINGS_MODULE: hat.settings DB_HOST: localhost DB_PORT: 5432 DB_NAME: imis DB_USER: postgres - DB_PASSWORD: GitHub999 - #DEV_SERVER: true - SITE_ROOT: api - CI_EXCLUDED_MODULE: ${{ vars.CI_EXCLUDED_MODULE }} - - - name: Django tests MSSQL - working-directory: ./openIMIS - # Run the tests regardless if previous steps failed (if setup fails the tests should crash instantly) - if: success() || failure() - run: | - python -V - export MODULES=$(jq -r '(.modules[].name)' ../openimis.json | xargs) - MODULES=$(echo "$MODULES" | sed -E "s/\b$(echo "${CI_EXCLUDED_MODULE// /\\b|\\b}")\b/ /g" | xargs) - echo $MODULES - python manage.py test --debug-mode --timing --keepdb $MODULES - env: - SECRET_KEY: secret - MODE: DEV - DB_ENGINE: mssql - #DJANGO_SETTINGS_MODULE: hat.settings - DB_HOST: localhost - DB_PORT: 1433 - DB_NAME: imis - DB_USER: sa - DB_PASSWORD: GitHub999 + DB_PASSWORD: YourStrong!Passw0rd #DEV_SERVER: true SITE_ROOT: api - #REMOTE_USER_AUTHENTICATION: False CI_EXCLUDED_MODULE: ${{ vars.CI_EXCLUDED_MODULE }} + OPENSEARCH_ADMIN: admin + OPENSEARCH_PASSWORD: B9wc9VrqX7pY + OPEN_SEARCH_HTTP_PORT: 9200 + OPENSEARCH_HOSTS: "localhost:9200" + + # - name: Django tests MSSQL + # working-directory: ./openIMIS + # # Run the tests regardless if previous steps failed (if setup fails the tests should crash instantly) + # if: success() || failure() + # run: | + # python -V + # export MODULES=$(jq -r '(.modules[].name)' ../openimis.json | xargs) + # MODULES=$(echo "$MODULES" | sed -E "s/\b$(echo "${CI_EXCLUDED_MODULE// /\\b|\\b}")\b/ /g" | xargs) + # echo $MODULES + # python manage.py test --debug-mode --timing --keepdb $MODULES + # env: + # SECRET_KEY: secret + # MODE: DEV + # DB_DEFAULT: mssql + # CELERY_BROKER_URL: "memory://openIMIS-test//" + # CELERY_RESULT_BACKEND: "cache+memory://openIMIS-test//" + # #DJANGO_SETTINGS_MODULE: hat.settings + # DB_HOST: localhost + # DB_PORT: 1433 + # DB_NAME: imis + # DB_USER: sa + # DB_PASSWORD: YourStrong!Passw0rd + # #DEV_SERVER: true + # SITE_ROOT: api + # #REMOTE_USER_AUTHENTICATION: False + # CI_EXCLUDED_MODULE: ${{ vars.CI_EXCLUDED_MODULE }} + # OPENSEARCH_DSL_AUTOSYNC: False + # OPEN_SEARCH_HTTP_PORT: 9200 + # OPENSEARCH_HOSTS: "localhost:9200" diff --git a/.github/workflows/ci_module.yml b/.github/workflows/ci_module.yml index b13567c3..e1e94ada 100755 --- a/.github/workflows/ci_module.yml +++ b/.github/workflows/ci_module.yml @@ -41,27 +41,51 @@ on: jobs: build_backend: name: Build Backend - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Set up Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.10' - name: Pull openIMIS Backend run: | - rm ./openimis -rf - git clone --depth=1 --branch=${{ inputs.branchName }} https://github.com/openimis/openimis-be_py.git ./openimis + # Determine the target branch + if [ "${{ github.event_name }}" == "pull_request" ]; then + TARGET_BRANCH="${{ github.event.pull_request.base.ref }}" + else + TARGET_BRANCH="${{ github.event.ref }}" + TARGET_BRANCH=${TARGET_BRANCH#refs/heads/} + fi + + # Use 'develop' as fallback if TARGET_BRANCH is empty or not a release branch + if [[ -z "$TARGET_BRANCH" ]]; then + TARGET_BRANCH="develop" + fi + + echo "Target branch: $TARGET_BRANCH" + + # Clone the repository + rm -rf ./openimis + git clone --depth=1 --branch=$TARGET_BRANCH https://github.com/openimis/openimis-be_py.git ./openimis || { + echo "Failed to clone $TARGET_BRANCH, falling back to develop" + git clone --depth=1 --branch=develop https://github.com/openimis/openimis-be_py.git ./openimis + } + + # Verify the cloned branch + cd ./openimis + CLONED_BRANCH=$(git rev-parse --abbrev-ref HEAD) + echo "Cloned branch: $CLONED_BRANCH" - name: Copy Current branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: # Disabling shallow clone is recommended for improving relevancy of reporting fetch-depth: 0 path: './current-module' - name: Cache dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} @@ -119,128 +143,34 @@ jobs: with: name: code-artifacts path: openimis.tar.gz - ci_module_mssql_tests: - name: Run Module Tests (MSSQL) - runs-on: ubuntu-20.04 - needs: build_backend - services: - mssql: - image: mcr.microsoft.com/mssql/server:2017-latest - env: - ACCEPT_EULA: Y - SA_PASSWORD: GitHub999 - ports: - - 1433:1433 - # this health-cmd needed because the mssql container does not provide a health check - options: >- - --health-interval=10s - --health-timeout=5s - --health-start-period=10s - --health-retries=10 - --health-cmd="/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P ${SA_PASSWORD} -Q 'SELECT 1' || exit 1" - steps: - - name: Set up Python 3.10 - uses: actions/setup-python@v2 - with: - python-version: '3.10' - - - name: Download site-packages artifact - uses: actions/download-artifact@v4 - with: - name: site-packages - path: /tmp/ - - - name: Locate site-packages - run: | - SITE_PACKAGES=$(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") - echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV - - - name: Download site-packages artifact - uses: actions/download-artifact@v4 - with: - name: code-artifacts - path: /tmp/ - - - name: Decompress artifacts - run: | - sudo tar -xzf /tmp/site-packages.tar.gz -C ${{ env.SITE_PACKAGES }} - sudo tar -xzf /tmp/openimis.tar.gz -C . - - name: Initialize MSSQL - run: | - export DB_NAME_TEST="test_$DB_NAME" - - if [ ${GITHUB_REF##*/} = "main" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi - echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database" - git clone --depth=1 --branch=$DBBRANCH https://github.com/openimis/database_ms_sqlserver.git ./sql_mssql - cd sql_mssql/ && bash concatenate_files.sh && cd .. - - curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list - sudo apt-get update - sudo apt-get install mssql-tools unixodbc-dev - - #Main db has to exists for tests to run correctly, psql container creates empty main db by default - /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "DROP DATABASE IF EXISTS $DB_NAME" - /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "CREATE DATABASE $DB_NAME" - - /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "DROP DATABASE IF EXISTS $DB_NAME_TEST" - /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "CREATE DATABASE $DB_NAME_TEST" - /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -d $DB_NAME_TEST -i ./sql_mssql/output/fullDemoDatabase.sql | grep . | uniq -c - env: - DB_HOST: localhost - DB_PORT: 1433 - DB_NAME: imis - DB_USER: sa - DB_PASSWORD: GitHub999 - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - ${{ runner.os }}-pip- - ${{ runner.os }}- - - name: Django tests MSSQL Module - working-directory: ./openimis/openIMIS - # Run the tests regardless if previous steps failed (if setup fails the tests should crash instantly) - if: success() || failure() - run: | - python -V - export MODULES=$(jq -r '(.modules[].name)' ../openimis.json | xargs) - MODULES=$(echo "$MODULES" | sed -E "s/\b$(echo "${CI_EXCLUDED_MODULE// /\\b|\\b}")\b/ /g" | xargs) - - export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')" - echo "MODULE_NAME=$MODULE_NAME" >> $GITHUB_ENV - - python manage.py test --debug-mode --timing --keepdb $MODULE_NAME - env: - SECRET_KEY: secret - DB_ENGINE: mssql - #DJANGO_SETTINGS_MODULE: hat.settings - DB_HOST: localhost - DB_PORT: 1433 - DB_NAME: imis - DB_USER: sa - DB_PASSWORD: GitHub999 - #DEV_SERVER: true - SITE_ROOT: api - #REMOTE_USER_AUTHENTICATION: False - CI_EXCLUDED_MODULE: ${{ vars.CI_EXCLUDED_MODULE }} - MODE: DEV ci_module_psql_test_module_only: name: Run Module Tests (PSQL) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 needs: build_backend services: + opensearch: + image: opensearchproject/opensearch:latest + ports: + - 9200:9200 + env: + discovery.type: single-node + cluster.name: my_opensearch_local + http.port: 9200 + plugins.security.ssl.http.enabled: false + OPENSEARCH_INITIAL_ADMIN_PASSWORD: B9wc9VrqX7pY + options: >- + --health-cmd "curl -f -u admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD} http://localhost:9200/_cluster/health || exit 1" + --health-interval 10s + --health-timeout 5s + --health-retries 10 pgsql: - image: postgres + image: ghcr.io/openimis/openimis-pgsql:develop env: DB_HOST: localhost DB_PORT: 5432 POSTGRES_DB: test_imis POSTGRES_USER: postgres - POSTGRES_PASSWORD: GitHub999 + POSTGRES_PASSWORD: YourStrong!Passw0rd ports: - 5432:5432 options: >- @@ -250,7 +180,7 @@ jobs: --health-retries 10 steps: - name: Set up Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.10' @@ -278,73 +208,61 @@ jobs: - name: Initialize PSQL run: | export DB_NAME_TEST="test_$DB_NAME" - if [ ${GITHUB_REF##*/} = "main" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database" git clone --depth=1 --branch=$DBBRANCH https://github.com/openimis/database_postgresql.git ./sql_psql - + sudo bash ./sql_psql/install_postgres_json_schema_extension.sh echo 'set search_path to public' >> ~/.psqlrc - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -c "DROP DATABASE IF EXISTS \"$DB_NAME_TEST\";" - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -c "CREATE DATABASE \"$DB_NAME_TEST\";" - - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/00_dump.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/02_aux_functions.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/03_views.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/04_functions.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/05_stored_procs.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/demo_db.sql | grep . | uniq -c - + PGPASSWORD=YourStrong!Passw0rd psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/json_schema_extension.sql | grep . | uniq -c env: DB_HOST: localhost DB_PORT: 5432 DB_NAME: imis DB_USER: postgres - DB_PASSWORD: GitHub999 + DB_PASSWORD: YourStrong!Passw0rd + + - name: Django tests PSQL working-directory: ./openimis/openIMIS + # Run the tests regardless if previous steps failed (if setup fails the tests should crash instantly) if: success() || failure() run: | - export DB_NAME_TEST="test_$DB_NAME" - - export MODULES=$(jq -r '(.modules[].name)' ../openimis.json | xargs) - - # Remove some module from MODULES - MODULES=$(echo "$MODULES" | sed -E "s/\b$(echo "${CI_EXCLUDED_MODULE// /\\b|\\b}")\b/ /g" | xargs) - echo $MODULES - - DB_NAME=test_imis python manage.py migrate - # Generate sonar report + python -V export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')" - - # This has to exit 0, or report is not generated - python -m coverage run --omit=*/migrations/* ./manage.py test --keepdb $MODULE_NAME + python manage.py test --keepdb $MODULE_NAME env: SECRET_KEY: secret - DB_ENGINE: django.db.backends.postgresql + MODE: DEV + DB_DEFAULT: postgresql + CELERY_BROKER_URL: "memory://openIMIS-test//" + CELERY_RESULT_BACKEND: "cache+memory://openIMIS-test//" #DJANGO_SETTINGS_MODULE: hat.settings DB_HOST: localhost DB_PORT: 5432 DB_NAME: imis DB_USER: postgres - DB_PASSWORD: GitHub999 + DB_PASSWORD: YourStrong!Passw0rd #DEV_SERVER: true SITE_ROOT: api CI_EXCLUDED_MODULE: ${{ vars.CI_EXCLUDED_MODULE }} - MODE: DEV + OPENSEARCH_ADMIN: admin + OPENSEARCH_PASSWORD: B9wc9VrqX7pY + OPEN_SEARCH_HTTP_PORT: 9200 + OPENSEARCH_HOSTS: "localhost:9200" ci_module_psql_test: name: Run All Tests (PSQL) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 needs: build_backend services: pgsql: - image: postgres + image: ghcr.io/openimis/openimis-pgsql:develop env: DB_HOST: localhost DB_PORT: 5432 - POSTGRES_DB: imis + POSTGRES_DB: test_imis POSTGRES_USER: postgres - POSTGRES_PASSWORD: GitHub999 + POSTGRES_PASSWORD: YourStrong!Passw0rd ports: - 5432:5432 options: >- @@ -354,7 +272,7 @@ jobs: --health-retries 10 steps: - name: Set up Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.10' @@ -382,29 +300,20 @@ jobs: - name: Initialize PSQL run: | export DB_NAME_TEST="test_$DB_NAME" - if [ ${GITHUB_REF##*/} = "main" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database" git clone --depth=1 --branch=$DBBRANCH https://github.com/openimis/database_postgresql.git ./sql_psql - + sudo bash ./sql_psql/install_postgres_json_schema_extension.sh echo 'set search_path to public' >> ~/.psqlrc - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -c "DROP DATABASE IF EXISTS \"$DB_NAME_TEST\";" - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -c "CREATE DATABASE \"$DB_NAME_TEST\";" - - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/00_dump.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/02_aux_functions.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/03_views.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/04_functions.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/05_stored_procs.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/demo_db.sql | grep . | uniq -c - + PGPASSWORD=YourStrong!Passw0rd psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/json_schema_extension.sql | grep . | uniq -c env: DB_HOST: localhost DB_PORT: 5432 DB_NAME: imis DB_USER: postgres - DB_PASSWORD: GitHub999 - MODE: DEV + DB_PASSWORD: YourStrong!Passw0rd + + - name: Django tests PSQL working-directory: ./openimis/openIMIS # Run the tests regardless if previous steps failed (if setup fails the tests should crash instantly) @@ -412,29 +321,29 @@ jobs: run: | python -V export MODULES=$(jq -r '(.modules[].name)' ../openimis.json | xargs) - - # Remove some module from MODULES MODULES=$(echo "$MODULES" | sed -E "s/\b$(echo "${CI_EXCLUDED_MODULE// /\\b|\\b}")\b/ /g" | xargs) echo $MODULES - # Generate sonar report - export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')" - - # This has to exit 0, or report is not generated - DB_NAME=test_imis python manage.py migrate - python -m coverage run --source=$MODULE_NAME --omit=*/migrations/* ./manage.py test --keepdb $MODULES + python manage.py test --keepdb $MODULES env: SECRET_KEY: secret - DB_ENGINE: django.db.backends.postgresql + MODE: DEV + DB_DEFAULT: postgresql + CELERY_BROKER_URL: "memory://openIMIS-test//" + CELERY_RESULT_BACKEND: "cache+memory://openIMIS-test//" #DJANGO_SETTINGS_MODULE: hat.settings DB_HOST: localhost DB_PORT: 5432 DB_NAME: imis DB_USER: postgres - DB_PASSWORD: GitHub999 + DB_PASSWORD: YourStrong!Passw0rd #DEV_SERVER: true SITE_ROOT: api CI_EXCLUDED_MODULE: ${{ vars.CI_EXCLUDED_MODULE }} - MODE: DEV + OPENSEARCH_ADMIN: admin + OPENSEARCH_PASSWORD: B9wc9VrqX7pY + OPEN_SEARCH_HTTP_PORT: 9200 + OPENSEARCH_HOSTS: "localhost:9200" + OPENSEARCH_DSL_AUTOSYNC: False - name: Generate Coverage Report if: always() working-directory: ./openimis/openIMIS @@ -456,11 +365,11 @@ jobs: path: ./openimis/openIMIS/coverage flake-8-linter: name: Run flake8 check - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 needs: build_backend steps: - name: Set up Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.10' - name: Download site-packages artifact @@ -492,10 +401,10 @@ jobs: # Flake8 Report export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')" MOD_DIR="../current-module/$MODULE_NAME" - python -m flake8 --output-file=flake8-report.txt $MOD_DIR --exit-zero + python -m flake8 --output-file=flake8-report.txt $MOD_DIR --exit-zero --ignore W503 sed -i 's|\.\./current-module/||g' flake8-report.txt - python -m flake8 $MOD_DIR + python -m flake8 $MOD_DIR --ignore W503 - name: Flake8 results upload if: success() || failure() uses: actions/upload-artifact@v4 @@ -504,7 +413,7 @@ jobs: path: ./openimis/flake8-report.txt sonar_scan: name: Run Sonar Scan - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 needs: [ci_module_psql_test, flake-8-linter] if: always() env: diff --git a/.github/workflows/docker-manual.yml b/.github/workflows/docker-manual.yml index 4229b736..762a2508 100644 --- a/.github/workflows/docker-manual.yml +++ b/.github/workflows/docker-manual.yml @@ -12,7 +12,7 @@ jobs: publish-manual-docker-image: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Login to GitHub Container Registry uses: docker/login-action@v1 with: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8146ca1a..e1a62f53 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,7 +7,7 @@ jobs: publish-docker-develop-image: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: ref: develop - name: Login to GitHub Container Registry diff --git a/.github/workflows/pgsql.yml b/.github/workflows/pgsql.yml deleted file mode 100644 index 1ca06a11..00000000 --- a/.github/workflows/pgsql.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: Automated CI PSQL testing -# This workflow run automatically for every commit on github it checks the syntax and launch the tests. -# | grep . | uniq -c filters out empty lines and then groups consecutive lines together with the number of occurrences -on: - push: - workflow_dispatch: - inputs: - comment: - description: Just a simple comment to know the purpose of the manual build - required: false - -jobs: - build: - runs-on: ubuntu-20.04 - services: - pgsql: - image: postgres - env: - DB_HOST: localhost - DB_PORT: 5432 - POSTGRES_DB: imis - POSTGRES_USER: postgres - POSTGRES_PASSWORD: GitHub999 - ports: - - 5432:5432 - # needed because the mssql container does not provide a health check - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - python modules-requirements.py openimis.json > modules-requirements.txt - pip install -r modules-requirements.txt - - - - name: Environment info - run: | - pip list - - export DBBRANCH="$([ $GITHUB_REF == 'main' ] && echo "main" || echo "develop")" - - if [ ${GITHUB_REF##*/} = "develop" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi - echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database" - - git clone --depth 1 --branch $DBBRANCH https://github.com/openimis/database_postgresql.git ./sql - - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/postgresql-pgdg.list > /dev/null - sudo apt-get install postgresql-client - - - name: Initialize DB - run: | - echo 'set search_path to public' >> ~/.psqlrc - PGPASSWORD=GitHub999 psql -U postgres -h localhost -d imis -U postgres -f ./sql/database\ scripts/00_dump.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U postgres -h localhost -d imis -U postgres -f ./sql/database\ scripts/02_aux_functions.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U postgres -h localhost -d imis -U postgres -f ./sql/database\ scripts/03_views.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U postgres -h localhost -d imis -U postgres -f ./sql/database\ scripts/04_functions.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U postgres -h localhost -d imis -U postgres -f ./sql/database\ scripts/05_stored_procs.sql | grep . | uniq -c - PGPASSWORD=GitHub999 psql -U postgres -h localhost -d imis -U postgres -f ./sql/database\ scripts/demo_db.sql | grep . | uniq -c - - - name: Django tests - run: | - python -V - ls -l - cd openIMIS - ls -l - python manage.py test --keepdb $(jq -r '(.modules[]|.name)' ../openimis.json) - env: - SECRET_KEY: secret - DEBUG: true - DB_ENGINE: django.db.backends.postgresql - #DJANGO_SETTINGS_MODULE: hat.settings - DB_HOST: localhost - DB_PORT: 5432 - DB_NAME: imis - DB_USER: postgres - DB_PASSWORD: GitHub999 - #DEV_SERVER: true - SITE_ROOT: api diff --git a/.travis.yml b/.travis.yml index 3f8b05c8..dc1ee6bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,8 @@ before_install: - sudo apt-get install -y apt-transport-https ca-certificates # Necessary for some pip modules - sudo apt-get install -y rustc cargo -- ACCEPT_EULA=Y sudo apt-get install -y msodbcsql17 odbcinst mssql-tools mssql-cli -- sudo ln -s /opt/mssql-tools/bin/sqlcmd /usr/bin/sqlcmd +- ACCEPT_EULA=Y sudo apt-get install -y msodbcsql17 odbcinst mssql-tools18 mssql-cli +- sudo ln -s /opt/mssql-tools18/bin/sqlcmd /usr/bin/sqlcmd - sudo apt-get install -y python3-dev unixodbc-dev - "./script/launch-mssql-in-docker.sh -p $DB_PASSWORD" - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 diff --git a/.vscode/launch.json b/.vscode/launch.json index 03212c31..c995f29f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -21,11 +21,13 @@ "product", "insuree", "policy", + "calcrule_validations", "contribution", "payer", "payment", "claim", "claim_batch", + "claim_sampling", "tools", "api_fhir_r4", "calculation", @@ -46,14 +48,17 @@ "opensearch_reports", "payment_cycle", "calcrule_social_protection", - "payroll" + "payroll", + "grievance_social_protection", + "deduplication", + "api_etl" ] }, { "id": "dbEngine", "type": "pickString", "description": "provide DB engine to use", - "options": ["MSSQL", "PSQL"] + "options": ["mssql", "psql"] } ], "configurations": [ @@ -144,7 +149,7 @@ "type": "python", "request": "launch", "program": "${workspaceFolder}/openIMIS/manage.py", - "args": ["makemigrationsy"], + "args": ["makemigrations"], "django": true, "cwd": "${workspaceRoot}/openIMIS", "env": { @@ -176,21 +181,23 @@ "--timing", "core", "individual", - "controls", "workflow", "tasks_management", "report", "location", "medical", + "controls", "medical_pricelist", "product", "insuree", "policy", + "calcrule_validations", "contribution", "payer", "payment", "claim", "claim_batch", + "claim_sampling", "tools", "api_fhir_r4", "calculation", @@ -211,7 +218,10 @@ "opensearch_reports", "payment_cycle", "calcrule_social_protection", - "payroll" + "payroll", + "grievance_social_protection", + "deduplication", + "api_etl" ], "django": true, "cwd": "${workspaceRoot}/openIMIS", @@ -241,6 +251,27 @@ "CELERY_RESULT_BACKEND": "cache+memory://openIMIS-test//" }, "justMyCode": false + }, { + "name": "Test module (just my code)", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/openIMIS/manage.py", + "args": [ + "test", + "--keepdb", + "--debug-mode", + "--timing", + "${input:moduleName}" + ], + "django": true, + "cwd": "${workspaceRoot}/openIMIS", + "env": { + "DB_DEFAULT": "${input:dbEngine}", + //"ASYNC": "PROD", + "CELERY_BROKER_URL": "memory://openIMIS-test//", + "CELERY_RESULT_BACKEND": "cache+memory://openIMIS-test//" + }, + "justMyCode": true }, { "name": "Python: Celery Workers", diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 4f842cba..89e5c4e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,10 +9,10 @@ RUN apt-get -y install git RUN test "$DB_DEFAULT" != "postgresql" && curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - || : RUN test "$DB_DEFAULT" != "postgresql" && curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list || : RUN test "$DB_DEFAULT" != "postgresql" && apt-get update || : -RUN test "$DB_DEFAULT" != "postgresql" && ACCEPT_EULA=Y apt-get install -y msodbcsql17 mssql-tools || : +RUN test "$DB_DEFAULT" != "postgresql" && ACCEPT_EULA=Y apt-get install -y msodbcsql17 mssql-tools18 || : RUN pip install --upgrade pip -RUN test "$DB_DEFAULT" != "postgresql" && pip install mssql-cli || : +#RUN test "$DB_DEFAULT" != "postgresql" && pip install mssql-cli || : RUN pip install gunicorn diff --git a/README.md b/README.md index 6b4d9f27..ae36ef00 100644 --- a/README.md +++ b/README.md @@ -1,74 +1,72 @@ # Environment Variables -| ENV | Values | Description | +| ENV | Values | Description | | --------------------------- | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| MODE | DEV, PROD | This is the mode of running the application. There are 2 modes available. DEV for the Development mode and PROD for the production mode. Certain settings will be changed according to the mode. Such as in the PROD mode, mutation will run asynchronously and synchronously otherwise. Same applies to DEBUG, it will be OFF in PROD and TRUE otherwise. | -| DB_ENGINE | django.db.backends.postgresql, mssql | Currently openIMIS supports 2 databases, as the values suggested, postgres and mssql. | -| DEMO_DATASET | true | Define if the database should be initiated with demo dataset. Comment for empty database. | -| DB_DEFAULT | String | String This variables sets the default database engine for the system. Possible values: postgresql, mssql. Default: postgresql. | -| DB_HOST | String | Define the name of your database server | -| DB_PORT | Integer | Define the port on which your database accepts the connection | -| DB_NAME | String | Define the name of the openIMIS database | -| DB_USER | String | Configure the username with which you want to connect to the database | -| DB_PASSWORD | String | Configure the database password | -| DB_TEST_NAME | String | If you are developing unit tests then this setting will create the testing database as per the name set | -| NO_DATABASE | true, false | If set to true, it will use sqlite.db3 as a database | -| DB_OPTIONS | String | Define any additional database options | -| PSQL_DB_ENGINE | String | Define a library to use to connect with postgres database. Default value is django.db.backends.postgresql | -| MSSQL_DB_ENGINE | String | Define a library to use to connect with MS SQL database Default value is mssql. | -| SITE_ROOT | String | Site root that will prefix all exposed endpoints. It's required when working with openIMIS frontend. For example, if the value is set `api` then the endpoint will appear like `your_domain_name/api/xxx` | -| DJANGO_LOG_LEVEL | INFO, WARNING, ERROR, DEBUG | Define the level of logs | -| DJANGO_LOG_HANDLER | console, debug-log | Depending on the value set, application will print the logs | -| DJANGO_DB_LOG_HANDLER | console, debug-log | Depending on the value set, application will print the logs | -| PHOTO_ROOT_PATH | String | Define the path for the photos of insurees. This setting is used in the Insuree module. The value set here will be overwritten by the InsureeConfig file. | -| DJANGO_MIGRATE | True, False | Based on the value set, application runs the migration command before starting up. If the SITE_ROOT value is set to api then the migration will always run regardless of the value | -| SCHEDULER_AUTOSTART | True, False | All the modules will be searched for the scheduled tasks, if the value is set to True | -| OPENSEARCH_HOST | String | Define the opensearch host | -| OPENSEARCH_ADMIN | String | Define the login name for open search | -| OPENSEARCH_PASSWORD | String | Define the admin password to login to open search | -| BE_BRANCH | String | Define the github branch for the Backend form which you wan to install the module. Default is develop. | -| FE_BRANCH | String | Define the github branch for the Front-end form which you wan to install the module. Default is develop. | -| DB_BRANCH | String | Define the github branch for the Database form which you wan to install the module. Default is develop. | -| ALLOWED_HOST | Comma separated String | Define the list of allowed hosts such as IP addresses or Domain names to access the application. If the value is not set it will allow all the IP addresses. | -| LOKALISE_APIKEY | String | Set the lokalise api key. Obtain this key form the lokalise project to be able to use the lokalise-upload | -| OPENIMIS_CONF_JSON | String | Define the path for the openimis config file. If not set the default config from the root folder will be taken. | -| DB_QUERIES_LOG_FILE | String | Define the path of the file to save the database queries. Default is db-queries.log | -| DEBUG_LOG_FILE | String | Define the path of the file to save the debug log. Default is debug.log | -| SENTRY_DSN | String | Set the unique Sentry DSN. This can be obtained from your Sentry account dashboard | -| SENTRY_SAMPLE_RATE | 0-1 | This configuration allows you to to control the rate at which traces are collected. Values are between 0 and 1. 0 means no traces will be collected (tracing is disabled). 1 means traces will be collected for every request. Any value between 0 and 1 represents the probability of capturing trace. For instance, a value 0.3 means that approximately 30% of requests will have traces collected. | -| IS_SENTRY_ENABLED | True, False | Defines if the Sentry error tracking and monitoring functionality is enabled or disabled. | -| SITE_URL | String | Define the base url. This is used to create links in FHIR module | -| SECRET_KEY | String | This is used internally by Django. Make sure to set it up in production server. | -| REMOTE_USER_AUTHENTICATION | true, false | Set it to true if you want to enable the RemoteUserBackend as an authentication backend in Django. By default it's false | -| CELERY_BROKER_URL | String | Define a message broker url for celery. Default value is amqp://rabitmq | -| CHANNELS_HOST | String | Set the host for the Django Channel. Default value is amqp://guest:guest@127.0.0.1/ | -| EMAIL_HOST | String | Define an Email Host. Default value is localhost | -| EMAIL_PORT | Integer | Define a valid port for the email server | -| EMAIL_HOST_USER | String | Set the username to send emails | -| EMAIL_HOST_PASSWORD | String | Set the password for the email | -| EMAIL_USE_TLS | True, False | Configure the TLS setting. Default value is False | -| EMAIL_USE_SSL | True, False | Configure the SSL settings for the emails. Default value is False | -| DATA_UPLOAD_MAX_MEMORY_SIZE | Integer | Define the upload size allowed in **bytes** via the POST request. Default is 10 MB | -| INSUREE_NUMBER_LENGTH | Integer | This configuration is used for Insuree module. Define the allowed length of the insurance number | -| INSUREE_NUMBER_MODULE_ROOT | Integer | Define the module to use for the last number of the insurance number | -| MASTER_DATA_PASSWORD | String | This setting is used in exporting entities. Configure the password to zip the exported entities. | -| FRONTEND_URL | String | Define the URL to access the front-end | -| DJANGO_SETTINGS_MODULE | String | Define the python import path to settings module for Django project. By default it is set to openIMIS.settings | -| OPEHNHIM_URL | String | This setting is used in fhir module. Define the url for openHIM | -| OPEHNHIM_USER | String | This setting is used in fhir module. Define the user for openHIM | -| OPEHNHIM_PASSWORD | String | This setting is used in fhir module. Define the password for openHIM | -| OPENIMIS_CONF | String | Define a path to the config file. By default it reads from ../openimis.json | -| CLAIMDOC_TOKEN | String | Used in backend caching. Define a token to communicate with the remote server. Default is set to 'TestToken' | -| CACHE_BACKEND | String | Specifies the [caching backend](https://docs.djangoproject.com/en/5.0/topics/cache/#setting-up-the-cache) to be used. Default is set to PyMemcached. | -| CACHE_URL | String | Defines the location of the cache backend. Default is `unix:/tmp/memcached.sock` for a Unix socket connection. | -| CACHE_OPTIONS | String | A JSON string representing a dictionary of additional options passed to the cache backend. Empty by default | -| RATELIMIT_CACHE | String | The cache alias to use for rate limiting. Defaults to `default`. | -| RATELIMIT_KEY | String | Key to identify the client for rate limiting; `ip` means it will use the client's IP address. Defaults to `ip`. | -| RATELIMIT_RATE | String | Rate limit value (e.g., `150/m` for 150 requests per minute). Defaults to `150/m`. | -| RATELIMIT_METHOD | String | HTTP methods to rate limit; `ALL` means all methods. Defaults to `ALL`. | -| RATELIMIT_GROUP | String | Group name for the rate limit. Defaults to `graphql`. | -| RATELIMIT_SKIP_TIMEOUT | Boolean | Whether to skip rate limiting during cache timeout. Defaults to `False`. | -| CSRF_TRUSTED_ORIGINS | String | Define the trusted origins for CSRF protection, separated by commas. Defaults to `http://localhost:3000,http://localhost:8000`. | +| MODE | DEV, PROD | This is the mode of running the application. There are 2 modes available. DEV for the Development mode and PROD for the production mode. Certain settings will be changed according to the mode. Such as in the PROD mode, mutation will run asynchronously and synchronously otherwise. Same applies to DEBUG, it will be OFF in PROD and TRUE otherwise. | +| DB_ENGINE | django.db.backends.postgresql, mssql | Currently openIMIS supports 2 databases, as the values suggested, postgres and mssql. | +| DEMO_DATASET | true | Define if the database should be initiated with demo dataset. Comment for empty database. | +| DB_DEFAULT | String | String This variables sets the default database engine for the system. Possible values: postgresql, mssql. Default: postgresql. | +| DB_HOST | String | Define the name of your database server | +| DB_PORT | Integer | Define the port on which your database accepts the connection | +| DB_NAME | String | Define the name of the openIMIS database | +| DB_USER | String | Configure the username with which you want to connect to the database | +| DB_PASSWORD | String | Configure the database password | +| DB_TEST_NAME | String | If you are developing unit tests then this setting will create the testing database as per the name set| +| NO_DATABASE | true, false | If set to true, it will use sqlite.db3 as a database | +| DB_OPTIONS | String | Define any additional database options | +| PSQL_DB_ENGINE | String | Define a library to use to connect with postgres database. Default value is django.db.backends.postgresql | +| MSSQL_DB_ENGINE | String | Define a library to use to connect with MS SQL database Default value is mssql. | +| SITE_ROOT | String | Site root that will prefix all exposed endpoints. It's required when working with openIMIS frontend. For example, if the value is set `api` then the endpoint will appear like `your_domain_name/api/xxx` | +| DJANGO_LOG_LEVEL | INFO, WARNING, ERROR, DEBUG | Define the level of logs | +| DJANGO_LOG_HANDLER | console, debug-log | Depending on the value set, application will print the logs | +| DJANGO_DB_LOG_HANDLER | console, debug-log | Depending on the value set, application will print the logs | +| PHOTO_ROOT_PATH | String | Define the path for the photos of insurees. This setting is used in the Insuree module. The value set here will be overwritten by the InsureeConfig file. | +| DJANGO_MIGRATE | True, False | Based on the value set, application runs the migration command before starting up. If the SITE_ROOT value is set to api then the migration will always run regardless of the value | +| SCHEDULER_AUTOSTART | True, False | All the modules will be searched for the scheduled tasks, if the value is set to True | +| OPENSEARCH_HOSTS | String | Define the opensearch hosts, comma separated http://opensearch:9200 | +| OPENSEARCH_ADMIN | String | Define the login name for open search | +| OPENSEARCH_PASSWORD | String | Define the admin password to login to open search | +| BE_BRANCH | String | Define the github branch for the Backend form which you wan to install the module. Default is develop. | +| FE_BRANCH | String | Define the github branch for the Front-end form which you wan to install the module. Default is develop. | +| DB_BRANCH | String | Define the github branch for the Database form which you wan to install the module. Default is develop.| | +| LOKALISE_APIKEY | String | Set the lokalise api key. Obtain this key form the lokalise project to be able to use the lokalise-upload | +| OPENIMIS_CONF_JSON | String | Define the path for the openimis config file. If not set the default config from the root folder will be taken. | +| DB_QUERIES_LOG_FILE | String | Define the path of the file to save the database queries. Default is db-queries.log | +| DEBUG_LOG_FILE | String | Define the path of the file to save the debug log. Default is debug.log | +| SENTRY_DSN | String | Set the unique Sentry DSN. This can be obtained from your Sentry account dashboard | +| SENTRY_SAMPLE_RATE | 0-1 | This configuration allows you to to control the rate at which traces are collected. Values are between 0 and 1. 0 means no traces will be collected (tracing is disabled). 1 means traces will be collected for every request. Any value between 0 and 1 represents the probability of capturing trace. For instance, a value 0.3 means that approximately 30% of requests will have traces collected. | +| IS_SENTRY_ENABLED | True, False | Defines if the Sentry error tracking and monitoring functionality is enabled or disabled. | +| SITE_URL | String | Define the base url. This is used to create links in FHIR module | +| SITE_FRONT | String | Define base uri for the frontend| +| FRONTEND_URL | String | Define the frontend URL if not aligned with SITE_URL/SITE_FRONT| +| SECRET_KEY | String | This is used internally by Django. Make sure to set it up in production server. | +| REMOTE_USER_AUTHENTICATION | true, false | Set it to true if you want to enable the RemoteUserBackend as an authentication backend in Django. By default it's false | +| CELERY_BROKER_URL | String | Define a message broker url for celery. Default value is amqp://rabitmq | +| CHANNELS_HOST | String | Set the host for the Django Channel. Default value is amqp://guest:guest@127.0.0.1/ | +| EMAIL_HOST | String | Define an Email Host. Default value is localhost | +| EMAIL_PORT | Integer | Define a valid port for the email server | +| EMAIL_HOST_USER | String | Set the username to send emails | +| EMAIL_HOST_PASSWORD | String | Set the password for the email | +| EMAIL_USE_TLS | True, False | Configure the TLS setting. Default value is False | +| EMAIL_USE_SSL | True, False | Configure the SSL settings for the emails. Default value is False | +| DATA_UPLOAD_MAX_MEMORY_SIZE | Integer | Define the upload size allowed in **bytes** via the POST request. Default is 10 MB | +| MASTER_DATA_PASSWORD | String | This setting is used in exporting entities. Configure the password to zip the exported entities. | +| DJANGO_SETTINGS_MODULE | String | Define the python import path to settings module for Django project. By default it is set to openIMIS.settings | +| OPEHNHIM_URL | String | This setting is used in fhir module. Define the url for openHIM | +| OPEHNHIM_USER | String | This setting is used in fhir module. Define the user for openHIM | +| OPEHNHIM_PASSWORD | String | This setting is used in fhir module. Define the password for openHIM | +| OPENIMIS_CONF | String | Define a path to the config file. By default it reads from ../openimis.json| +| CLAIMDOC_TOKEN | String | Used in backend caching. Define a token to communicate with the remote server. Default is set to 'TestToken' | +| CACHE_BACKEND | String | Specifies the [caching backend](https://docs.djangoproject.com/en/5.0/topics/cache/#setting-up-the-cache) to be used. Default is set to PyMemcached. | +| CACHE_URL | String | Defines the location of the cache backend. Default is `unix:/tmp/memcached.sock` for a Unix socket connection. | +| CACHE_OPTIONS | String | A JSON string representing a dictionary of additional options passed to the cache backend. Empty by default | +| RATELIMIT_CACHE | String | The cache alias to use for rate limiting. Defaults to `default`. | +| RATELIMIT_KEY | String | Key to identify the client for rate limiting; `ip` means it will use the client's IP address. Defaults to `ip`. | +| RATELIMIT_RATE | String | Rate limit value (e.g., `150/m` for 150 requests per minute). Defaults to `150/m`. | +| RATELIMIT_METHOD | String | HTTP methods to rate limit; `ALL` means all methods. Defaults to `ALL`. | +| RATELIMIT_GROUP | String | Group name for the rate limit. Defaults to `graphql`. | +| RATELIMIT_SKIP_TIMEOUT | Boolean | Whether to skip rate limiting during cache timeout. Defaults to `False`. | +| HOSTS | String | Define the trusted domains that are used for PROD CORS and CSRF protection, separated by commas. Defaults to `localhost, 192.168.0.1`. | ## Developers setup @@ -80,61 +78,59 @@ When programming for openIMIS backend, you are highly encouraged to use the feat - install python 3, recommended in a [venv](https://docs.python.org/3/library/venv.html) or [virtualenv](https://virtualenv.pypa.io) - install [pip](https://pip.pypa.io) - within `openimis-be_py` directory - - install openIMIS (external) dependencies: `pip install -r -requirements.txt`. For development workstations, one can use `pip -install -r dev-requirements.txt` instead, for more modules. - - generate the openIMIS modules dependencies file (from openimis.json config): `python modules-requirements.py openimis.json > modules-requirements.txt` - - if you previously installed openIMIS on another version, it seems safe to uninstall all previous modules-requirement to be sure it match current version `pip uninstall -r modules-requirements.txt` - - install openIMIS current modules: `pip install -r modules-requirements.txt` - - Copy the example environment setup and adjust the settings (like database connection): `cp .env.example .env`. - Refer to .env.example or the Environment Variable tables above for more info. + - install openIMIS (external) dependencies: `pip install -r requirements.txt`. For development workstations, one can use `pip install -r dev-requirements.txt` instead, for more modules. + - In the script directory, + - generate the openIMIS modules dependencies file (from openimis.json config): `python modules-requirements.py openimis.json > modules-requirements.txt` + - if you previously installed openIMIS on another version, it seems safe to uninstall all previous modules-requirement to be sure it match current version `pip uninstall -r modules-requirements.txt` + - install openIMIS current modules: `pip install -r modules-requirements.txt` + - Copy the example environment setup and adjust the settings (like database connection): `cp .env.example .env`. + Refer to .env.example or the Environment Variable tables above for more info. - start openIMIS from within `openimis-be_py/openIMIS`: `python manage.py runserver` At this stage, you may (depends on the database you connect to) need to: * apply django migrations, from `openimis-be_py/openIMIS`: `python manage.py migrate`. See [PostgresQL section](#postgresql) if you are using postgresql for dev DB. * create a superuser for django admin console, from `openimis-be_py/openIMIS`: `python manage.py createsuperuser` (will - not prompt for a password) and then `python manage.py changepassword -` + not prompt for a password) and then `python manage.py changepassword ` ### To edit (modify) an existing openIMIS module (e.g. `openimis-be-claim`) - checkout the module's git repo NEXT TO (not within!) `openimis-be_py` directory and create a git branch for your changes - from `openimis-be_py` - - uninstall the packaged module you want to work on (example: openimis-be-claim): `pip uninstall openimis-be-claim` - - install the 'local' version of the module: `pip install -e ../openimis-be-claim_py/` + - uninstall the packaged module you want to work on (example: openimis-be-claim): `pip uninstall openimis-be-claim` + - install the 'local' version of the module: `pip install -e ../openimis-be-claim_py/` - from here on, openIMIS is using the local content of the module (with live update) ### To create a new openIMIS module (e.g. `openimis-be-mymodule`) - create a (git-enabled) directory next to the other modules, with a subdirectory named as your module 'logical' name: `/openimis-be-mymodule_py/mymodule` - from `/openimis-be_py/openIMIS`: - - create the module skeleton: `python manage.py startapp mymodule ../../openimis-be-mymodule_py/mymodule` - - prepare your module to be mounted via pip: create and complete the `/openimis-be-mymodule_py/setup.py` (and README.md,... files) - - every openIMIS module must provide its urlpatterns (even if empty): - - create the file `/openimis-be-mymodule_py/mymodule/urls.py` - - with content: `urlpatterns = []` - - register your module in the pip requirements of openIMIS, referencing your 'local' codebase: `pip install -e ../../openimis-be-mymodule_py/` - - register your module to openIMIS django site in `/openimis-be_py/openimis.json` + - create the module skeleton: `python manage.py startapp mymodule ../../openimis-be-mymodule_py/mymodule` + - prepare your module to be mounted via pip: create and complete the `/openimis-be-mymodule_py/setup.py` (and README.md,... files) + - every openIMIS module must provide its urlpatterns (even if empty): + - create the file `/openimis-be-mymodule_py/mymodule/urls.py` + - with content: `urlpatterns = []` + - register your module in the pip requirements of openIMIS, referencing your 'local' codebase: `pip install -e ../../openimis-be-mymodule_py/` + - register your module to openIMIS django site in `/openimis-be_py/openimis.json` - from here on, your local openIMIS has a new module, directly loaded from your directory. ### To create a distinct implementation of an existing openIMIS module (e.g. `openimis-be-location-dhis2`) - from `openimis-be_py`, uninstall the packaged module you want to replace: `pip uninstall openimis-be-location` - follow the same procedure as for a brand new openIMIS module, - ... but give it the same logical name as the one you want to replace: `/openimis-be-location-dhis2_py/location` + ... but give it the same logical name as the one you want to replace: `/openimis-be-location-dhis2_py/location` ### To manage translations of your module - from your module root dir, execute '../openimis-be_py/script/gettext.sh' - ... this extract all your translations keys from your code into your module root dir/locale/en/LC_MESSAGES/django.po + ... this extract all your translations keys from your code into your module root dir/locale/en/LC_MESSAGES/django.po - you may want to provide translation in generated django.po file... or manage them via lokalize (need to upload the keys,...) ### To run unit tests on a module (example openimis-be-claim) - from `openimis-be_py` - - (re)initialize test database (at this stage structure is not managed by django): - - launch unit tests, with the 'keep database' option: `python + - (re)initialize test database (at this stage structure is not managed by django): + - launch unit tests, with the 'keep database' option: `python manage.py test --keep claim` ### To get profiler report (DEBUG mode only) @@ -146,7 +142,7 @@ In request query include additional parameters: #### Example: -`http://localhost:8000/api/graphql?prof=True&download=True` +`http://localhost:8000/api/graphql?prof=True&download=True` creates profiler report for execution of query/mutation defined in request's POST body. ### To publish (in PyPI) the modified (or new) module @@ -154,8 +150,8 @@ creates profiler report for execution of query/mutation defined in request's POS - adapt the `openimis-be-mymodule_py/setup.py` to (at least) bump version number (e.g. 1.2.3) - commit your changes to the git repo and merge into master - tag the git repo according to your new version number: - - `git tag -a v1.2.3 -m "v1.2.3"` - - `git push --tags` + - `git tag -a v1.2.3 -m "v1.2.3"` + - `git push --tags` - create the PyPI package (can be automated on a ci-build): `python setup.py bdist_wheel` - upload the created package (in `dist/`) to PyPI.org: `twine upload -r pypi dist/openimis_be_mymodule-1.2.3*` @@ -168,12 +164,12 @@ Note: as a distributor, you may want to run an openIMIS version without docker. - clone this repo (creates the `openimis-be_py` directory) and create a git branch (named according to the release you want to bundle) - adapt the `openimis-be_py/openimis.json` to specify the modules (and their versions) to be bundled - make release candidates docker image from `openimis-be_py/`: `docker build . -t openimis-be-2.3.4 [--build-arg="DB_DEFAULT=postgresql"]` - - change the version (openimis-be-2.3.4) to the actual version you want to build - - if only postgresql database is used, include the build-arg argument + - change the version (openimis-be-2.3.4) to the actual version you want to build + - if only postgresql database is used, include the build-arg argument - configure the database connection (see section here below) - run the docker image, referring to environment variables file: `docker run --env-file .env openimis-be-2.3.4` - Note: when starting, the docker image will automatically apply the necessary database migrations to - the database + Note: when starting, the docker image will automatically apply the necessary database migrations to + the database When release candidate is accepted: @@ -185,11 +181,11 @@ When release candidate is accepted: - clone this repo (creates the `openimis-be_py` directory) and create a git branch (named according to the release you want to bundle) - adapt the `openimis-be_py/openimis.json` to specify the modules (and their versions) to be bundled, the "pip" params can be: - - standard pip: `openimis-be-core==1.2.0rc1` - - from local: `-e ../openimis-be-core_py` - - from git: `git+https://github.com/openimis/openimis-be-core_py.git@develop` - - the egg can be specified so pip know what to look `git+https://github.com/openimis/openimis-be-core_py.git@develop#egg=openimis-be-core` - - from tarball: `https://github.com/openimis/openimis-be_py/archive/v1.1.0.tar.gz` + - standard pip: `openimis-be-core==1.2.0rc1` + - from local: `-e ../openimis-be-core_py` + - from git: `git+https://github.com/openimis/openimis-be-core_py.git@develop` + - the egg can be specified so pip know what to look `git+https://github.com/openimis/openimis-be-core_py.git@develop#egg=openimis-be-core` + - from tarball: `https://github.com/openimis/openimis-be_py/archive/v1.1.0.tar.gz` - (required only once)`python -m venv ./venv`: create the python venv - `./venv/Script/activate[.sh/.ps1]`: Activate the venv - `python script/modules-requirements.py openimis.json > modules-requirements.txt`: list the source of the module to install @@ -203,24 +199,24 @@ When release candidate is accepted: The configuration for connection to the database is identical for developers and distributors: - By default, openIMIS is connected to MS-SQL Server: - - via ODBC (and pyodbc) driver - - using TCP/IP protocol (with server DNS name as hostname... or localhost) and fixed port (leave `DB_PORT` here below empty for dynamic port) - - SQL Server (not Windows/AD) authentication (user name password managed in SQL Server admin) - Download and install the ODBC that correspond to your OS and MS-SQL Server version (https://docs.microsoft.com/en-us/sql/connect/odbc/) + - via ODBC (and pyodbc) driver + - using TCP/IP protocol (with server DNS name as hostname... or localhost) and fixed port (leave `DB_PORT` here below empty for dynamic port) + - SQL Server (not Windows/AD) authentication (user name password managed in SQL Server admin) + Download and install the ODBC that correspond to your OS and MS-SQL Server version (https://docs.microsoft.com/en-us/sql/connect/odbc/) - Copy the example environment setup and adjust the variables: `cp .env.example .env`. Refer to .env.example for more info. - ``` - DB_HOST=mssql-host-server - DB_PORT=mssql-port - DB_NAME=database-name - DB_USER=database-user - DB_PASSWORD=database-password - ``` - Notes: + ``` + DB_HOST=mssql-host-server + DB_PORT=mssql-port + DB_NAME=database-name + DB_USER=database-user + DB_PASSWORD=database-password + ``` + Notes: - instead of `.env` file, you can use environment variables (e.g. provided as parameters in the docker-compose.yml) - default used django database 'engine' in openIMIS is - `sql_server.pyodbc`. If you need to use another one, use the `DB_ENGINE` entry in the `.env` file + `sql_server.pyodbc`. If you need to use another one, use the `DB_ENGINE` entry in the `.env` file - default 'options' in openIMIS are `{'driver': 'ODBC Driver 17 for SQL Server','unicode_results': True}` - If you need to provide other options, use the `DB_OPTIONS` entry in the `.env` file (be complete: the new json string will entirely replace the default one) + If you need to provide other options, use the `DB_OPTIONS` entry in the `.env` file (be complete: the new json string will entirely replace the default one) ### PostgresQL @@ -232,13 +228,13 @@ To use postgresql as the dev database, specify `DB_ENGINE` in `.env` alongside o DB_ENGINE=django.db.backends.postgresql DB_HOST=localhost DB_PORT=5432 -DB_NAME=coremis +DB_NAME=imis DB_USER=postgres ``` -Create the database, named `coremis` using the example `DB_NAME` above: +Create the database, named `imis` using the example `DB_NAME` above: ```bash -psql postgres -c 'create database coremis' +psql postgres -c 'create database imis' ``` Before applying django migrations, the database needs to be prepared using scripts from https://github.com/openimis/database_postgresql: @@ -251,11 +247,18 @@ cd database_postgresql bash concatenate_files.sh # prepare the database - replace fullDemoDatabase.sql with EmptyDatabase.sql if you don't need demo data -psql -d coremis -a -f output/fullDemoDatabase.sql +psql -d imis -a -f output/fullDemoDatabase.sql ``` From here on django's `python manage.py migrate` should execute succesfully. +Repeat the same steps for `test_imis`, which is used for unit tests: + +```bash +psql postgres -c 'create database test_imis' +psql -d test_imis -a -f output/fullDemoDatabase.sql +``` + ## OpenSearch @@ -281,122 +284,122 @@ provided in the README section. ### To create backend module skeleton in single command - from `/openimis-be_py/openIMIS`: - - run this command: `python manage.py create_openimis_module [--template