diff --git a/.github/workflows/build-and-deploy.yaml b/.github/workflows/build-and-deploy.yaml index 9f706dc..75cb898 100644 --- a/.github/workflows/build-and-deploy.yaml +++ b/.github/workflows/build-and-deploy.yaml @@ -6,7 +6,8 @@ jobs: runs-on: ubuntu-latest # CI/CD will run on these branches if: > - github.ref == 'refs/heads/master' || + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/development' strategy: matrix: @@ -27,7 +28,7 @@ jobs: baseRef="${GITHUB_REF#*/}" baseRef="${baseRef#*/}" case "${baseRef}" in - master) + main) image_tag="latest" ;; *) diff --git a/README.md b/README.md index 652f8ef..5febf89 100644 --- a/README.md +++ b/README.md @@ -82,11 +82,14 @@ Open `http://localhost:8000` to view it in the browser Though probably not necessary in most cases, if you want to test/modify/emulate how this will run in production you can: - In `.env` set: -``` + +```shell DEBUG = 'False' ``` + - Modify the app container's command in `docker-compose.yaml` to run: -``` + +```shell gunicorn signal_documentation.wsgi:application --bind 0.0.0.0:8000" *(Essentially you'll replace just the last line of the command, switching out the "runserver" line) @@ -96,6 +99,10 @@ Open `http://localhost` to view it in the browser. In this usage your request wi The primary use case for this will be when making changes to the Nginx container image that runs in production and hosts the static file content, or also if making changes to the Gunicorn config. +Additionally, though again not required for local development, you can also specify an env var of `MAIN_PAGE = $name`, and the app will be served at `http://localhost:8000/$name` (if running in debug mode), or if you've set `DEBUG = 'False'` to run it in Nginx/production mode at `http://localhost/$name/`. Note the ending slash when in Nginx/production mode _and_ using the `MAIN_PAGE` env var. + +The primary use case is so that we have flexibility to serve the application at something other than the "bare" URL, though doing this is not necessary for local development. + Changes of this sort should be carefully evaluated as they may require interaction with systems managed by devops folks. ## [Django admin](https://docs.djangoproject.com/en/4.1/ref/contrib/admin/) web interface (user should be `is_staff` or `is_superuser`) @@ -200,9 +207,15 @@ Each environment is essentially a bunch of different services all governed by `d ### Basic workflow -- A PR merged to either `development` or `master` will trigger CI to build container images that are then tagged (based on the branch name and ":latest" respectively) and stored in our GitHub Packages container image repository. +- A PR merged to either `development`, `staging`, or `main` will trigger CI to build container images that are then tagged with the branch name (or ":latest", in the cast of `main`), and stored in our GitHub Packages container image repository. - CI triggers a webhook that tells the host systems to pull and run new container images and restart any services that have been updated. +As a developer, your path to getting changes into production should be something like this: + +- Source your working branch from `development`, do work, PR and merge when complete +- PR and merge to `staging` in order to get your changes deployed to https://staging.delphi.cmu.edu/signals for review +- PR and merge to `main` to go to production + **IMPORTANT!** - The CI/CD process uses Docker Compose to build the specific container images that will be used in external environments. Success of the the build-and-deploy workflow is dependent on constructed services in `docker-compose.yaml`. If considering making changes there, please have a PR reviewed by devops folks :pray: :pray: :pray: ### Control of the deployed environment diff --git a/docker-compose.yaml b/docker-compose.yaml index 6c2c5de..196007b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -31,15 +31,7 @@ services: restart: on-failure command: sh -c "python3 /usr/src/signal_documentation/src/manage.py migrate --noinput && python3 /usr/src/signal_documentation/src/manage.py collectstatic --noinput && - python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/available_geography.json && - python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/pathogens.json && - python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/signal_types.json && - python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/signal_categories.json && - python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/county.json && - python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/hhs.json && - python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/hrr.json && - python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/msa.json && - python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/state.json && + python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/* && python3 /usr/src/signal_documentation/src/manage.py runserver 0.0.0.0:8000" volumes: - .:/usr/src/signal_documentation diff --git a/nginx/default.conf.template b/nginx/default.conf.template index fef97cc..24a6b99 100644 --- a/nginx/default.conf.template +++ b/nginx/default.conf.template @@ -1,10 +1,9 @@ server { listen 80; server_name sdnginx; - - location /static/ { + location ~* /static/(.*)$ { autoindex on; - alias /staticfiles/; + alias /staticfiles/$1; } location / { diff --git a/src/base/admin.py b/src/base/admin.py index e9e5d33..dcf1cf8 100644 --- a/src/base/admin.py +++ b/src/base/admin.py @@ -6,12 +6,13 @@ DescriptedFilter, DescriptedFilterField, Link, + License ) class DescriptedFilterFieldInline(admin.TabularInline): model = DescriptedFilterField - fields = ('description',) + fields = ('description', 'filter_field') extra = 0 can_create = False @@ -27,3 +28,12 @@ class LinkAdmin(admin.ModelAdmin): Admin interface for managing link objects. """ list_display: tuple[Literal['url'], Literal['link_type']] = ('url', 'link_type') + + +@admin.register(License) +class GeographyAdmin(admin.ModelAdmin): + """ + Admin interface for managing license objects. + """ + list_display: tuple[Literal['name'], Literal['use_restrictions']] = ('name', 'use_restrictions') + search_fields: tuple[Literal['name']] = ('name',) diff --git a/src/base/migrations/0004_license.py b/src/base/migrations/0004_license.py new file mode 100644 index 0000000..405857e --- /dev/null +++ b/src/base/migrations/0004_license.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.10 on 2024-06-07 12:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0003_descriptedfilter_alter_link_link_type_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='License', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='License', max_length=256, unique=True)), + ('use_restrictions', models.TextField(blank=True, help_text='Use Restrictions', null=True)), + ], + ), + ] diff --git a/src/base/models.py b/src/base/models.py index 96003c7..52c16dc 100644 --- a/src/base/models.py +++ b/src/base/models.py @@ -123,3 +123,20 @@ def get_preview(self) -> LinkPreview: return { 'description': _('No description available'), } + + +class License(models.Model): + """ + A model representing a License. + """ + name: models.CharField = models.CharField(help_text=_('License'), max_length=256, unique=True) + use_restrictions: models.TextField = models.TextField(help_text=_('Use Restrictions'), blank=True, null=True) + + def __str__(self) -> str: + """ + Returns the name of the license as a string. + + :return: The name of the license as a string. + :rtype: str + """ + return self.name diff --git a/src/datasources/admin.py b/src/datasources/admin.py index 4874cd7..2467e68 100644 --- a/src/datasources/admin.py +++ b/src/datasources/admin.py @@ -12,14 +12,11 @@ class SourceSubdivisionAdmin(ImportExportModelAdmin): """ Admin interface for managing source subdivision objects. """ - list_display: tuple[Literal['name'], Literal['db_source']] = ('name', 'db_source') + list_display: tuple[Literal['name'], Literal['db_source'], Literal['external_name']] = ('name', 'db_source', 'external_name') search_fields: tuple[Literal['name'], Literal['db_source']] = ('name', 'db_source') resource_classes: list[type[SourceSubdivisionResource]] = [SourceSubdivisionResource] -data_source_search_fields_type = tuple[Literal['name'], Literal['source_subdivision__db_source'], Literal['source_subdivision__name'], Literal['description']] - - @admin.register(DataSource) class DataSourceAdmin(ImportExportModelAdmin): """ diff --git a/src/datasources/migrations/0005_sourcesubdivision_external_name.py b/src/datasources/migrations/0005_sourcesubdivision_external_name.py new file mode 100644 index 0000000..5c7bc4a --- /dev/null +++ b/src/datasources/migrations/0005_sourcesubdivision_external_name.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.3 on 2024-06-05 10:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('datasources', '0004_alter_datasource_options_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='sourcesubdivision', + name='external_name', + field=models.CharField(help_text='External Name', max_length=128, null=True), + ), + ] diff --git a/src/datasources/migrations/0006_alter_datasource_source_license.py b/src/datasources/migrations/0006_alter_datasource_source_license.py new file mode 100644 index 0000000..7f6f41a --- /dev/null +++ b/src/datasources/migrations/0006_alter_datasource_source_license.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.10 on 2024-06-07 12:15 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0004_license'), + ('datasources', '0005_sourcesubdivision_external_name'), + ] + + operations = [ + migrations.AlterField( + model_name='datasource', + name='source_license', + field=models.ForeignKey(help_text='License', on_delete=django.db.models.deletion.PROTECT, related_name='source_license', to='base.license'), + ), + ] diff --git a/src/datasources/migrations/0007_alter_datasource_source_license.py b/src/datasources/migrations/0007_alter_datasource_source_license.py new file mode 100644 index 0000000..cc5729f --- /dev/null +++ b/src/datasources/migrations/0007_alter_datasource_source_license.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.10 on 2024-06-07 12:47 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0004_license'), + ('datasources', '0006_alter_datasource_source_license'), + ] + + operations = [ + migrations.AlterField( + model_name='datasource', + name='source_license', + field=models.ForeignKey(help_text='License', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='data_sources', to='base.license'), + ), + ] diff --git a/src/datasources/models.py b/src/datasources/models.py index 179ee83..95bc15e 100644 --- a/src/datasources/models.py +++ b/src/datasources/models.py @@ -17,6 +17,11 @@ class SourceSubdivision(TimeStampedModel): max_length=128, unique=True ) + external_name: models.CharField = models.CharField( + help_text=_('External Name'), + max_length=128, + null=True, + ) description: models.TextField = models.TextField( help_text=_('Source description'), max_length=1000, @@ -73,10 +78,15 @@ class DataSource(TimeStampedModel): null=True, blank=True ) - source_license: models.CharField = models.CharField( + + source_license: models.ForeignKey = models.ForeignKey( + 'base.License', + related_name='data_sources', help_text=_('License'), - max_length=128 + on_delete=models.PROTECT, + null=True ) + links: models.ManyToManyField = models.ManyToManyField( 'base.Link', help_text=_('DataSource links'), diff --git a/src/datasources/resources.py b/src/datasources/resources.py index b2bc3b1..7e0d0b2 100644 --- a/src/datasources/resources.py +++ b/src/datasources/resources.py @@ -5,13 +5,14 @@ from import_export import resources from import_export.fields import Field, widgets -from base.models import Link, LinkTypeChoices +from base.models import Link, LinkTypeChoices, License from datasources.models import DataSource, SourceSubdivision class SourceSubdivisionResource(resources.ModelResource): name = Field(attribute='name', column_name='Source Subdivision') display_name = Field(attribute='display_name', column_name='Source Subdivision') + external_name = Field(attribute='external_name', column_name='External Name') description = Field(attribute='description', column_name='Description') db_source = Field(attribute='db_source', column_name='DB Source') data_source = Field( @@ -28,8 +29,8 @@ class SourceSubdivisionResource(resources.ModelResource): class Meta: model = SourceSubdivision fields: tuple[Literal['name'], Literal['display_name'], Literal['description'], - Literal['data_source'], Literal['reference_signal'], Literal['links']] - fields = ('name', 'display_name', 'description', 'data_source', 'reference_signal', 'links') + Literal['data_source'], Literal['reference_signal'], Literal['links'], Literal['external_name']] + fields = ('name', 'display_name', 'description', 'data_source', 'reference_signal', 'links', 'external_name') import_id_fields: list[str] = ['name'] skip_unchanged = True @@ -39,6 +40,7 @@ def before_import_row(self, row, **kwargs) -> None: any additional links specified in 'DUA' or 'Link' columns. """ self.process_links(row) + self.process_licenses(row) self.process_datasource(row) def process_links(self, row) -> None: @@ -57,6 +59,13 @@ def process_links(self, row) -> None: link, created = Link.objects.get_or_create(url=link_url, link_type=link_type) row['Links'] += row['Links'] + f'|{link.url}' + def process_licenses(self, row) -> None: + if row['License']: + license: License + created: bool + license, created = License.objects.get_or_create(name=row['License']) + row['License'] = license + def process_datasource(self, row) -> None: if row['Name']: data_source: DataSource @@ -70,4 +79,6 @@ def process_datasource(self, row) -> None: } ) links: QuerySet[Link] = Link.objects.filter(url__in=row['Links'].split('|')).values_list('id', flat=True) + license: License = License.objects.filter(name=row['License']).first() data_source.links.add(*links) + data_source.source_license = license diff --git a/src/fixtures/available_geography.json b/src/fixtures/available_geography.json index 96fb7ca..503155b 100644 --- a/src/fixtures/available_geography.json +++ b/src/fixtures/available_geography.json @@ -4,6 +4,8 @@ "pk": 1, "fields": { "name": "county", + "display_name": "ADM2 (e.g. U.S. counties)", + "order_id": 3, "created": "2023-08-09T19:23:22.597131", "modified": "2023-08-09T19:23:22.597131" } @@ -13,6 +15,8 @@ "pk": 2, "fields": { "name": "hhs", + "display_name": "HHS Regions", + "order_id": 4, "created": "2023-08-09T19:23:22.597131", "modified": "2023-08-09T19:23:22.597131" } @@ -22,6 +26,8 @@ "pk": 3, "fields": { "name": "hrr", + "display_name": "Hospital Referral Regions (HRRs)", + "order_id": 6, "created": "2023-08-09T19:23:22.597131", "modified": "2023-08-09T19:23:22.597131" } @@ -31,6 +37,8 @@ "pk": 4, "fields": { "name": "msa", + "display_name": "Metropolitan Statistical Areas (MSAs)", + "order_id": 5, "created": "2023-08-09T19:23:22.597131", "modified": "2023-08-09T19:23:22.597131" } @@ -40,6 +48,8 @@ "pk": 5, "fields": { "name": "nation", + "display_name": "National", + "order_id": 1, "created": "2023-08-09T19:23:22.597131", "modified": "2023-08-09T19:23:22.597131" } @@ -49,6 +59,8 @@ "pk": 6, "fields": { "name": "state", + "display_name": "ADM1 (e.g. U.S. states)", + "order_id": 2, "created": "2023-08-09T19:23:22.597131", "modified": "2023-08-09T19:23:22.597131" } @@ -58,6 +70,8 @@ "pk": 7, "fields": { "name": "dma", + "display_name": "Designated Market Areas (DMAs)", + "order_id": 7, "created": "2023-08-09T19:23:22.597131", "modified": "2023-08-09T19:23:22.597131" } diff --git a/src/fixtures/demographic_scopes.json b/src/fixtures/demographic_scopes.json new file mode 100644 index 0000000..b3b01b6 --- /dev/null +++ b/src/fixtures/demographic_scopes.json @@ -0,0 +1,47 @@ +[ + { + "model": "signals.DemographicScope", + "pk": 1, + "fields": { + "name": "nationwide Change Healthcare network", + "created": "2022-01-01T00:00:00Z", + "modified": "2022-01-01T00:00:00Z" + } + }, + { + "model": "signals.DemographicScope", + "pk": 2, + "fields": { + "name": "nationwide Optum network", + "created": "2022-01-01T00:00:00Z", + "modified": "2022-01-01T00:00:00Z" + } + }, + { + "model": "signals.DemographicScope", + "pk": 3, + "fields": { + "name": "Adult Facebook users", + "created": "2022-01-01T00:00:00Z", + "modified": "2022-01-01T00:00:00Z" + } + }, + { + "model": "signals.DemographicScope", + "pk": 4, + "fields": { + "name": "Google search users", + "created": "2022-01-01T00:00:00Z", + "modified": "2022-01-01T00:00:00Z" + } + }, + { + "model": "signals.DemographicScope", + "pk": 5, + "fields": { + "name": "Smartphone users", + "created": "2022-01-01T00:00:00Z", + "modified": "2022-01-01T00:00:00Z" + } + } +] diff --git a/src/signal_documentation/settings.py b/src/signal_documentation/settings.py index 7b28461..e190577 100644 --- a/src/signal_documentation/settings.py +++ b/src/signal_documentation/settings.py @@ -24,7 +24,8 @@ from sentry_sdk.integrations.redis import RedisIntegration EPIVIS_URL = os.environ.get("EPIVIS_URL", "https://deploy-preview-36--cmu-delphi-epivis.netlify.app/") -DATA_EXPORT_URL = os.environ.get("DATA_EXPORT_URL", "https://api.covidcast.cmu.edu/epidata/covidcast/csv") +DATA_EXPORT_URL = os.environ.get("DATA_EXPORT_URL", "https://api.delphi.cmu.edu/epidata/covidcast/csv") +COVIDCAST_URL = os.environ.get("COVIDCAST_URL", "https://api.delphi.cmu.edu/epidata/covidcast/") SENTRY_DSN = os.environ.get('SENTRY_DSN') if SENTRY_DSN: @@ -48,8 +49,7 @@ # SECURITY WARNING: don't run with debug turned on in production! -# DEBUG = bool(strtobool(os.getenv('DEBUG', 'True'))) -DEBUG = True +DEBUG = bool(strtobool(os.getenv('DEBUG', 'True'))) # SECURITY WARNING: keep the secret key used in production secret! @@ -261,6 +261,8 @@ } } +CACHE_TIME = int(os.environ.get('CACHE_TIME', 60 * 60 * 24)) # 24 hours + # Celery # https://docs.celeryq.dev/en/stable/index.html diff --git a/src/signals/admin.py b/src/signals/admin.py index f71546a..6f26f2a 100644 --- a/src/signals/admin.py +++ b/src/signals/admin.py @@ -4,12 +4,15 @@ from import_export.admin import ImportExportModelAdmin from signals.models import ( + DemographicScope, Geography, GeographyUnit, Pathogen, Signal, SignalCategory, SignalType, + GeographySignal, + GeographicScope, ) from signals.resources import SignalBaseResource, SignalResource @@ -41,6 +44,15 @@ class GeographyAdmin(admin.ModelAdmin): search_fields: tuple[Literal['name']] = ('name',) +@admin.register(GeographySignal) +class GeographySignalAdmin(admin.ModelAdmin): + """ + Admin interface for managing signal geography objects. + """ + list_display: tuple[Literal['geography']] = ('geography', 'signal', 'aggregated_by_delphi') + search_fields: tuple[Literal['geography']] = ('geography', 'signal', 'aggregated_by_delphi') + + @admin.register(Pathogen) class PathogenAdmin(admin.ModelAdmin): """ @@ -59,6 +71,15 @@ class SignalTypeAdmin(admin.ModelAdmin): search_fields: tuple[Literal['name']] = ('name',) +@admin.register(DemographicScope) +class DemographicScopeAdmin(admin.ModelAdmin): + """ + Admin interface for managing demographic scope objects. + """ + list_display: tuple[Literal['name']] = ('name',) + search_fields: tuple[Literal['name']] = ('name',) + + @admin.register(Signal) class SignalAdmin(ImportExportModelAdmin): """ @@ -80,3 +101,12 @@ class SignalAdmin(ImportExportModelAdmin): 'has_sample_size', ) resource_classes: list[type] = [SignalResource, SignalBaseResource] + + +@admin.register(GeographicScope) +class GeographicScopeAdmin(admin.ModelAdmin): + """ + Admin interface for managing signal type objects. + """ + list_display: tuple[Literal['name']] = ('name',) + search_fields: tuple[Literal['name']] = ('name',) diff --git a/src/signals/filters.py b/src/signals/filters.py index 3777f74..d9269c7 100644 --- a/src/signals/filters.py +++ b/src/signals/filters.py @@ -1,4 +1,5 @@ from typing import Any +import logging import django_filters from django.db.models import Q @@ -15,9 +16,14 @@ FormatChoices, Signal, TimeTypeChoices, + GeographicScope, + SeverityPyramidRungsChoices, ) +logger = logging.getLogger(__name__) + + class NumberInFilter(BaseInFilter, NumberFilter): pass @@ -41,9 +47,22 @@ class SignalFilter(django_filters.FilterSet): ) ) format_type = django_filters.MultipleChoiceFilter(choices=FormatChoices.choices) - source = django_filters.ModelMultipleChoiceFilter(queryset=SourceSubdivision.objects.all()) + severenity_pyramid_rungs = django_filters.MultipleChoiceFilter(choices=SeverityPyramidRungsChoices.choices) + source = django_filters.ModelMultipleChoiceFilter(queryset=SourceSubdivision.objects.all(), + field_name="source_id__external_name", + to_field_name='external_name') time_type = django_filters.MultipleChoiceFilter(choices=TimeTypeChoices.choices) - base_signal = django_filters.BooleanFilter(lookup_expr='isnull', field_name='base_for') + from_date = django_filters.DateFilter(field_name='from_date', lookup_expr='gte') + to_date = django_filters.DateFilter(field_name='to_date', lookup_expr='lte') + signal_availability_days = django_filters.NumberFilter(field_name='signal_availability_days', lookup_expr='gte') + + def __init__(self, data, *args, **kwargs): + data = data.copy() + try: + data.setdefault('geographic_scope', GeographicScope.objects.get(name='USA').id) + except GeographicScope.DoesNotExist: + logger.warning("Default Geographic Scope was not found in the database. Using an empty list.") + super().__init__(data, *args, **kwargs) class Meta: model = Signal @@ -53,12 +72,14 @@ class Meta: 'pathogen', 'active', 'available_geography', - 'signal_type', + 'severenity_pyramid_rungs', 'category', - 'format_type', + 'geographic_scope', 'source', 'time_type', - 'base_signal', + 'from_date', + 'to_date', + 'signal_availability_days', ] def filter_search(self, queryset, name, value) -> Any: diff --git a/src/signals/forms.py b/src/signals/forms.py index 040689d..b6b9f82 100644 --- a/src/signals/forms.py +++ b/src/signals/forms.py @@ -4,10 +4,11 @@ from datasources.models import SourceSubdivision from signals.models import ( ActiveChoices, - FormatChoices, Pathogen, Signal, TimeTypeChoices, + GeographicScope, + SeverityPyramidRungsChoices, ) @@ -27,10 +28,17 @@ class SignalFilterForm(forms.ModelForm): search = forms.CharField(min_length=3) pathogen = forms.ModelChoiceField(queryset=Pathogen.objects.all(), widget=forms.CheckboxSelectMultiple()) active = forms.TypedMultipleChoiceField(choices=ActiveChoices.choices, coerce=bool, widget=forms.CheckboxSelectMultiple()) - format_type = forms.ChoiceField(choices=FormatChoices.choices, widget=forms.CheckboxSelectMultiple()) - source = forms.ModelMultipleChoiceField(queryset=SourceSubdivision.objects.all(), widget=forms.CheckboxSelectMultiple()) + source = forms.MultipleChoiceField( + choices=[], + widget=forms.CheckboxSelectMultiple() + ) time_type = forms.ChoiceField(choices=TimeTypeChoices.choices, widget=forms.CheckboxSelectMultiple()) - base_signal = forms.ChoiceField(choices=[('', _('All')), (True, _('Yes')), (False, _('No'))], required=False, widget=forms.RadioSelect()) + geographic_scope = forms.ModelMultipleChoiceField(queryset=GeographicScope.objects.all(), widget=forms.CheckboxSelectMultiple()) + severenity_pyramid_rungs = forms.ChoiceField(choices=SeverityPyramidRungsChoices.choices, widget=forms.CheckboxSelectMultiple()) + + from_date = forms.DateField(widget=forms.DateInput(attrs={'type': 'date'}), required=False) + to_date = forms.DateField(widget=forms.DateInput(attrs={'type': 'date'}), required=False) + signal_availability_days = forms.IntegerField(required=False) class Meta: model = Signal @@ -41,11 +49,13 @@ class Meta: 'pathogen', 'active', 'available_geography', - 'category', - 'format_type', - 'signal_type', + 'severenity_pyramid_rungs', 'source', 'time_type', + 'geographic_scope', + 'from_date', + 'to_date', + 'signal_availability_days', ] widgets = { @@ -60,27 +70,11 @@ class Meta: 'data-bs-toggle': 'tooltip', 'data-bs-placement': 'bottom', }), - 'signal_type': forms.CheckboxSelectMultiple(attrs={ - 'class': 'form-select', - 'data-bs-toggle': 'tooltip', - 'data-bs-placement': 'bottom', - }), - 'category': forms.CheckboxSelectMultiple(attrs={ - 'class': 'form-select', - 'data-bs-toggle': 'tooltip', - 'data-bs-placement': 'bottom', - }), - 'format_type': forms.CheckboxSelectMultiple(attrs={ - 'class': 'form-select', - 'data-bs-toggle': 'tooltip', - 'data-bs-placement': 'bottom', - }), 'source': forms.CheckboxSelectMultiple(attrs={ 'class': 'form-select', 'data-bs-toggle': 'tooltip', 'data-bs-placement': 'bottom', }), - } def __init__(self, *args, **kwargs) -> None: @@ -90,7 +84,15 @@ def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) # Set required attribute to False and disable helptext for all fields + self.fields["available_geography"].queryset = self.fields["available_geography"].queryset.order_by("order_id") + try: + self.fields["source"].choices = set(SourceSubdivision.objects.values_list('external_name', 'external_name')) + except SourceSubdivision.DoesNotExist: + self.fields["source"].choices = [] for field_name, field in self.fields.items(): field.required = False field.help_text = '' field.label = '' + self.fields['from_date'].label = _('Available Since') + self.fields['to_date'].label = _('Available Until') + self.fields['signal_availability_days'].label = _('Available for at least (days)') diff --git a/src/signals/migrations/0009_demographicscope_geographicscope_licence_and_more.py b/src/signals/migrations/0009_demographicscope_geographicscope_licence_and_more.py new file mode 100644 index 0000000..55b0a7e --- /dev/null +++ b/src/signals/migrations/0009_demographicscope_geographicscope_licence_and_more.py @@ -0,0 +1,173 @@ +# Generated by Django 5.0.3 on 2024-06-06 16:14 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('signals', '0008_remove_geographyunit_postal'), + ] + + operations = [ + migrations.CreateModel( + name='DemographicScope', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('name', models.CharField(help_text='Name', max_length=128, unique=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='GeographicScope', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('name', models.CharField(help_text='Name', max_length=128, unique=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Licence', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('name', models.CharField(help_text='Name', max_length=128, unique=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Organisation', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('organisation_name', models.CharField(help_text='Organisation Name', max_length=128, unique=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.RemoveField( + model_name='signal', + name='available_geography', + ), + migrations.AddField( + model_name='signal', + name='age_breakdown', + field=models.CharField(choices=[('0-17', '0-17'), ('18-64', '18-64'), ('65+', '65+')], help_text='Age Breakdown', max_length=128, null=True), + ), + migrations.AddField( + model_name='signal', + name='data_censoring', + field=models.TextField(blank=True, help_text='Data Censoring', null=True), + ), + migrations.AddField( + model_name='signal', + name='gender_breakdown', + field=models.BooleanField(default=False, help_text='Gender Breakdown'), + ), + migrations.AddField( + model_name='signal', + name='missingness', + field=models.TextField(blank=True, help_text='Missingness', null=True), + ), + migrations.AddField( + model_name='signal', + name='race_breakdown', + field=models.BooleanField(default=False, help_text='Race Breakdown'), + ), + migrations.AddField( + model_name='signal', + name='reporting_cadence', + field=models.CharField(choices=[('daily', 'Daily'), ('weekly', 'Weekly')], help_text='Reporting Cadence', max_length=128, null=True), + ), + migrations.AddField( + model_name='signal', + name='restrictions', + field=models.TextField(blank=True, help_text='Restrictions', null=True), + ), + migrations.AddField( + model_name='signal', + name='severenity_pyramid_rungs', + field=models.CharField(choices=[('population', 'Population'), ('infected', 'Infected'), ('symptomatic', 'Symptomatic'), ('outpatient_visit', 'Outpatient visit'), ('ascertained', 'Ascertained (case)'), ('hospitalized', 'Hospitalized'), ('icu', 'ICU'), ('dead', 'Dead')], help_text='Severity Pyramid Rungs', max_length=128, null=True), + ), + migrations.AddField( + model_name='signal', + name='temporal_scope_end', + field=models.DateField(blank=True, help_text='Temporal Scope End', null=True), + ), + migrations.AddField( + model_name='signal', + name='temporal_scope_end_note', + field=models.TextField(blank=True, help_text='Temporal Scope End Note', null=True), + ), + migrations.AddField( + model_name='signal', + name='temporal_scope_start', + field=models.DateField(blank=True, help_text='Temporal Scope Start', null=True), + ), + migrations.AddField( + model_name='signal', + name='temporal_scope_start_note', + field=models.TextField(blank=True, help_text='Temporal Scope Start Note', null=True), + ), + migrations.AddField( + model_name='signal', + name='typical_reporting_lag', + field=models.CharField(blank=True, help_text='Typical Reporting Lag', max_length=128, null=True), + ), + migrations.AddField( + model_name='signal', + name='typical_revision_cadence', + field=models.CharField(blank=True, help_text='Typical Revision Cadence', max_length=512, null=True), + ), + migrations.AddField( + model_name='signal', + name='demographic_scope', + field=models.ManyToManyField(help_text='Demographic Scope', related_name='signals', to='signals.demographicscope'), + ), + migrations.AddField( + model_name='signal', + name='geographic_scope', + field=models.ManyToManyField(help_text='Geographic Scope', to='signals.geographicscope'), + ), + migrations.AddField( + model_name='signal', + name='licence', + field=models.ManyToManyField(help_text='Licence', related_name='signals', to='signals.licence'), + ), + migrations.AddField( + model_name='signal', + name='organisations_access_list', + field=models.ManyToManyField(help_text='Organisations Access List. Who may access this signal?', related_name='accessed_signals', to='signals.organisation'), + ), + migrations.AddField( + model_name='signal', + name='organisations_sharing_list', + field=models.ManyToManyField(help_text='Organisations Sharing List. Who may be told about this signal?', related_name='shared_signals', to='signals.organisation'), + ), + migrations.CreateModel( + name='GeographySignal', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('aggregated_by_delphi', models.BooleanField(default=False)), + ('geography', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='signals.geography')), + ('signal', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='signals.signal')), + ], + options={ + 'unique_together': {('geography', 'signal')}, + }, + ), + ] diff --git a/src/signals/migrations/0010_signal_available_geography.py b/src/signals/migrations/0010_signal_available_geography.py new file mode 100644 index 0000000..8bf7534 --- /dev/null +++ b/src/signals/migrations/0010_signal_available_geography.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.3 on 2024-06-06 16:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('signals', '0009_demographicscope_geographicscope_licence_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='signal', + name='available_geography', + field=models.ManyToManyField(help_text='Available geography', through='signals.GeographySignal', to='signals.geography'), + ), + ] diff --git a/src/signals/migrations/0011_alter_geographysignal_geography_and_more.py b/src/signals/migrations/0011_alter_geographysignal_geography_and_more.py new file mode 100644 index 0000000..798b121 --- /dev/null +++ b/src/signals/migrations/0011_alter_geographysignal_geography_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.0.3 on 2024-06-06 16:54 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('signals', '0010_signal_available_geography'), + ] + + operations = [ + migrations.AlterField( + model_name='geographysignal', + name='geography', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='geography_signals', to='signals.geography'), + ), + migrations.AlterField( + model_name='geographysignal', + name='signal', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='geography_signals', to='signals.signal'), + ), + ] diff --git a/src/signals/migrations/0012_remove_signal_licence_signal_license_delete_licence.py b/src/signals/migrations/0012_remove_signal_licence_signal_license_delete_licence.py new file mode 100644 index 0000000..e9d6ef4 --- /dev/null +++ b/src/signals/migrations/0012_remove_signal_licence_signal_license_delete_licence.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.10 on 2024-06-07 12:47 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0004_license'), + ('signals', '0011_alter_geographysignal_geography_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='signal', + name='licence', + ), + migrations.AddField( + model_name='signal', + name='license', + field=models.ForeignKey(help_text='License', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='signals', to='base.license'), + ), + migrations.DeleteModel( + name='Licence', + ), + ] diff --git a/src/signals/migrations/0013_remove_signal_signal_type_signal_signal_type.py b/src/signals/migrations/0013_remove_signal_signal_type_signal_signal_type.py new file mode 100644 index 0000000..4dc6cbf --- /dev/null +++ b/src/signals/migrations/0013_remove_signal_signal_type_signal_signal_type.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.10 on 2024-06-07 14:14 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('signals', '0012_remove_signal_licence_signal_license_delete_licence'), + ] + + operations = [ + migrations.RemoveField( + model_name='signal', + name='signal_type', + ), + migrations.AddField( + model_name='signal', + name='signal_type', + field=models.ForeignKey(help_text='Source Type', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='signals', to='signals.signaltype'), + ), + ] diff --git a/src/signals/migrations/0014_remove_signal_from_date_remove_signal_to_date.py b/src/signals/migrations/0014_remove_signal_from_date_remove_signal_to_date.py new file mode 100644 index 0000000..c975f5a --- /dev/null +++ b/src/signals/migrations/0014_remove_signal_from_date_remove_signal_to_date.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.10 on 2024-06-07 15:16 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('signals', '0013_remove_signal_signal_type_signal_signal_type'), + ] + + operations = [ + migrations.RemoveField( + model_name='signal', + name='from_date', + ), + migrations.RemoveField( + model_name='signal', + name='to_date', + ), + ] diff --git a/src/signals/migrations/0015_signal_from_date_signal_to_date_and_more.py b/src/signals/migrations/0015_signal_from_date_signal_to_date_and_more.py new file mode 100644 index 0000000..0034ced --- /dev/null +++ b/src/signals/migrations/0015_signal_from_date_signal_to_date_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.10 on 2024-06-07 15:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('signals', '0014_remove_signal_from_date_remove_signal_to_date'), + ] + + operations = [ + migrations.AddField( + model_name='signal', + name='from_date', + field=models.DateField(blank=True, help_text='From Date', null=True), + ), + migrations.AddField( + model_name='signal', + name='to_date', + field=models.DateField(blank=True, help_text='To Date', null=True), + ), + migrations.AlterField( + model_name='signal', + name='temporal_scope_end', + field=models.CharField(blank=True, help_text='Temporal Scope End', max_length=128, null=True), + ), + migrations.AlterField( + model_name='signal', + name='temporal_scope_start', + field=models.CharField(blank=True, help_text='Temporal Scope Start', max_length=128, null=True), + ), + ] diff --git a/src/signals/migrations/0016_remove_signal_geographic_scope_and_more.py b/src/signals/migrations/0016_remove_signal_geographic_scope_and_more.py new file mode 100644 index 0000000..4800ef0 --- /dev/null +++ b/src/signals/migrations/0016_remove_signal_geographic_scope_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.10 on 2024-06-10 13:32 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('signals', '0015_signal_from_date_signal_to_date_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='signal', + name='geographic_scope', + ), + migrations.AddField( + model_name='signal', + name='geographic_scope', + field=models.ForeignKey(help_text='Geographic Scope', null=True, on_delete=django.db.models.deletion.SET_NULL, to='signals.geographicscope'), + ), + ] diff --git a/src/signals/migrations/0017_geography_display_name.py b/src/signals/migrations/0017_geography_display_name.py new file mode 100644 index 0000000..2990003 --- /dev/null +++ b/src/signals/migrations/0017_geography_display_name.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.10 on 2024-06-11 14:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('signals', '0016_remove_signal_geographic_scope_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='geography', + name='display_name', + field=models.CharField(blank=True, help_text='Display Name', max_length=128, null=True), + ), + ] diff --git a/src/signals/migrations/0018_geography_order_id.py b/src/signals/migrations/0018_geography_order_id.py new file mode 100644 index 0000000..8b9c1b3 --- /dev/null +++ b/src/signals/migrations/0018_geography_order_id.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.10 on 2024-06-11 18:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('signals', '0017_geography_display_name'), + ] + + operations = [ + migrations.AddField( + model_name='geography', + name='order_id', + field=models.IntegerField(blank=True, help_text='Order ID', null=True), + ), + ] diff --git a/src/signals/migrations/0019_alter_signal_severenity_pyramid_rungs.py b/src/signals/migrations/0019_alter_signal_severenity_pyramid_rungs.py new file mode 100644 index 0000000..2d9ad1e --- /dev/null +++ b/src/signals/migrations/0019_alter_signal_severenity_pyramid_rungs.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.10 on 2024-06-11 19:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('signals', '0018_geography_order_id'), + ] + + operations = [ + migrations.AlterField( + model_name='signal', + name='severenity_pyramid_rungs', + field=models.CharField(choices=[('population', 'Population'), ('infected', 'Infected'), ('symptomatic', 'Symptomatic'), ('outpatient_visit', 'Outpatient / ED'), ('ascertained', 'Ascertained (case)'), ('hospitalized', 'Hospitalized'), ('icu', 'ICU'), ('dead', 'Dead')], help_text='Severity Pyramid Rungs', max_length=128, null=True), + ), + ] diff --git a/src/signals/migrations/0020_signal_signal_availability_days.py b/src/signals/migrations/0020_signal_signal_availability_days.py new file mode 100644 index 0000000..9f5557f --- /dev/null +++ b/src/signals/migrations/0020_signal_signal_availability_days.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.10 on 2024-06-20 06:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('signals', '0019_alter_signal_severenity_pyramid_rungs'), + ] + + operations = [ + migrations.AddField( + model_name='signal', + name='signal_availability_days', + field=models.IntegerField(blank=True, help_text='Number of days the signal is available for', null=True), + ), + ] diff --git a/src/signals/models.py b/src/signals/models.py index acbdfc8..24337c3 100644 --- a/src/signals/models.py +++ b/src/signals/models.py @@ -12,6 +12,14 @@ class TimeTypeChoices(models.TextChoices): WEEK = 'week', _('Week') +class ReportingCadence(models.TextChoices): + """ + A class representing choices for reporting cadences. + """ + DAILY = 'daily', _('Daily') + WEEKLY = 'weekly', _('Weekly') + + class TimeLabelChoices(models.TextChoices): """ A class representing choices for time labels. @@ -45,8 +53,30 @@ class ActiveChoices(models.TextChoices): """ A class representing choices for active signals. """ - ACTIVE = True, _('Active') - HISTORICAL = False, _('Historical') + ACTIVE = True, _('Current Surveillance Only') + + +class SeverityPyramidRungsChoices(models.TextChoices): + """ + A class representing choices for severity pyramid rungs. + """ + POPULATION = 'population', _('Population') + INFECTED = 'infected', _('Infected') + SYMPTOMATIC = 'symptomatic', _('Symptomatic') + OUTPATIENT_VISIT = 'outpatient_visit', _('Outpatient / ED') + ASCERTAINED = 'ascertained', _('Ascertained (case)') + HOSPITALIZED = 'hospitalized', _('Hospitalized') + ICU = 'icu', _('ICU') + DEAD = 'dead', _('Dead') + + +class AgeBreakdownChoices(models.TextChoices): + """ + A class representing choices for age breakdown. + """ + CILDREN = '0-17', '0-17' + ADULTS = '18-64', '18-64' + SENIORS = '65+', '65+' class SignalCategory(TimeStampedModel): @@ -123,6 +153,19 @@ class Geography(TimeStampedModel): unique=True ) + display_name: models.CharField = models.CharField( + help_text=_('Display Name'), + max_length=128, + null=True, + blank=True + ) + + order_id: models.IntegerField = models.IntegerField( + help_text=_('Order ID'), + null=True, + blank=True + ) + class Meta: verbose_name_plural: str = "geographies" ordering: list[str] = ["name"] @@ -134,7 +177,26 @@ def __str__(self) -> str: :return: The name of the available geography as a string. :rtype: str """ - return str(self.name) + return str(self.display_name) + + +class GeographySignal(models.Model): + geography = models.ForeignKey('signals.Geography', on_delete=models.CASCADE, related_name='geography_signals') + signal = models.ForeignKey('signals.Signal', on_delete=models.CASCADE, related_name='geography_signals') + aggregated_by_delphi = models.BooleanField(default=False) + + class Meta: + unique_together = ('geography', 'signal') + + @property + def display_name(self) -> str: + """ + Returns the display name of the geography signal. + + :return: The display name of the geography signal. + :rtype: str + """ + return f'{self.geography.name} (by Delphi)' if self.aggregated_by_delphi else self.geography.name class GeographyUnit(TimeStampedModel): @@ -173,6 +235,57 @@ def __str__(self) -> str: return str(self.name) +class GeographicScope(TimeStampedModel): + """ + A model representing a geographic scope. + """ + name: models.CharField = models.CharField( + help_text=_('Name'), + max_length=128, + unique=True + ) + + def __str__(self) -> str: + """ + Returns the name of the geographic scope as a string. + + :return: The name of the geographic scope as a string. + :rtype: str + """ + return str(self.name) + + +class DemographicScope(TimeStampedModel): + """ + A model representing a demographic scope. + """ + name: models.CharField = models.CharField( + help_text=_('Name'), + max_length=128, + unique=True + ) + + def __str__(self) -> str: + """ + Returns the name of the demographic scope as a string. + + :return: The name of the demographic scope as a string. + :rtype: str + """ + return str(self.name) + + +class Organisation(TimeStampedModel): + """ + A model representing an access list. + """ + organisation_name: models.CharField = models.CharField( + help_text=_('Organisation Name'), + max_length=128, + unique=True + ) + + class Signal(TimeStampedModel): """ A model representing a signal. @@ -198,10 +311,12 @@ class Signal(TimeStampedModel): related_name='signals', help_text=_('Pathogen/Disease Area'), ) - signal_type: models.ManyToManyField = models.ManyToManyField( - 'signals.SignalType', + signal_type: models.ForeignKey = models.ForeignKey( + 'signals.signalType', related_name='signals', - help_text=_('Signal Type') + help_text=_('Source Type'), + on_delete=models.PROTECT, + null=True ) active: models.BooleanField = models.BooleanField( help_text=_('Active'), @@ -234,6 +349,35 @@ class Signal(TimeStampedModel): max_length=128, choices=TimeLabelChoices.choices ) + reporting_cadence: models.CharField = models.CharField( + help_text=_('Reporting Cadence'), + max_length=128, + choices=ReportingCadence.choices, + null=True + ) + typical_reporting_lag: models.CharField = models.CharField( + help_text=_('Typical Reporting Lag'), + max_length=128, + null=True, + blank=True + ) + typical_revision_cadence: models.CharField = models.CharField( + help_text=_('Typical Revision Cadence'), + max_length=512, + null=True, + blank=True + ) + demographic_scope: models.ManyToManyField = models.ManyToManyField( + 'signals.DemographicScope', + help_text=_('Demographic Scope'), + related_name='signals', + ) + severenity_pyramid_rungs: models.CharField = models.CharField( + help_text=_('Severity Pyramid Rungs'), + max_length=128, + choices=SeverityPyramidRungsChoices.choices, + null=True + ) category: models.ForeignKey = models.ForeignKey( 'signals.SignalCategory', related_name='signals', @@ -246,9 +390,50 @@ class Signal(TimeStampedModel): help_text=_('Signal links'), related_name="signals" ) + geographic_scope: models.ForeignKey = models.ForeignKey( + 'signals.GeographicScope', + help_text=_('Geographic Scope'), + on_delete=models.SET_NULL, + null=True + ) available_geography: models.ManyToManyField = models.ManyToManyField( 'signals.Geography', - help_text=_('Available geography') + help_text=_('Available geography'), + through='signals.GeographySignal' + ) + temporal_scope_start: models.DateField = models.DateField( + help_text=_('Temporal Scope Start'), + null=True, + blank=True + ) + temporal_scope_start_note = models.TextField( + help_text=_('Temporal Scope Start Note'), + null=True, + blank=True + ) + temporal_scope_end: models.DateField = models.DateField( + help_text=_('Temporal Scope End'), + null=True, + blank=True + ) + temporal_scope_end_note = models.TextField( + help_text=_('Temporal Scope End Note'), + null=True, + blank=True + ) + gender_breakdown: models.BooleanField = models.BooleanField( + help_text=_('Gender Breakdown'), + default=False + ) + race_breakdown: models.BooleanField = models.BooleanField( + help_text=_('Race Breakdown'), + default=False, + ) + age_breakdown: models.CharField = models.CharField( + help_text=_('Age Breakdown'), + max_length=128, + choices=AgeBreakdownChoices.choices, + null=True, ) is_smoothed: models.BooleanField = models.BooleanField( help_text=_('Is Smoothed'), @@ -281,6 +466,42 @@ class Signal(TimeStampedModel): help_text=_('Source Subdivision'), on_delete=models.PROTECT, ) + data_censoring: models.TextField = models.TextField( + help_text=_('Data Censoring'), + null=True, + blank=True + ) + missingness: models.TextField = models.TextField( + help_text=_('Missingness'), + null=True, + blank=True + ) + organisations_access_list: models.ManyToManyField = models.ManyToManyField( + 'signals.Organisation', + help_text=_('Organisations Access List. Who may access this signal?'), + related_name='accessed_signals' + ) + + organisations_sharing_list: models.ManyToManyField = models.ManyToManyField( + 'signals.Organisation', + help_text=_('Organisations Sharing List. Who may be told about this signal?'), + related_name='shared_signals' + ) + + license: models.ForeignKey = models.ForeignKey( + 'base.License', + related_name='signals', + help_text=_('License'), + on_delete=models.PROTECT, + null=True + ) + + restrictions: models.TextField = models.TextField( + help_text=_('Restrictions'), + null=True, + blank=True + ) + last_updated: models.DateField = models.DateField( help_text=_('Last Updated'), null=True, @@ -297,11 +518,65 @@ class Signal(TimeStampedModel): blank=True ) + signal_availability_days: models.IntegerField = models.IntegerField( + help_text=_('Number of days the signal is available for'), + null=True, + blank=True + ) + + temporal_scope_start: models.CharField = models.CharField( + help_text=_('Temporal Scope Start'), + null=True, + blank=True, + max_length=128 + ) + temporal_scope_end: models.CharField = models.CharField( + help_text=_('Temporal Scope End'), + null=True, + blank=True, + max_length=128 + ) + @property - def example_url(self): + def is_access_public(self) -> bool: + """ + Returns True if the signal is public, False otherwise. + + :return: True if the signal is public, False otherwise. + :rtype: bool + """ + return self.organisations_access_list.count() == 0 + + def is_sharing_public(self) -> bool: + """ + Returns True if the signal is public, False otherwise. + + :return: True if the signal is public, False otherwise. + :rtype: bool + """ + return self.organisations_sharing_list.count() == 0 + + @property + def example_url(self) -> str | None: + """ + Returns the example URL of the signal. + + :return: The example URL of the signal. + :rtype: str | None + """ example_url = self.links.filter(link_type="example_url").first() return example_url.url if example_url else None + @property + def has_all_demographic_scopes(self) -> bool: + """ + Returns True if the signal has all demographic scopes, False otherwise. + + :return: True if the signal has all demographic scopes, False otherwise. + :rtype: bool + """ + return self.demographic_scope.count() == DemographicScope.objects.count() + @property def same_base_signals(self): """ diff --git a/src/signals/resources.py b/src/signals/resources.py index ffed940..f731925 100644 --- a/src/signals/resources.py +++ b/src/signals/resources.py @@ -5,14 +5,18 @@ from import_export import resources from import_export.fields import Field, widgets -from base.models import Link, LinkTypeChoices +from base.models import Link, LinkTypeChoices, License from datasources.models import SourceSubdivision from signals.models import ( + DemographicScope, Geography, + GeographySignal, + Organisation, Pathogen, Signal, SignalCategory, SignalType, + GeographicScope, ) @@ -36,7 +40,7 @@ class SignalBaseResource(resources.ModelResource): class Meta: model = Signal - fields: list[str] = ['base'] + fields: list[str] = ['base', 'name', 'source', 'display_name'] import_id_fields: list[str] = ['name', 'source', 'display_name'] def before_import_row(self, row, **kwargs) -> None: @@ -67,14 +71,23 @@ class SignalResource(resources.ModelResource): signal_type = Field( attribute='signal_type', column_name='Signal Type', - widget=widgets.ManyToManyWidget(SignalType, field='name', separator=','), + widget=widgets.ForeignKeyWidget(SignalType, field='name'), ) active = Field(attribute='active', column_name='Active') short_description = Field(attribute='short_description', column_name='Short Description') description = Field(attribute='description', column_name='Description') - format = Field(attribute='format', column_name='Format') + format_type = Field(attribute='format_type', column_name='Format') time_type = Field(attribute='time_type', column_name='Time Type') time_label = Field(attribute='time_label', column_name='Time Label') + typical_reporting_lag = Field(attribute='typical_reporting_lag', column_name='Typical Reporting Lag') + typical_revision_cadence = Field(attribute='typical_revision_cadence', column_name='Typical Revision Cadence') + reporting_cadence = Field(attribute='reporting_cadence', column_name='Reporting Cadence') + demographic_scope = Field( + attribute='demographic_scope', + column_name='Demographic Scope', + widget=widgets.ManyToManyWidget(DemographicScope, field='name', separator=','), + ) + severenity_pyramid_rungs = Field(attribute='severenity_pyramid_rungs', column_name='Severity Pyramid Rungs') category = Field( attribute='category', column_name='Category', @@ -85,6 +98,11 @@ class SignalResource(resources.ModelResource): column_name='Available Geography', widget=widgets.ManyToManyWidget(Geography, field='name', separator=','), ) + # delphi_aggregated_geography = Field( + # attribute='delphi_aggregated_geography', + # column_name='Delphi-Aggregated Geography', + # widget=widgets.ManyToManyWidget(Geography, field='name', separator=','), + # ) is_smoothed = Field(attribute='is_smoothed', column_name='Is Smoothed') is_weighted = Field(attribute='is_weighted', column_name='Is Weighted') is_cumulative = Field(attribute='is_cumulative', column_name='Is Cumulative') @@ -96,11 +114,45 @@ class SignalResource(resources.ModelResource): column_name='Source Subdivision', widget=widgets.ForeignKeyWidget(SourceSubdivision, field='name'), ) + data_censoring = Field(attribute='data_censoring', column_name='Data Censoring') + missingness = Field(attribute='missingness', column_name='Missingness') + organisations_access_list = Field( + attribute='organisations_access_list', + column_name='Who may access this signal?', + widget=widgets.ManyToManyWidget(Organisation, field='organisation_name', separator=','), + ) + organisations_sharing_list = Field( + attribute='organisations_sharing_list', + column_name='Who may be told about this signal?', + widget=widgets.ManyToManyWidget(Organisation, field='organisation_name', separator=','), + ) + restrictions = Field(attribute='restrictions', column_name='Use Restrictions') + license = Field( + attribute='license', + column_name='License', + widget=widgets.ForeignKeyWidget(License, field='name'), + ) links = Field( attribute='links', column_name='Links', widget=widgets.ManyToManyWidget(Link, field='url', separator='|'), ) + temporal_scope_start = Field( + attribute='temporal_scope_start', + column_name='Temporal Scope Start', + ) + temporal_scope_end = Field( + attribute='temporal_scope_end', + column_name='Temporal Scope End', + ) + geographic_scope = Field( + attribute='geographic_scope', + column_name='Geographic Scope', + widget=widgets.ForeignKeyWidget(GeographicScope, field='name'), + ) + # gender_breakdown = Field(attribute='gender_breakdown', column_name='Gender Breakdown') + # race_breakdown = Field(attribute='race_breakdown', column_name='Race Breakdown') + # age_breakdown = Field(attribute='age_breakdown', column_name='Age Breakdown') class Meta: model = Signal @@ -112,9 +164,12 @@ class Meta: 'active', 'short_description', 'description', - 'format', + 'format_type', 'time_type', 'time_label', + 'reporting_cadence', + 'demographic_scope', + 'severenity_pyramid_rungs', 'available_geography', 'is_smoothed', 'is_weighted', @@ -123,18 +178,45 @@ class Meta: 'has_sample_size', 'high_values_are', 'source', - 'links' + 'links', + 'data_censoring', + 'missingness', + 'temporal_scope_start', + 'temporal_scope_end', + 'geographic_scope', + 'typical_reporting_lag', + 'license', + 'restrictions', + 'typical_revision_cadence', + # 'gender_breakdown', + # 'race_breakdown', + # 'age_breakdown', ] import_id_fields: list[str] = ['name', 'source', 'display_name'] + store_instance = True def before_import_row(self, row, **kwargs) -> None: """ Pre-processes each row before importing. """ - self.fix_boolean_fields(row, ['Active', 'Is Smoothed', 'Is Weighted', 'Is Cumulative', 'Has StdErr', 'Has Sample Size']) + self.fix_boolean_fields(row, [ + 'Active', + 'Is Smoothed', + 'Is Weighted', + 'Is Cumulative', + 'Has StdErr', + 'Has Sample Size', + # 'gender_breakdown', + # 'race_breakdown', + ]) self.process_links(row) self.process_pathogen(row) + self.process_license(row) + self.process_signal_category(row) + self.process_signal_type(row) + self.process_geographic_scope(row) + self.process_demographic_scope(row) def is_url_in_domain(self, url, domain) -> Any: """ @@ -193,3 +275,111 @@ def process_pathogen(self, row) -> None: pathogens: str = row['Pathogen/ Disease Area'].split(',') for pathogen in pathogens: Pathogen.objects.get_or_create(name=pathogen.strip()) + + def process_demographic_scope(self, row) -> None: + """ + Processes demographic scope. + """ + + if row['Demographic Scope']: + if row['Demographic Scope'] == 'All': + DemographicScope.objects.all() + else: + demographic_scopes: str = row['Demographic Scope'].split(',') + for demographic_scope in demographic_scopes: + DemographicScope.objects.get_or_create(name=demographic_scope.strip()) + + def process_severenity_pyramid_rungs(self, row) -> None: + """ + Processes severenity pyramid rungs. + """ + + if row['Severity Pyramid Rungs']: + if row['Severity Pyramid Rungs'].startswith('None'): + row['Severity Pyramid Rungs'] = None + + def process_organisations_access_list(self, row): + """ + Processes organisations access list. + """ + + if row['Who may access this signal?']: + if row['Who may access this signal?'] == 'public': + organisations = [] + else: + organisations: str = row['Who may access this signal?'].split(',') + for organisation in organisations: + Organisation.objects.get_or_create(organisation_name=organisation.strip()) + + def process_organisations_sharing_list(self, row): + """ + Processes organisations sharing list. + """ + + if row['Who may be told about this signal?']: + if row['Who may be told about this signal?'] == 'public': + organisations = [] + else: + organisations: str = row['Who may be told about this signal?'].split(',') + for organisation in organisations: + Organisation.objects.get_or_create(organisation_name=organisation.strip()) + + def process_license(self, row): + """ + Processes license. + """ + + if row['License']: + license, created = License.objects.get_or_create(name=row['License']) + license.use_restrictions = row['Use Restrictions'] + license.save() + row['License'] = license + + def process_signal_category(self, row): + """ + Processes signal category. + """ + + if row['Category']: + category, created = SignalCategory.objects.get_or_create(name=row['Category']) + row['Category'] = category + + def process_signal_type(self, row): + if row['Signal Type']: + signal_type, created = SignalType.objects.get_or_create(name=row['Signal Type']) + row['Signal Type'] = signal_type + + def process_geographic_scope(self, row): + if row['Geographic Scope']: + geographic_scope, created = GeographicScope.objects.get_or_create(name=row['Geographic Scope']) + row['Geographic Scope'] = geographic_scope + + def after_import_row(self, row, row_result, **kwargs) -> None: + """ + Post-processes each row after importing. + """ + geographies: str = row['Available Geography'].split(',') + delphi_aggregated_geographies: str = row['Delphi-Aggregated Geography'].split(',') + for geography in geographies: + geography_instance, _ = Geography.objects.get_or_create(name=geography.strip()) + if geography in delphi_aggregated_geographies: + signal = Signal.objects.get(id=row_result.object_id) + geography_signal, _ = GeographySignal.objects.get_or_create(geography=geography_instance, signal=signal) + geography_signal.aggregated_by_delphi = True + geography_signal.save() + + # def process_available_geography(self, row): + # """ + # Processes available geography. + # """ + + # if row['Available Geography']: + # geographies: str = row['Available Geography'].split(',') + # delphi_aggregated_geographies: str = row['Delphi-Aggregated Geography'].split(',') + # for geography in geographies: + # geography_instance = Geography.objects.get_or_create(name=geography.strip()) + # if geography in delphi_aggregated_geographies: + # signal = Signal.objects.get(name=row['Signal']) + # signal_geography = GeographySignal.objects.filter(geography=geography_instance, signal=signal).first() + # signal_geography.delphi_aggregated = True + # signal_geography.save() diff --git a/src/signals/serializers.py b/src/signals/serializers.py index ff3671d..e876c6e 100644 --- a/src/signals/serializers.py +++ b/src/signals/serializers.py @@ -21,7 +21,7 @@ class SignalSerializer(ModelSerializer): links = LinkSerializer(many=True) pathogen = SlugRelatedField(many=True, read_only=True, slug_field='name') - signal_type = SlugRelatedField(many=True, read_only=True, slug_field='name') + signal_type = SlugRelatedField(read_only=True, slug_field='name') available_geography = SlugRelatedField(many=True, read_only=True, slug_field='name') category = SlugRelatedField(read_only=True, slug_field='name') source = SlugRelatedField(read_only=True, slug_field='name') diff --git a/src/signals/tasks.py b/src/signals/tasks.py index e8ec324..7ea4b57 100644 --- a/src/signals/tasks.py +++ b/src/signals/tasks.py @@ -2,18 +2,19 @@ import os import requests +from rest_framework.status import is_client_error, is_server_error from signal_documentation.celery import BaseTaskWithRetry, app from signals.tools import SignalLastUpdatedParser -COVID_CAST_META_URL = os.environ.get('COVID_CAST_META_URL', 'https://api.covidcast.cmu.edu/epidata/covidcast/meta') +COVID_CAST_META_URL = os.environ.get('COVID_CAST_META_URL', 'https://api.delphi.cmu.edu/epidata/covidcast/meta') @app.task(bind=BaseTaskWithRetry) def get_covidcast_meta(self): response = requests.get(COVID_CAST_META_URL, timeout=5) - if response is None: - return f'Not response, url {COVID_CAST_META_URL}' + if is_client_error(response.status_code) or is_server_error(response.status_code): + return f'{COVID_CAST_META_URL}. Error: {response.status_code} - {response.content}' if response.status_code == 200: parser = SignalLastUpdatedParser(covidcast_meta_data=json.loads(response.content)) diff --git a/src/signals/tools.py b/src/signals/tools.py index e505778..545d290 100644 --- a/src/signals/tools.py +++ b/src/signals/tools.py @@ -10,7 +10,7 @@ class SignalLastUpdatedParser: def __init__(self, covidcast_meta_data: list) -> None: self.covidcast_meta_data = covidcast_meta_data - self.year_month_date_format = '%Y%m' + self.year_week_date_format = '%Y-%W-%w' self.year_month_day_date_format = '%Y%m%d' def format_date(self, date: str,) -> datetime: @@ -23,7 +23,9 @@ def format_date(self, date: str,) -> datetime: """ formated_date: datetime if len(date) == 6: - formated_date = datetime.strptime(date, self.year_month_date_format) + year, week = date[:4], date[4:] + logger.info(f"Date: {date}, year: {year}, week: {int(week)-1}") + formated_date = datetime.strptime(f"{int(year)}-{int(week)-1}-1", self.year_week_date_format) elif len(date) == 8: formated_date = datetime.strptime(date, self.year_month_day_date_format) return formated_date @@ -36,14 +38,15 @@ def set_data(self) -> None: for db_source in self.covidcast_meta_data: for signal_data in db_source['signals']: try: - signal = Signal.objects.get(name=signal_data['signal_basename'], source__name=signal_data['source']) + signal = Signal.objects.get(name=signal_data['signal'], source__name=signal_data['source']) except Signal.DoesNotExist: logger.warning( - f"Signal {signal_data['signal_basename']} not found in db. Update failed." + f"Signal {signal_data['signal']} not found in db. Update failed." ) continue signal.last_updated = self.format_date(str(signal_data['max_issue'])) signal.from_date = self.format_date(str(signal_data['min_time'])) signal.to_date = self.format_date(str(signal_data['max_time'])) + signal.signal_availability_days = abs((signal.to_date - signal.from_date).days) signal.save() - logger.info(f"Signal {signal_data['signal_basename']} successfully updated.") + logger.info(f"Signal {signal_data['signal']} successfully updated.") diff --git a/src/signals/urls.py b/src/signals/urls.py index 51f63c2..c4b6542 100644 --- a/src/signals/urls.py +++ b/src/signals/urls.py @@ -1,14 +1,16 @@ from django.urls import path from django.urls.resolvers import URLPattern +from django.views.decorators.cache import cache_page from signals.views import ( SignalsDetailView, SignalsListApiView, SignalsListView, ) +from signal_documentation.settings import CACHE_TIME urlpatterns: list[URLPattern] = [ - path('', SignalsListView.as_view(), name='signals'), + path('', cache_page(CACHE_TIME)(SignalsListView.as_view()), name='signals'), path('signals//', SignalsDetailView.as_view(), name='signal'), path('signals//', SignalsDetailView.as_view(), name='signal'), diff --git a/src/signals/views.py b/src/signals/views.py index af80898..5d39d7d 100644 --- a/src/signals/views.py +++ b/src/signals/views.py @@ -1,4 +1,5 @@ from typing import Any, Dict +import logging from django.conf import settings from django.views.generic import DetailView, ListView @@ -8,10 +9,13 @@ from signals.filters import SignalFilter from signals.forms import SignalFilterForm -from signals.models import Signal +from signals.models import Signal, GeographicScope from signals.serializers import SignalSerializer +logger = logging.getLogger(__name__) + + class SignalsListView(ListView): """ ListView for displaying a list of Signal objects. @@ -45,16 +49,17 @@ def get_url_params(self): ] if self.request.GET.get("available_geography") else None, - "signal_type": [int(el) for el in self.request.GET.getlist("signal_type")] - if self.request.GET.get("signal_type") + "severenity_pyramid_rungs": [el for el in self.request.GET.getlist("severenity_pyramid_rungs")] + if self.request.GET.get("severenity_pyramid_rungs") else None, - "category": self.request.GET.getlist("category") - if self.request.GET.get("category") + "geographic_scope": [el for el in self.request.GET.getlist("geographic_scope")] + if self.request.GET.get("geographic_scope") else None, - "format_type": [el for el in self.request.GET.getlist("format_type")], - "source": [int(el) for el in self.request.GET.getlist("source")], + "source": [el for el in self.request.GET.getlist("source")], "time_type": [el for el in self.request.GET.getlist("time_type")], - "base_signal": self.request.GET.get("base_signal"), + "from_date": self.request.GET.get("from_date"), + "to_date": self.request.GET.get("to_date"), + "signal_availability_days": self.request.GET.get("signal_availability_days"), } url_params_str = "" for param_name, param_value in url_params_dict.items(): @@ -76,12 +81,29 @@ def get_context_data(self, **kwargs) -> Dict[str, Any]: context: Dict[str, Any] = super().get_context_data(**kwargs) url_params_dict, url_params_str = self.get_url_params() + if not url_params_dict.get("geographic_scope"): + default_geographic_scope = [] + try: + default_geographic_scope = [GeographicScope.objects.get(name="USA").id] + except GeographicScope.DoesNotExist: + logger.warning("Default Geographic Scope was not found in the database. Using an empty list.") + url_params_dict["geographic_scope"] = default_geographic_scope context["url_params_dict"] = url_params_dict context["form"] = SignalFilterForm(initial=url_params_dict) context["url_params_str"] = url_params_str context["filter"] = SignalFilter(self.request.GET, queryset=self.get_queryset()) - context["signals"] = self.get_queryset() + context["signals"] = self.get_queryset().prefetch_related( + "pathogen", + "available_geography", + "geographic_scope", + "source", + ).select_related( + "base", + "signal_type", + "category", + "license" + ) return context @@ -103,6 +125,7 @@ def get_context_data(self, **kwargs) -> Dict[str, Any]: context: Dict[str, Any] = super().get_context_data(**kwargs) context["epivis_url"] = settings.EPIVIS_URL context["data_export_url"] = settings.DATA_EXPORT_URL + context["covidcast_url"] = settings.COVIDCAST_URL return context @@ -120,10 +143,9 @@ class SignalsListApiView(ListAPIView): "display_name", "pathogen__name", "available_geography__name", - "signal_type__name", - "category__name", - "format_type", + "severenity_pyramid_rungs", "base", "source__name", "time_label", + "geographic_scope__name", ) diff --git a/src/templates/signals/data_visualization_export.html b/src/templates/signals/data_visualization_export.html index 3334862..17c976a 100644 --- a/src/templates/signals/data_visualization_export.html +++ b/src/templates/signals/data_visualization_export.html @@ -62,7 +62,7 @@
Plot / Export data
@@ -209,7 +209,7 @@
Plot / Export data
$(document).ready(function () { {% for geography in signal.available_geography.all %} {% for unit in geography.geography_units.all %} - geoValues.push({'id': '{{ unit.geo_id }}', 'geoType': '{{ unit.geography }}', 'text': '{{ unit.display_name }}'}); + geoValues.push({'id': '{{ unit.geo_id }}', 'geoType': '{{ unit.geography.name }}', 'text': '{{ unit.display_name }}'}); {% endfor %} {% endfor %} @@ -275,7 +275,7 @@
Plot / Export data
if (!requestSent) { $.ajax({ - url: 'https://api.covidcast.cmu.edu/epidata/covidcast/', + url: '{{ covidcast_url }}', type: 'GET', data: { 'time_type': timeType, diff --git a/src/templates/signals/signal_detail.html b/src/templates/signals/signal_detail.html index 6ea20a2..9bddf36 100644 --- a/src/templates/signals/signal_detail.html +++ b/src/templates/signals/signal_detail.html @@ -83,13 +83,13 @@
About this signal
- Geography + Geographic Granularity, aka Geo-Level