Merge pull request #164 from cmu-delphi/development

Merge development into staging

Author: korlaxxalrok

.github/workflows/build-and-deploy.yaml

@@ -6,7 +6,8 @@ jobs:
     runs-on: ubuntu-latest
     # CI/CD will run on these branches
     if: >
-      github.ref == 'refs/heads/master' ||
+      github.ref == 'refs/heads/main' ||
+      github.ref == 'refs/heads/staging' ||
       github.ref == 'refs/heads/development'
     strategy:
       matrix:
@@ -27,7 +28,7 @@ jobs:
           baseRef="${GITHUB_REF#*/}"
           baseRef="${baseRef#*/}"
           case "${baseRef}" in
-          master)
+          main)
             image_tag="latest"
             ;;
           *)

README.md

@@ -82,11 +82,14 @@ Open `http://localhost:8000` to view it in the browser
 Though probably not necessary in most cases, if you want to test/modify/emulate how this will run in production you can:
 
 - In `.env` set:
-```
+
+```shell
 DEBUG = 'False'
 ```
+
 - Modify the app container's command in `docker-compose.yaml` to run:
-```
+
+```shell
 gunicorn signal_documentation.wsgi:application --bind 0.0.0.0:8000"
 
 *(Essentially you'll replace just the last line of the command, switching out the "runserver" line)
@@ -96,6 +99,10 @@ Open `http://localhost` to view it in the browser. In this usage your request wi
 
 The primary use case for this will be when making changes to the Nginx container image that runs in production and hosts the static file content, or also if making changes to the Gunicorn config.
 
+Additionally, though again not required for local development, you can also specify an env var of `MAIN_PAGE = $name`, and the app will be served at `http://localhost:8000/$name` (if running in debug mode), or if you've set `DEBUG = 'False'` to run it in Nginx/production mode at `http://localhost/$name/`. Note the ending slash when in Nginx/production mode _and_ using the `MAIN_PAGE` env var.
+
+The primary use case is so that we have flexibility to serve the application at something other than the "bare" URL, though doing this is not necessary for local development.
+
 Changes of this sort should be carefully evaluated as they may require interaction with systems managed by devops folks.
 
 ## [Django admin](https://docs.djangoproject.com/en/4.1/ref/contrib/admin/) web interface (user should be `is_staff` or `is_superuser`)
@@ -200,9 +207,15 @@ Each environment is essentially a bunch of different services all governed by `d
 
 ### Basic workflow
 
-- A PR merged to either `development` or `master` will trigger CI to build container images that are then tagged (based on the branch name and ":latest" respectively) and stored in our GitHub Packages container image repository.
+- A PR merged to either `development`, `staging`, or `main` will trigger CI to build container images that are then tagged with the branch name (or ":latest", in the cast of `main`), and stored in our GitHub Packages container image repository.
 - CI triggers a webhook that tells the host systems to pull and run new container images and restart any services that have been updated.
 
+As a developer, your path to getting changes into production should be something like this:
+
+- Source your working branch from `development`, do work, PR and merge when complete
+- PR and merge to `staging` in order to get your changes deployed to https://staging.delphi.cmu.edu/signals for review
+- PR and merge to `main` to go to production
+
 **IMPORTANT!** - The CI/CD process uses Docker Compose to build the specific container images that will be used in external environments. Success of the the build-and-deploy workflow is dependent on constructed services in `docker-compose.yaml`. If considering making changes there, please have a PR reviewed by devops folks :pray: :pray: :pray:
 
 ### Control of the deployed environment

docker-compose.yaml

@@ -31,15 +31,7 @@ services:
     restart: on-failure
     command: sh -c "python3 /usr/src/signal_documentation/src/manage.py migrate --noinput &&
                     python3 /usr/src/signal_documentation/src/manage.py collectstatic --noinput &&
-                    python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/available_geography.json &&
-                    python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/pathogens.json &&
-                    python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/signal_types.json &&
-                    python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/signal_categories.json &&
-                    python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/county.json &&
-                    python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/hhs.json &&
-                    python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/hrr.json &&
-                    python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/msa.json &&
-                    python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/state.json &&
+                    python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/* &&
                     python3 /usr/src/signal_documentation/src/manage.py runserver 0.0.0.0:8000"
     volumes:
       - .:/usr/src/signal_documentation

nginx/default.conf.template

@@ -1,10 +1,9 @@
 server {
     listen 80;
     server_name sdnginx;
-
-    location /static/ {
+    location ~* /static/(.*)$ {
         autoindex on;
-        alias /staticfiles/;
+        alias /staticfiles/$1;
     }
 
     location / {

src/base/admin.py

@@ -6,12 +6,13 @@
     DescriptedFilter,
     DescriptedFilterField,
     Link,
+    License
 )
 
 
 class DescriptedFilterFieldInline(admin.TabularInline):
     model = DescriptedFilterField
-    fields = ('description',)
+    fields = ('description', 'filter_field')
     extra = 0
     can_create = False
 
@@ -27,3 +28,12 @@ class LinkAdmin(admin.ModelAdmin):
     Admin interface for managing link objects.
     """
     list_display: tuple[Literal['url'], Literal['link_type']] = ('url', 'link_type')
+
+
+@admin.register(License)
+class GeographyAdmin(admin.ModelAdmin):
+    """
+    Admin interface for managing license objects.
+    """
+    list_display: tuple[Literal['name'], Literal['use_restrictions']] = ('name', 'use_restrictions')
+    search_fields: tuple[Literal['name']] = ('name',)

src/base/migrations/0004_license.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.10 on 2024-06-07 12:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0003_descriptedfilter_alter_link_link_type_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='License',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(help_text='License', max_length=256, unique=True)),
+                ('use_restrictions', models.TextField(blank=True, help_text='Use Restrictions', null=True)),
+            ],
+        ),
+    ]

src/base/models.py

@@ -123,3 +123,20 @@ def get_preview(self) -> LinkPreview:
             return {
                 'description': _('No description available'),
             }
+
+
+class License(models.Model):
+    """
+    A model representing a License.
+    """
+    name: models.CharField = models.CharField(help_text=_('License'), max_length=256, unique=True)
+    use_restrictions: models.TextField = models.TextField(help_text=_('Use Restrictions'), blank=True, null=True)
+
+    def __str__(self) -> str:
+        """
+        Returns the name of the license as a string.
+
+        :return: The name of the license as a string.
+        :rtype: str
+        """
+        return self.name

src/datasources/admin.py

@@ -12,14 +12,11 @@ class SourceSubdivisionAdmin(ImportExportModelAdmin):
     """
     Admin interface for managing source subdivision objects.
     """
-    list_display: tuple[Literal['name'], Literal['db_source']] = ('name', 'db_source')
+    list_display: tuple[Literal['name'], Literal['db_source'], Literal['external_name']] = ('name', 'db_source', 'external_name')
     search_fields: tuple[Literal['name'], Literal['db_source']] = ('name', 'db_source')
     resource_classes: list[type[SourceSubdivisionResource]] = [SourceSubdivisionResource]
 
 
-data_source_search_fields_type = tuple[Literal['name'], Literal['source_subdivision__db_source'], Literal['source_subdivision__name'], Literal['description']]
-
-
 @admin.register(DataSource)
 class DataSourceAdmin(ImportExportModelAdmin):
     """

src/datasources/migrations/0005_sourcesubdivision_external_name.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.3 on 2024-06-05 10:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('datasources', '0004_alter_datasource_options_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sourcesubdivision',
+            name='external_name',
+            field=models.CharField(help_text='External Name', max_length=128, null=True),
+        ),
+    ]

src/datasources/migrations/0006_alter_datasource_source_license.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.10 on 2024-06-07 12:15
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0004_license'),
+        ('datasources', '0005_sourcesubdivision_external_name'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datasource',
+            name='source_license',
+            field=models.ForeignKey(help_text='License', on_delete=django.db.models.deletion.PROTECT, related_name='source_license', to='base.license'),
+        ),
+    ]

src/datasources/migrations/0007_alter_datasource_source_license.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.10 on 2024-06-07 12:47
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0004_license'),
+        ('datasources', '0006_alter_datasource_source_license'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datasource',
+            name='source_license',
+            field=models.ForeignKey(help_text='License', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='data_sources', to='base.license'),
+        ),
+    ]

src/datasources/models.py

@@ -17,6 +17,11 @@ class SourceSubdivision(TimeStampedModel):
         max_length=128,
         unique=True
     )
+    external_name: models.CharField = models.CharField(
+        help_text=_('External Name'),
+        max_length=128,
+        null=True,
+    )
     description: models.TextField = models.TextField(
         help_text=_('Source description'),
         max_length=1000,
@@ -73,10 +78,15 @@ class DataSource(TimeStampedModel):
         null=True,
         blank=True
     )
-    source_license: models.CharField = models.CharField(
+
+    source_license: models.ForeignKey = models.ForeignKey(
+        'base.License',
+        related_name='data_sources',
         help_text=_('License'),
-        max_length=128
+        on_delete=models.PROTECT,
+        null=True
     )
+
     links: models.ManyToManyField = models.ManyToManyField(
         'base.Link',
         help_text=_('DataSource links'),

src/datasources/resources.py

@@ -5,13 +5,14 @@
 from import_export import resources
 from import_export.fields import Field, widgets
 
-from base.models import Link, LinkTypeChoices
+from base.models import Link, LinkTypeChoices, License
 from datasources.models import DataSource, SourceSubdivision
 
 
 class SourceSubdivisionResource(resources.ModelResource):
     name = Field(attribute='name', column_name='Source Subdivision')
     display_name = Field(attribute='display_name', column_name='Source Subdivision')
+    external_name = Field(attribute='external_name', column_name='External Name')
     description = Field(attribute='description', column_name='Description')
     db_source = Field(attribute='db_source', column_name='DB Source')
     data_source = Field(
@@ -28,8 +29,8 @@ class SourceSubdivisionResource(resources.ModelResource):
     class Meta:
         model = SourceSubdivision
         fields: tuple[Literal['name'], Literal['display_name'], Literal['description'],
-                      Literal['data_source'], Literal['reference_signal'], Literal['links']]
-        fields = ('name', 'display_name', 'description', 'data_source', 'reference_signal', 'links')
+                      Literal['data_source'], Literal['reference_signal'], Literal['links'], Literal['external_name']]
+        fields = ('name', 'display_name', 'description', 'data_source', 'reference_signal', 'links', 'external_name')
         import_id_fields: list[str] = ['name']
         skip_unchanged = True
 
@@ -39,6 +40,7 @@ def before_import_row(self, row, **kwargs) -> None:
         any additional links specified in 'DUA' or 'Link' columns.
         """
         self.process_links(row)
+        self.process_licenses(row)
         self.process_datasource(row)
 
     def process_links(self, row) -> None:
@@ -57,6 +59,13 @@ def process_links(self, row) -> None:
             link, created = Link.objects.get_or_create(url=link_url, link_type=link_type)
             row['Links'] += row['Links'] + f'|{link.url}'
 
+    def process_licenses(self, row) -> None:
+        if row['License']:
+            license: License
+            created: bool
+            license, created = License.objects.get_or_create(name=row['License'])
+            row['License'] = license
+
     def process_datasource(self, row) -> None:
         if row['Name']:
             data_source: DataSource
@@ -70,4 +79,6 @@ def process_datasource(self, row) -> None:
                 }
             )
             links: QuerySet[Link] = Link.objects.filter(url__in=row['Links'].split('|')).values_list('id', flat=True)
+            license: License = License.objects.filter(name=row['License']).first()
             data_source.links.add(*links)
+            data_source.source_license = license

src/fixtures/available_geography.json

@@ -4,6 +4,8 @@
       "pk": 1,
       "fields": {
         "name": "county",
+        "display_name": "ADM2 (e.g. U.S. counties)",
+        "order_id": 3,
         "created": "2023-08-09T19:23:22.597131",
         "modified": "2023-08-09T19:23:22.597131"
       }
@@ -13,6 +15,8 @@
       "pk": 2,
       "fields": {
         "name": "hhs",
+        "display_name": "HHS Regions",
+        "order_id": 4,
         "created": "2023-08-09T19:23:22.597131",
         "modified": "2023-08-09T19:23:22.597131"
       }
@@ -22,6 +26,8 @@
       "pk": 3,
       "fields": {
         "name": "hrr",
+        "display_name": "Hospital Referral Regions (HRRs)",
+        "order_id": 6,
         "created": "2023-08-09T19:23:22.597131",
         "modified": "2023-08-09T19:23:22.597131"
       }
@@ -31,6 +37,8 @@
       "pk": 4,
       "fields": {
         "name": "msa",
+        "display_name": "Metropolitan Statistical Areas (MSAs)",
+        "order_id": 5,
         "created": "2023-08-09T19:23:22.597131",
         "modified": "2023-08-09T19:23:22.597131"
       }
@@ -40,6 +48,8 @@
       "pk": 5,
       "fields": {
         "name": "nation",
+        "display_name": "National",
+        "order_id": 1,
         "created": "2023-08-09T19:23:22.597131",
         "modified": "2023-08-09T19:23:22.597131"
       }
@@ -49,6 +59,8 @@
       "pk": 6,
       "fields": {
         "name": "state",
+        "display_name": "ADM1 (e.g. U.S. states)",
+        "order_id": 2,
         "created": "2023-08-09T19:23:22.597131",
         "modified": "2023-08-09T19:23:22.597131"
       }
@@ -58,6 +70,8 @@
       "pk": 7,
       "fields": {
         "name": "dma",
+        "display_name": "Designated Market Areas (DMAs)",
+        "order_id": 7,
         "created": "2023-08-09T19:23:22.597131",
         "modified": "2023-08-09T19:23:22.597131"
       }