Merge pull request #3 from cmu-delphi/main

development

Author: dmytrotsko

src/base/resources.py

@@ -54,15 +54,25 @@
         "display_order_number": 10,
         "short_name": "DMA",
     },
-    "facility": {
-        "display_name": 'Hospital ("Facility")',
+    "other_substate_region": {
+        "display_name": "Other Substate Region",
         "display_order_number": 11,
-        "short_name": "Facility",
+        "short_name": "Other Substate Region",
     },
     "FluSurv-Net site": {
         "display_name": "FluSurv-Net site (see documentation)",
         "display_order_number": 12,
         "short_name": "FluSurv-Net site",
     },
-    "N/A": {"display_name": "N/A", "display_order_number": 13, "short_name": "N/A"},
+    "facility": {
+        "display_name": 'Hospital ("Facility")',
+        "display_order_number": 13,
+        "short_name": "Facility",
+    },
+    "lat_long": {
+        "display_name": "Lat/Long",
+        "display_order_number": 14,
+        "short_name": "Lat/Long",
+    },
+    "N/A": {"display_name": "N/A", "display_order_number": 15, "short_name": "N/A"},
 }

src/indicators/admin.py

@@ -8,11 +8,13 @@
     IndicatorGeography,
     IndicatorType,
     OtherEndpointIndicator,
+    NonDelphiIndicator
 )
 from indicators.resources import (
     IndicatorResource,
     IndicatorBaseResource,
     OtherEndpointIndicatorResource,
+    NonDelphiIndicatorResource,
 )
 
 
@@ -90,3 +92,21 @@ class OtherEndpointIndicatorAdmin(ImportExportModelAdmin):
     list_display_links = ("name",)
 
     resource_classes = [OtherEndpointIndicatorResource]
+
+
+@admin.register(NonDelphiIndicator)
+class NonDelphiIndicatorAdmin(ImportExportModelAdmin):
+    list_display = (
+        "name",
+        "member_name",
+        "description",
+        "indicator_set",
+    )
+    search_fields = ("name", "description")
+    ordering = ("name",)
+    list_per_page = 50
+    list_select_related = True
+    list_editable = ("member_name", "description", "indicator_set")
+    list_display_links = ("name",)
+
+    resource_classes = [NonDelphiIndicatorResource]

src/indicators/migrations/0002_nondelphiindicator.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.0.7 on 2025-05-01 14:57
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('indicators', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='NonDelphiIndicator',
+            fields=[
+            ],
+            options={
+                'verbose_name': 'Non-Delphi Indicator',
+                'verbose_name_plural': 'Non-Delphi Indicators',
+                'proxy': True,
+                'indexes': [],
+                'constraints': [],
+            },
+            bases=('indicators.indicator',),
+        ),
+    ]

src/indicators/migrations/0003_alter_indicator_geographic_scope_and_more.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.0.7 on 2025-05-01 15:04
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0001_initial'),
+        ('datasources', '0001_initial'),
+        ('indicators', '0002_nondelphiindicator'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='indicator',
+            name='geographic_scope',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='indicators', to='base.geographicscope', verbose_name='Geographic Scope'),
+        ),
+        migrations.AlterField(
+            model_name='indicator',
+            name='source',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='indicators', to='datasources.sourcesubdivision', verbose_name='Source Subdivision'),
+        ),
+    ]

src/indicators/models.py

@@ -235,6 +235,8 @@ class Indicator(models.Model):
         verbose_name="Geographic Scope",
         on_delete=models.PROTECT,
         related_name="indicators",
+        null=True,
+        blank=True,
     )
     available_geographies: models.ManyToManyField = models.ManyToManyField(
         "base.Geography",
@@ -300,6 +302,8 @@ class Indicator(models.Model):
         verbose_name="Source Subdivision",
         on_delete=models.PROTECT,
         related_name="indicators",
+        null=True,
+        blank=True,
     )
     data_censoring: models.TextField = models.TextField(
         verbose_name="Data Censoring",
@@ -437,3 +441,11 @@ class Meta:
         proxy = True
         verbose_name = "Other Endpoint Indicator"
         verbose_name_plural = "Other Endpoint Indicators"
+
+
+class NonDelphiIndicator(Indicator):
+
+    class Meta:
+        proxy = True
+        verbose_name = "Non-Delphi Indicator"
+        verbose_name_plural = "Non-Delphi Indicators"

src/indicators/resources.py

@@ -1,8 +1,10 @@
+import logging
+
 from django.db.models import Max
 from import_export import resources
 from import_export.fields import Field
-from import_export.widgets import ForeignKeyWidget, ManyToManyWidget
 from import_export.results import RowResult
+from import_export.widgets import ForeignKeyWidget, ManyToManyWidget
 
 from base.models import GeographicScope, Geography, Pathogen, SeverityPyramidRung
 from base.resources import GEOGRAPHIC_GRANULARITY_MAPPING
@@ -13,8 +15,12 @@
     Indicator,
     IndicatorGeography,
     IndicatorType,
+    NonDelphiIndicator,
+    OtherEndpointIndicator,
 )
-from indicatorsets.models import IndicatorSet
+from indicatorsets.models import IndicatorSet, NonDelphiIndicatorSet
+
+logger = logging.getLogger(__name__)
 
 
 def fix_boolean_fields(row) -> None:
@@ -29,11 +35,11 @@ def fix_boolean_fields(row) -> None:
     ]
 
     for field in fields:
-        if row[field] == "TRUE":
+        if row.get(field, "") == "TRUE":
             row[field] = True
-        elif row[field] == "FALSE":
+        elif row.get(field, "") == "FALSE":
             row[field] = False
-        elif row[field] == "":
+        elif row.get(field, None) == "":
             row[field] = False
     return row
 
@@ -225,18 +231,27 @@ def import_row(self, row, instance_loader, **kwargs):
         return import_result
 
 
+class PermissiveForeignKeyWidget(ForeignKeyWidget):
+
+    def clean(self, value, row=None, **kwargs):
+        try:
+            return super().clean(value)
+        except self.model.DoesNotExist:
+            logger.warning(f"instance matching '{value}' does not exist")
+
+
 class IndicatorBaseResource(ModelResource):
     name = Field(attribute="name", column_name="Signal")
     display_name = Field(attribute="display_name", column_name="Name")
     base = Field(
         attribute="base",
         column_name="base",
-        widget=ForeignKeyWidget(Indicator, field="id"),
+        widget=PermissiveForeignKeyWidget(Indicator, field="id"),
     )
     source = Field(
         attribute="source",
         column_name="Source Subdivision",
-        widget=ForeignKeyWidget(SourceSubdivision, field="name"),
+        widget=PermissiveForeignKeyWidget(SourceSubdivision, field="name"),
     )
 
     class Meta:
@@ -271,13 +286,13 @@ class IndicatorResource(ModelResource):
     indicator_type = Field(
         attribute="indicator_type",
         column_name="Indicator Type",
-        widget=ForeignKeyWidget(IndicatorType),
+        widget=PermissiveForeignKeyWidget(IndicatorType),
     )
     active = Field(attribute="active", column_name="Active")
     format_type = Field(
         attribute="format_type",
         column_name="Format",
-        widget=ForeignKeyWidget(FormatType),
+        widget=PermissiveForeignKeyWidget(FormatType),
     )
     time_type = Field(attribute="time_type", column_name="Time Type")
     time_label = Field(attribute="time_label", column_name="Time Label")
@@ -299,12 +314,12 @@ class IndicatorResource(ModelResource):
     category = Field(
         attribute="category",
         column_name="Category",
-        widget=ForeignKeyWidget(Category),
+        widget=PermissiveForeignKeyWidget(Category),
     )
     geographic_scope = Field(
         attribute="geographic_scope",
         column_name="Geographic Coverage",
-        widget=ForeignKeyWidget(GeographicScope),
+        widget=PermissiveForeignKeyWidget(GeographicScope),
     )
     available_geographies = Field(
         attribute="available_geographies",
@@ -332,7 +347,7 @@ class IndicatorResource(ModelResource):
     source = Field(
         attribute="source",
         column_name="Source Subdivision",
-        widget=ForeignKeyWidget(SourceSubdivision),
+        widget=PermissiveForeignKeyWidget(SourceSubdivision),
     )
     data_censoring = Field(attribute="data_censoring", column_name="Data Censoring")
     missingness = Field(attribute="missingness", column_name="Missingness")
@@ -349,7 +364,7 @@ class IndicatorResource(ModelResource):
     indicator_set = Field(
         attribute="indicator_set",
         column_name="Indicator Set",
-        widget=ForeignKeyWidget(IndicatorSet, field="name"),
+        widget=PermissiveForeignKeyWidget(IndicatorSet, field="name"),
     )
 
     class Meta:
@@ -425,7 +440,7 @@ class OtherEndpointIndicatorResource(ModelResource):
         attribute="short_description", column_name="Short Description"
     )
     description = Field(attribute="description", column_name="Description")
-    member_name = Field(attribute="member_name", column_name="Member API Name")
+    member_name = Field(attribute="member_name", column_name="Member Name")
     member_short_name = Field(
         attribute="member_short_name", column_name="Member Short Name"
     )
@@ -440,13 +455,13 @@ class OtherEndpointIndicatorResource(ModelResource):
     indicator_type = Field(
         attribute="indicator_type",
         column_name="Indicator Type",
-        widget=ForeignKeyWidget(IndicatorType),
+        widget=PermissiveForeignKeyWidget(IndicatorType),
     )
     active = Field(attribute="active", column_name="Active")
     format_type = Field(
         attribute="format_type",
         column_name="Format",
-        widget=ForeignKeyWidget(FormatType),
+        widget=PermissiveForeignKeyWidget(FormatType),
     )
     time_type = Field(attribute="time_type", column_name="Time Type")
     time_label = Field(attribute="time_label", column_name="Time Label")
@@ -468,12 +483,12 @@ class OtherEndpointIndicatorResource(ModelResource):
     category = Field(
         attribute="category",
         column_name="Category",
-        widget=ForeignKeyWidget(Category),
+        widget=PermissiveForeignKeyWidget(Category),
     )
     geographic_scope = Field(
         attribute="geographic_scope",
         column_name="Geographic Coverage",
-        widget=ForeignKeyWidget(GeographicScope),
+        widget=PermissiveForeignKeyWidget(GeographicScope),
     )
     available_geographies = Field(
         attribute="available_geographies",
@@ -501,7 +516,7 @@ class OtherEndpointIndicatorResource(ModelResource):
     source = Field(
         attribute="source",
         column_name="Source Subdivision",
-        widget=ForeignKeyWidget(SourceSubdivision),
+        widget=PermissiveForeignKeyWidget(SourceSubdivision),
     )
     data_censoring = Field(attribute="data_censoring", column_name="Data Censoring")
     missingness = Field(attribute="missingness", column_name="Missingness")
@@ -518,11 +533,11 @@ class OtherEndpointIndicatorResource(ModelResource):
     indicator_set = Field(
         attribute="indicator_set",
         column_name="Indicator Set",
-        widget=ForeignKeyWidget(IndicatorSet, field="name"),
+        widget=PermissiveForeignKeyWidget(IndicatorSet, field="name"),
     )
 
     class Meta:
-        model = Indicator
+        model = OtherEndpointIndicator
         fields: list[str] = [
             "name",
             "display_name",
@@ -585,3 +600,36 @@ def skip_row(self, instance, original, row, import_validation_errors=None):
 
     def after_import_row(self, row, row_result, **kwargs):
         process_indicator_geography(row)
+
+
+class NonDelphiIndicatorResource(resources.ModelResource):
+
+    name = Field(attribute="name", column_name="Indicator Name")
+    display_name = Field(attribute="display_name", column_name="Indicator Name")
+    member_name = Field(attribute="member_name", column_name="Indicator API Name")
+    description = Field(attribute="description", column_name="Indicator Description")
+    indicator_set = Field(
+        attribute="indicator_set",
+        column_name="Indicator Set",
+        widget=PermissiveForeignKeyWidget(NonDelphiIndicatorSet, field="name"),
+    )
+
+    class Meta:
+        model = NonDelphiIndicator
+        fields: list[str] = [
+            "name",
+            "display_name",
+            "member_name",
+            "description",
+            "indicator_set",
+        ]
+        import_id_fields: list[str] = ["name"]
+        skip_unchanged = True
+
+    def before_import_row(self, row, **kwargs) -> None:
+        """Post-processes each row after importing."""
+        fix_boolean_fields(row)
+
+    def skip_row(self, instance, original, row, import_validation_errors=None):
+        if not row["Include in indicator app"]:
+            return True