Merge latest from staging

.ci.env

@@ -0,0 +1,22 @@
+# This file is used to allow CI to start the compose services. It will typcially
+# not need to be modified.
+
+MYSQL_DATABASE=mysql_database
+MYSQL_USER=mysql_user
+MYSQL_PASSWORD=mysql_password
+MYSQL_PORT=3306
+MYSQL_ROOT_PASSWORD=test123!
+MYSQL_HOST=db
+
+ALLOWED_HOSTS='127.0.0.1,localhost'
+CORS_ORIGIN_WHITELIST='http://127.0.0.1:3000,http://localhost:3000'
+CSRF_TRUSTED_ORIGINS='http://127.0.0.1:8000,http://localhost:8000'
+
+SECRET_KEY='secret_key'
+DEBUG='True'
+
+# Add the following to your local .env file. They will be used in the CI process
+# and you can largely forget about them, but including them in your .env file
+# will act like a safe default and help suppress warnings.
+REGISTRY=""
+TAG=""

.github/workflows/build-and-deploy.yaml

@@ -0,0 +1,60 @@
+name: 'Build and deploy application containers'
+on:
+  push:
+jobs:
+  build-tag-push-deploy:
+    runs-on: ubuntu-latest
+    # CI/CD will run on these branches
+    if: >
+      github.ref == 'refs/heads/main' ||
+      github.ref == 'refs/heads/staging' ||
+      github.ref == 'refs/heads/development'
+
+    strategy:
+      matrix:
+        # Specify the docker-compose services to build images from. These should match the service
+        # names in the docker-compose.yml file.
+        service: [epwebapp, epnginx]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Login to GitHub container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: cmu-delphi-deploy-machine
+          password: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
+      - name: Create container image tags
+        id: image-tag
+        run: |
+          baseRef="${GITHUB_REF#*/}"
+          baseRef="${baseRef#*/}"
+          case "${baseRef}" in
+          main)
+            image_tag="latest"
+            ;;
+          *)
+            image_tag="${baseRef//\//_}" # replace `/` with `_` in branch name
+            ;;
+          esac
+          echo "IMAGE_TAG=${image_tag}" >> $GITHUB_OUTPUT
+      - name: Copy env file
+        run: |
+          cp ./.ci.env ./.env
+      - name: Set up docker-compose
+        uses: ndeloof/[email protected]
+      - name: docker-compose build --push
+        run: |
+          docker-compose build --push ${{ matrix.service }}
+        env:
+          TAG: ":${{ steps.image-tag.outputs.IMAGE_TAG }}"
+          REGISTRY: "ghcr.io/${{ github.repository_owner }}/"
+      - name: docker-compose down
+        run: |
+          docker-compose down
+      - name: Trigger smee.io webhook to pull new container images
+        run: |
+          curl -H "Authorization: Bearer ${{ secrets.DELPHI_DEPLOY_WEBHOOK_TOKEN }}" \
+               -X POST ${{ secrets.DELPHI_DEPLOY_WEBHOOK_URL }} \
+               -H "Content-Type: application/x-www-form-urlencoded" \
+               -d "repository=ghcr.io/${{ github.repository }}-${{ matrix.service }}&tag=${{ steps.image-tag.outputs.IMAGE_TAG }}"

src/assets/js/indicatorSetsTable.js

@@ -44,11 +44,10 @@ new DataTable.Buttons(table, {
 table.buttons(0, null).container().appendTo("#colvis");
 
 function format(indicatorSetId, relatedIndicators, indicatorSetDescription) {
-    console.lopg;
     var indicators = relatedIndicators.filter(
         (indicator) => indicator.indicator_set === indicatorSetId
     );
-    var disabled, restricted;
+    var disabled, restricted, sourceType;
 
     if (indicators.length > 0) {
         var data = `<p style="width: 40%;">${indicatorSetDescription}</p>`;
@@ -71,6 +70,7 @@ function format(indicatorSetId, relatedIndicators, indicatorSetDescription) {
             var checkboxTitle = "";
             checked = checked ? "checked" : "";
             disabled = indicator.endpoint ? "" : "disabled";
+            sourceType = indicator.source_type;
             var restricted = indicator.restricted != "No";
             if (disabled === "disabled") {
                 checkboxTitle =
@@ -92,11 +92,19 @@ function format(indicatorSetId, relatedIndicators, indicatorSetDescription) {
         });
         tableMarkup += "</tbody></table>";
         if (disabled === "disabled" || restricted) {
-            data +=
-                `<div class="alert alert-warning" data-mdb-alert-init role="alert">` +
-                `   <div>This indicator set is available via the <a href="https://cmu-delphi.github.io/delphi-epidata/">Epidata API</a>, and directly via <a href="https://delphi.cmu.edu/epivis/">Epivis</a>, but is not yet available via this interface.</div>` +
-                "</div>";
+            if (sourceType === "non_delphi") {
+                data +=
+                    `<div class="alert alert-warning" data-mdb-alert-init role="alert">` +
+                    `   <div>This indicator set is not available via Delphi.  It is included here for general discoverability only, and may or may not be available from the Original Data Provider.</div>` +
+                    "</div>";
+            } else {
+                data +=
+                    `<div class="alert alert-warning" data-mdb-alert-init role="alert">` +
+                    `   <div>This indicator set is available via the <a href="https://cmu-delphi.github.io/delphi-epidata/">Epidata API</a>, and directly via <a href="https://delphi.cmu.edu/epivis/">Epivis</a>, but is not yet available via this interface.</div>` +
+                    "</div>";
+            }
         }
+
         data += tableMarkup;
     } else {
         data = "<p>No available indicators yet.</p>";

src/indicators/migrations/0004_alter_indicator_base.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.7 on 2025-05-05 16:22
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('indicators', '0003_alter_indicator_geographic_scope_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='indicator',
+            name='base',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='base_for', to='indicators.indicator', verbose_name='Base Indicator'),
+        ),
+    ]

src/indicators/migrations/0005_indicator_source_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.7 on 2025-05-05 19:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('indicators', '0004_alter_indicator_base'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='indicator',
+            name='source_type',
+            field=models.CharField(blank=True, choices=[('covidcast', 'Covidcast'), ('other_endpoint', 'Other Endpoint'), ('non_delphi', 'Non Delphi')], default='covidcast', help_text='Type of source for the indicator', max_length=255, null=True, verbose_name='Source Type'),
+        ),
+    ]

src/indicators/models.py

@@ -4,6 +4,13 @@
 from django.db import models
 
 
+SOURCE_TYPES = [
+    ("covidcast", "Covidcast"),
+    ("other_endpoint", "Other Endpoint"),
+    ("non_delphi", "Non Delphi"),
+]
+
+
 # Create your models here.
 class IndicatorType(models.Model):
 
@@ -122,9 +129,7 @@ def display_name(self):
 
 class Indicator(models.Model):
 
-    name: models.CharField = models.CharField(
-        verbose_name="Name", max_length=255
-    )
+    name: models.CharField = models.CharField(verbose_name="Name", max_length=255)
     display_name: models.CharField = models.CharField(
         verbose_name="Display Name", max_length=255, blank=True
     )
@@ -367,7 +372,7 @@ class Indicator(models.Model):
         "indicators.Indicator",
         verbose_name="Base Indicator",
         related_name="base_for",
-        on_delete=models.PROTECT,
+        on_delete=models.CASCADE,
         null=True,
         blank=True,
     )
@@ -380,6 +385,16 @@ class Indicator(models.Model):
         blank=True,
     )
 
+    source_type: models.CharField = models.CharField(
+        verbose_name="Source Type",
+        max_length=255,
+        choices=SOURCE_TYPES,
+        default="covidcast",
+        help_text="Type of source for the indicator",
+        blank=True,
+        null=True,
+    )
+
     class Meta:
         verbose_name = "Indicator"
         verbose_name_plural = "Indicators"
@@ -397,7 +412,8 @@ class Meta:
                 fields=["name", "source"], name="unique_indicator_name"
             ),
             models.UniqueConstraint(
-                fields=["name", "indicator_set"], name="unique_indicator_indicator_set_name"
+                fields=["name", "indicator_set"],
+                name="unique_indicator_indicator_set_name",
             ),
         ]
 

src/indicators/resources.py

@@ -428,6 +428,10 @@ def before_import_row(self, row, **kwargs) -> None:
     def after_import_row(self, row, row_result, **kwargs):
         process_indicator_geography(row)
 
+    def after_save_instance(self, instance, row, **kwargs):
+        instance.source_type = "covidcast"
+        instance.save()
+
     def skip_row(self, instance, original, row, import_validation_errors=None):
         if not row["Include in indicator app"]:
             return True
@@ -601,6 +605,10 @@ def skip_row(self, instance, original, row, import_validation_errors=None):
     def after_import_row(self, row, row_result, **kwargs):
         process_indicator_geography(row)
 
+    def after_save_instance(self, instance, row, **kwargs):
+        instance.source_type = "other_endpoint"
+        instance.save()
+
 
 class NonDelphiIndicatorResource(resources.ModelResource):
 
@@ -633,3 +641,7 @@ def before_import_row(self, row, **kwargs) -> None:
     def skip_row(self, instance, original, row, import_validation_errors=None):
         if not row["Include in indicator app"]:
             return True
+
+    def after_save_instance(self, instance, row, **kwargs):
+        instance.source_type = "non_delphi"
+        instance.save()

src/indicatorsets/filters.py

@@ -7,24 +7,16 @@
 
 
 from indicatorsets.models import IndicatorSet
-from indicatorsets.utils import get_list_of_indicators_filtered_by_geo
+from indicatorsets.utils import (
+    get_list_of_indicators_filtered_by_geo,
+    get_original_data_provider_choices,
+)
 from indicators.models import Indicator
 from base.models import Pathogen, GeographicScope, Geography, SeverityPyramidRung
 
 
 logger = logging.getLogger(__name__)
 
-try:
-    ORIGINAL_DATA_PROVIDER_CHOICES = [
-        (el, el)
-        for el in set(
-            IndicatorSet.objects.values_list("original_data_provider", flat=True)
-        )
-    ]
-except Exception as e:
-    ORIGINAL_DATA_PROVIDER_CHOICES = [("", "No original data provider available")]
-    print(f"Error fetching original data provider choices: {e}")
-
 
 class IndicatorSetFilter(django_filters.FilterSet):
 
@@ -60,7 +52,7 @@ class IndicatorSetFilter(django_filters.FilterSet):
 
     original_data_provider = django_filters.MultipleChoiceFilter(
         field_name="original_data_provider",
-        choices=ORIGINAL_DATA_PROVIDER_CHOICES,
+        choices=get_original_data_provider_choices,
         widget=QueryArrayWidget,
         lookup_expr="exact",
         required=False,

src/indicatorsets/forms.py

@@ -2,18 +2,7 @@
 
 from base.models import Pathogen, GeographicScope, Geography, SeverityPyramidRung
 from indicatorsets.models import IndicatorSet
-
-
-try:
-    ORIGINAL_DATA_PROVIDER_CHOICES = [
-        (el, el)
-        for el in set(
-            IndicatorSet.objects.values_list("original_data_provider", flat=True)
-        )
-    ]
-except Exception as e:
-    ORIGINAL_DATA_PROVIDER_CHOICES = [("", "No original data provider available")]
-    print(f"Error fetching original data provider choices: {e}")
+from indicatorsets.utils import get_original_data_provider_choices
 
 
 class IndicatorSetFilterForm(forms.ModelForm):
@@ -44,7 +33,7 @@ class IndicatorSetFilterForm(forms.ModelForm):
     )
 
     original_data_provider = forms.ChoiceField(
-        choices=ORIGINAL_DATA_PROVIDER_CHOICES,
+        choices=get_original_data_provider_choices,
         widget=forms.CheckboxSelectMultiple(),
     )
 

src/indicatorsets/utils.py

@@ -5,6 +5,7 @@
 import requests
 from django.conf import settings
 from epiweeks import Week
+from indicatorsets.models import IndicatorSet
 
 
 def list_to_dict(lst):
@@ -53,3 +54,12 @@ def get_epiweek(start_date, end_date):
     end_date = Week.fromdate(end_date)
     end_date = f"{end_date.year}{end_date.week if end_date.week >= 10 else '0' + str(end_date.week)}"
     return [start_date, end_date]
+
+
+def get_original_data_provider_choices():
+    return [
+        (el, el)
+        for el in IndicatorSet.objects.values_list("original_data_provider", flat=True)
+        .order_by("original_data_provider")
+        .distinct()
+    ]

src/indicatorsets/views.py

@@ -89,6 +89,7 @@ def get_related_indicators(self, queryset, indicator_set_ids: list):
                     "description": indicator.description if indicator.description else "",
                     "member_description": indicator.member_description if indicator.member_description else indicator.description,
                     "restricted": indicator.indicator_set.dua_required if indicator.indicator_set else "",
+                    "source_type": indicator.source_type,
                 }
             )
         return related_indicators

src/staticfiles/js/indicatorSetsTable.js

@@ -44,11 +44,10 @@ new DataTable.Buttons(table, {
 table.buttons(0, null).container().appendTo("#colvis");
 
 function format(indicatorSetId, relatedIndicators, indicatorSetDescription) {
-    console.lopg;
     var indicators = relatedIndicators.filter(
         (indicator) => indicator.indicator_set === indicatorSetId
     );
-    var disabled, restricted;
+    var disabled, restricted, sourceType;
 
     if (indicators.length > 0) {
         var data = `<p style="width: 40%;">${indicatorSetDescription}</p>`;
@@ -71,6 +70,7 @@ function format(indicatorSetId, relatedIndicators, indicatorSetDescription) {
             var checkboxTitle = "";
             checked = checked ? "checked" : "";
             disabled = indicator.endpoint ? "" : "disabled";
+            sourceType = indicator.source_type;
             var restricted = indicator.restricted != "No";
             if (disabled === "disabled") {
                 checkboxTitle =
@@ -92,11 +92,19 @@ function format(indicatorSetId, relatedIndicators, indicatorSetDescription) {
         });
         tableMarkup += "</tbody></table>";
         if (disabled === "disabled" || restricted) {
-            data +=
-                `<div class="alert alert-warning" data-mdb-alert-init role="alert">` +
-                `   <div>This indicator set is available via the <a href="https://cmu-delphi.github.io/delphi-epidata/">Epidata API</a>, and directly via <a href="https://delphi.cmu.edu/epivis/">Epivis</a>, but is not yet available via this interface.</div>` +
-                "</div>";
+            if (sourceType === "non_delphi") {
+                data +=
+                    `<div class="alert alert-warning" data-mdb-alert-init role="alert">` +
+                    `   <div>This indicator set is not available via Delphi.  It is included here for general discoverability only, and may or may not be available from the Original Data Provider.</div>` +
+                    "</div>";
+            } else {
+                data +=
+                    `<div class="alert alert-warning" data-mdb-alert-init role="alert">` +
+                    `   <div>This indicator set is available via the <a href="https://cmu-delphi.github.io/delphi-epidata/">Epidata API</a>, and directly via <a href="https://delphi.cmu.edu/epivis/">Epivis</a>, but is not yet available via this interface.</div>` +
+                    "</div>";
+            }
         }
+
         data += tableMarkup;
     } else {
         data = "<p>No available indicators yet.</p>";