Skip to content

Commit e0d8e22

Browse files
dmytrotskodmytrotsko
authored
Merge pull request #7 from cmu-delphi/staging
Merge latest from staging
2 parents 4bc7cda + 4a5d53b commit e0d8e22

File tree

17 files changed

+311
-43
lines changed

17 files changed

+311
-43
lines changed

‎.ci.env

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# This file is used to allow CI to start the compose services. It will typcially
2+
# not need to be modified.
3+
4+
MYSQL_DATABASE=mysql_database
5+
MYSQL_USER=mysql_user
6+
MYSQL_PASSWORD=mysql_password
7+
MYSQL_PORT=3306
8+
MYSQL_ROOT_PASSWORD=test123!
9+
MYSQL_HOST=db
10+
11+
ALLOWED_HOSTS='127.0.0.1,localhost'
12+
CORS_ORIGIN_WHITELIST='http://127.0.0.1:3000,http://localhost:3000'
13+
CSRF_TRUSTED_ORIGINS='http://127.0.0.1:8000,http://localhost:8000'
14+
15+
SECRET_KEY='secret_key'
16+
DEBUG='True'
17+
18+
# Add the following to your local .env file. They will be used in the CI process
19+
# and you can largely forget about them, but including them in your .env file
20+
# will act like a safe default and help suppress warnings.
21+
REGISTRY=""
22+
TAG=""

‎.github/workflows/build-and-deploy.yaml

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: 'Build and deploy application containers'
2+
on:
3+
push:
4+
jobs:
5+
build-tag-push-deploy:
6+
runs-on: ubuntu-latest
7+
# CI/CD will run on these branches
8+
if: >
9+
github.ref == 'refs/heads/main' ||
10+
github.ref == 'refs/heads/staging' ||
11+
github.ref == 'refs/heads/development'
12+
13+
strategy:
14+
matrix:
15+
# Specify the docker-compose services to build images from. These should match the service
16+
# names in the docker-compose.yml file.
17+
service: [epwebapp, epnginx]
18+
steps:
19+
- name: Checkout
20+
uses: actions/checkout@v3
21+
- name: Login to GitHub container registry
22+
uses: docker/login-action@v1
23+
with:
24+
registry: ghcr.io
25+
username: cmu-delphi-deploy-machine
26+
password: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
27+
- name: Create container image tags
28+
id: image-tag
29+
run: |
30+
baseRef="${GITHUB_REF#*/}"
31+
baseRef="${baseRef#*/}"
32+
case "${baseRef}" in
33+
main)
34+
image_tag="latest"
35+
;;
36+
*)
37+
image_tag="${baseRef//\//_}" # replace `/` with `_` in branch name
38+
;;
39+
esac
40+
echo "IMAGE_TAG=${image_tag}" >> $GITHUB_OUTPUT
41+
- name: Copy env file
42+
run: |
43+
cp ./.ci.env ./.env
44+
- name: Set up docker-compose
45+
uses: ndeloof/[email protected]
46+
- name: docker-compose build --push
47+
run: |
48+
docker-compose build --push ${{ matrix.service }}
49+
env:
50+
TAG: ":${{ steps.image-tag.outputs.IMAGE_TAG }}"
51+
REGISTRY: "ghcr.io/${{ github.repository_owner }}/"
52+
- name: docker-compose down
53+
run: |
54+
docker-compose down
55+
- name: Trigger smee.io webhook to pull new container images
56+
run: |
57+
curl -H "Authorization: Bearer ${{ secrets.DELPHI_DEPLOY_WEBHOOK_TOKEN }}" \
58+
-X POST ${{ secrets.DELPHI_DEPLOY_WEBHOOK_URL }} \
59+
-H "Content-Type: application/x-www-form-urlencoded" \
60+
-d "repository=ghcr.io/${{ github.repository }}-${{ matrix.service }}&tag=${{ steps.image-tag.outputs.IMAGE_TAG }}"

‎src/assets/js/indicatorSetsTable.js

+14-6
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,10 @@ new DataTable.Buttons(table, {
4444
table.buttons(0, null).container().appendTo("#colvis");
4545

4646
function format(indicatorSetId, relatedIndicators, indicatorSetDescription) {
47-
console.lopg;
4847
var indicators = relatedIndicators.filter(
4948
(indicator) => indicator.indicator_set === indicatorSetId
5049
);
51-
var disabled, restricted;
50+
var disabled, restricted, sourceType;
5251

5352
if (indicators.length > 0) {
5453
var data = `<p style="width: 40%;">${indicatorSetDescription}</p>`;
@@ -71,6 +70,7 @@ function format(indicatorSetId, relatedIndicators, indicatorSetDescription) {
7170
var checkboxTitle = "";
7271
checked = checked ? "checked" : "";
7372
disabled = indicator.endpoint ? "" : "disabled";
73+
sourceType = indicator.source_type;
7474
var restricted = indicator.restricted != "No";
7575
if (disabled === "disabled") {
7676
checkboxTitle =
@@ -92,11 +92,19 @@ function format(indicatorSetId, relatedIndicators, indicatorSetDescription) {
9292
});
9393
tableMarkup += "</tbody></table>";
9494
if (disabled === "disabled" || restricted) {
95-
data +=
96-
`<div class="alert alert-warning" data-mdb-alert-init role="alert">` +
97-
` <div>This indicator set is available via the <a href="https://cmu-delphi.github.io/delphi-epidata/">Epidata API</a>, and directly via <a href="https://delphi.cmu.edu/epivis/">Epivis</a>, but is not yet available via this interface.</div>` +
98-
"</div>";
95+
if (sourceType === "non_delphi") {
96+
data +=
97+
`<div class="alert alert-warning" data-mdb-alert-init role="alert">` +
98+
` <div>This indicator set is not available via Delphi. It is included here for general discoverability only, and may or may not be available from the Original Data Provider.</div>` +
99+
"</div>";
100+
} else {
101+
data +=
102+
`<div class="alert alert-warning" data-mdb-alert-init role="alert">` +
103+
` <div>This indicator set is available via the <a href="https://cmu-delphi.github.io/delphi-epidata/">Epidata API</a>, and directly via <a href="https://delphi.cmu.edu/epivis/">Epivis</a>, but is not yet available via this interface.</div>` +
104+
"</div>";
105+
}
99106
}
107+
100108
data += tableMarkup;
101109
} else {
102110
data = "<p>No available indicators yet.</p>";

‎src/indicators/migrations/0004_alter_indicator_base.py

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Generated by Django 5.0.7 on 2025-05-05 16:22
2+
3+
import django.db.models.deletion
4+
from django.db import migrations, models
5+
6+
7+
class Migration(migrations.Migration):
8+
9+
dependencies = [
10+
('indicators', '0003_alter_indicator_geographic_scope_and_more'),
11+
]
12+
13+
operations = [
14+
migrations.AlterField(
15+
model_name='indicator',
16+
name='base',
17+
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='base_for', to='indicators.indicator', verbose_name='Base Indicator'),
18+
),
19+
]

‎src/indicators/migrations/0005_indicator_source_type.py

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Generated by Django 5.0.7 on 2025-05-05 19:33
2+
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
8+
dependencies = [
9+
('indicators', '0004_alter_indicator_base'),
10+
]
11+
12+
operations = [
13+
migrations.AddField(
14+
model_name='indicator',
15+
name='source_type',
16+
field=models.CharField(blank=True, choices=[('covidcast', 'Covidcast'), ('other_endpoint', 'Other Endpoint'), ('non_delphi', 'Non Delphi')], default='covidcast', help_text='Type of source for the indicator', max_length=255, null=True, verbose_name='Source Type'),
17+
),
18+
]

‎src/indicators/models.py

+21-5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,13 @@
44
from django.db import models
55

66

7+
SOURCE_TYPES = [
8+
("covidcast", "Covidcast"),
9+
("other_endpoint", "Other Endpoint"),
10+
("non_delphi", "Non Delphi"),
11+
]
12+
13+
714
# Create your models here.
815
class IndicatorType(models.Model):
916

@@ -122,9 +129,7 @@ def display_name(self):
122129

123130
class Indicator(models.Model):
124131

125-
name: models.CharField = models.CharField(
126-
verbose_name="Name", max_length=255
127-
)
132+
name: models.CharField = models.CharField(verbose_name="Name", max_length=255)
128133
display_name: models.CharField = models.CharField(
129134
verbose_name="Display Name", max_length=255, blank=True
130135
)
@@ -367,7 +372,7 @@ class Indicator(models.Model):
367372
"indicators.Indicator",
368373
verbose_name="Base Indicator",
369374
related_name="base_for",
370-
on_delete=models.PROTECT,
375+
on_delete=models.CASCADE,
371376
null=True,
372377
blank=True,
373378
)
@@ -380,6 +385,16 @@ class Indicator(models.Model):
380385
blank=True,
381386
)
382387

388+
source_type: models.CharField = models.CharField(
389+
verbose_name="Source Type",
390+
max_length=255,
391+
choices=SOURCE_TYPES,
392+
default="covidcast",
393+
help_text="Type of source for the indicator",
394+
blank=True,
395+
null=True,
396+
)
397+
383398
class Meta:
384399
verbose_name = "Indicator"
385400
verbose_name_plural = "Indicators"
@@ -397,7 +412,8 @@ class Meta:
397412
fields=["name", "source"], name="unique_indicator_name"
398413
),
399414
models.UniqueConstraint(
400-
fields=["name", "indicator_set"], name="unique_indicator_indicator_set_name"
415+
fields=["name", "indicator_set"],
416+
name="unique_indicator_indicator_set_name",
401417
),
402418
]
403419

‎src/indicators/resources.py

+12
Original file line numberDiff line numberDiff line change
@@ -428,6 +428,10 @@ def before_import_row(self, row, **kwargs) -> None:
428428
def after_import_row(self, row, row_result, **kwargs):
429429
process_indicator_geography(row)
430430

431+
def after_save_instance(self, instance, row, **kwargs):
432+
instance.source_type = "covidcast"
433+
instance.save()
434+
431435
def skip_row(self, instance, original, row, import_validation_errors=None):
432436
if not row["Include in indicator app"]:
433437
return True
@@ -601,6 +605,10 @@ def skip_row(self, instance, original, row, import_validation_errors=None):
601605
def after_import_row(self, row, row_result, **kwargs):
602606
process_indicator_geography(row)
603607

608+
def after_save_instance(self, instance, row, **kwargs):
609+
instance.source_type = "other_endpoint"
610+
instance.save()
611+
604612

605613
class NonDelphiIndicatorResource(resources.ModelResource):
606614

@@ -633,3 +641,7 @@ def before_import_row(self, row, **kwargs) -> None:
633641
def skip_row(self, instance, original, row, import_validation_errors=None):
634642
if not row["Include in indicator app"]:
635643
return True
644+
645+
def after_save_instance(self, instance, row, **kwargs):
646+
instance.source_type = "non_delphi"
647+
instance.save()

‎src/indicatorsets/filters.py

+5-13
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,16 @@
77

88

99
from indicatorsets.models import IndicatorSet
10-
from indicatorsets.utils import get_list_of_indicators_filtered_by_geo
10+
from indicatorsets.utils import (
11+
get_list_of_indicators_filtered_by_geo,
12+
get_original_data_provider_choices,
13+
)
1114
from indicators.models import Indicator
1215
from base.models import Pathogen, GeographicScope, Geography, SeverityPyramidRung
1316

1417

1518
logger = logging.getLogger(__name__)
1619

17-
try:
18-
ORIGINAL_DATA_PROVIDER_CHOICES = [
19-
(el, el)
20-
for el in set(
21-
IndicatorSet.objects.values_list("original_data_provider", flat=True)
22-
)
23-
]
24-
except Exception as e:
25-
ORIGINAL_DATA_PROVIDER_CHOICES = [("", "No original data provider available")]
26-
print(f"Error fetching original data provider choices: {e}")
27-
2820

2921
class IndicatorSetFilter(django_filters.FilterSet):
3022

@@ -60,7 +52,7 @@ class IndicatorSetFilter(django_filters.FilterSet):
6052

6153
original_data_provider = django_filters.MultipleChoiceFilter(
6254
field_name="original_data_provider",
63-
choices=ORIGINAL_DATA_PROVIDER_CHOICES,
55+
choices=get_original_data_provider_choices,
6456
widget=QueryArrayWidget,
6557
lookup_expr="exact",
6658
required=False,

‎src/indicatorsets/forms.py

+2-13
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,7 @@
22

33
from base.models import Pathogen, GeographicScope, Geography, SeverityPyramidRung
44
from indicatorsets.models import IndicatorSet
5-
6-
7-
try:
8-
ORIGINAL_DATA_PROVIDER_CHOICES = [
9-
(el, el)
10-
for el in set(
11-
IndicatorSet.objects.values_list("original_data_provider", flat=True)
12-
)
13-
]
14-
except Exception as e:
15-
ORIGINAL_DATA_PROVIDER_CHOICES = [("", "No original data provider available")]
16-
print(f"Error fetching original data provider choices: {e}")
5+
from indicatorsets.utils import get_original_data_provider_choices
176

187

198
class IndicatorSetFilterForm(forms.ModelForm):
@@ -44,7 +33,7 @@ class IndicatorSetFilterForm(forms.ModelForm):
4433
)
4534

4635
original_data_provider = forms.ChoiceField(
47-
choices=ORIGINAL_DATA_PROVIDER_CHOICES,
36+
choices=get_original_data_provider_choices,
4837
widget=forms.CheckboxSelectMultiple(),
4938
)
5039

‎src/indicatorsets/utils.py

+10
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import requests
66
from django.conf import settings
77
from epiweeks import Week
8+
from indicatorsets.models import IndicatorSet
89

910

1011
def list_to_dict(lst):
@@ -53,3 +54,12 @@ def get_epiweek(start_date, end_date):
5354
end_date = Week.fromdate(end_date)
5455
end_date = f"{end_date.year}{end_date.week if end_date.week >= 10 else '0' + str(end_date.week)}"
5556
return [start_date, end_date]
57+
58+
59+
def get_original_data_provider_choices():
60+
return [
61+
(el, el)
62+
for el in IndicatorSet.objects.values_list("original_data_provider", flat=True)
63+
.order_by("original_data_provider")
64+
.distinct()
65+
]

‎src/indicatorsets/views.py

+1
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ def get_related_indicators(self, queryset, indicator_set_ids: list):
8989
"description": indicator.description if indicator.description else "",
9090
"member_description": indicator.member_description if indicator.member_description else indicator.description,
9191
"restricted": indicator.indicator_set.dua_required if indicator.indicator_set else "",
92+
"source_type": indicator.source_type,
9293
}
9394
)
9495
return related_indicators

‎src/staticfiles/js/indicatorSetsTable.js

+14-6
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,10 @@ new DataTable.Buttons(table, {
4444
table.buttons(0, null).container().appendTo("#colvis");
4545

4646
function format(indicatorSetId, relatedIndicators, indicatorSetDescription) {
47-
console.lopg;
4847
var indicators = relatedIndicators.filter(
4948
(indicator) => indicator.indicator_set === indicatorSetId
5049
);
51-
var disabled, restricted;
50+
var disabled, restricted, sourceType;
5251

5352
if (indicators.length > 0) {
5453
var data = `<p style="width: 40%;">${indicatorSetDescription}</p>`;
@@ -71,6 +70,7 @@ function format(indicatorSetId, relatedIndicators, indicatorSetDescription) {
7170
var checkboxTitle = "";
7271
checked = checked ? "checked" : "";
7372
disabled = indicator.endpoint ? "" : "disabled";
73+
sourceType = indicator.source_type;
7474
var restricted = indicator.restricted != "No";
7575
if (disabled === "disabled") {
7676
checkboxTitle =
@@ -92,11 +92,19 @@ function format(indicatorSetId, relatedIndicators, indicatorSetDescription) {
9292
});
9393
tableMarkup += "</tbody></table>";
9494
if (disabled === "disabled" || restricted) {
95-
data +=
96-
`<div class="alert alert-warning" data-mdb-alert-init role="alert">` +
97-
` <div>This indicator set is available via the <a href="https://cmu-delphi.github.io/delphi-epidata/">Epidata API</a>, and directly via <a href="https://delphi.cmu.edu/epivis/">Epivis</a>, but is not yet available via this interface.</div>` +
98-
"</div>";
95+
if (sourceType === "non_delphi") {
96+
data +=
97+
`<div class="alert alert-warning" data-mdb-alert-init role="alert">` +
98+
` <div>This indicator set is not available via Delphi. It is included here for general discoverability only, and may or may not be available from the Original Data Provider.</div>` +
99+
"</div>";
100+
} else {
101+
data +=
102+
`<div class="alert alert-warning" data-mdb-alert-init role="alert">` +
103+
` <div>This indicator set is available via the <a href="https://cmu-delphi.github.io/delphi-epidata/">Epidata API</a>, and directly via <a href="https://delphi.cmu.edu/epivis/">Epivis</a>, but is not yet available via this interface.</div>` +
104+
"</div>";
105+
}
99106
}
107+
100108
data += tableMarkup;
101109
} else {
102110
data = "<p>No available indicators yet.</p>";

0 commit comments

Comments
 (0)