Skip to content

Commit 77a7368

Browse files
minhkhulminhkhul
authored
Merge pull request #10 from cmu-delphi/SD-24
SD-24
2 parents 743743f + 4a132bd commit 77a7368

File tree

13 files changed

+506
-131
lines changed

13 files changed

+506
-131
lines changed

‎Pipfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ factory-boy = "*"
2121
linkpreview = "*"
2222
django-redis = "*"
2323
mysqlclient = "*"
24+
django-import-export = "*"
2425

2526
[dev-packages]
2627
flake8 = "*"

‎Pipfile.lock

Lines changed: 182 additions & 113 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/datasources/admin.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,25 @@
11
from django.contrib import admin
2+
from import_export.admin import ImportExportModelAdmin
23

34
from datasources.models import DataSource, SourceSubdivision
5+
from datasources.resources import DataSourceResource, SourceSubdivisionResource
46

57

68
@admin.register(SourceSubdivision)
7-
class SourceSubdivisionAdmin(admin.ModelAdmin):
9+
class SourceSubdivisionAdmin(ImportExportModelAdmin):
810
"""
911
Admin interface for managing source subdivision objects.
1012
"""
1113
list_display = ('name', 'db_source')
1214
search_fields = ('name', 'db_source')
15+
resource_classes = [SourceSubdivisionResource]
1316

1417

1518
@admin.register(DataSource)
16-
class DataSourceAdmin(admin.ModelAdmin):
19+
class DataSourceAdmin(ImportExportModelAdmin):
1720
"""
1821
Admin interface for managing data source objects.
1922
"""
2023
list_display = ('name',)
2124
search_fields = ('name', 'source_subdivision__db_source', 'source_subdivision__name', 'description')
25+
resource_classes = [DataSourceResource]

‎src/datasources/migrations/0002_alter_sourcesubdivision_db_source.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Generated by Django 4.2.2 on 2023-07-03 12:28
2+
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
8+
dependencies = [
9+
('datasources', '0001_initial'),
10+
]
11+
12+
operations = [
13+
migrations.AlterField(
14+
model_name='sourcesubdivision',
15+
name='db_source',
16+
field=models.CharField(help_text='DB Source', max_length=128),
17+
),
18+
]

‎src/datasources/models.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ class SourceSubdivision(models.Model):
2525
db_source = models.CharField(
2626
help_text=_('DB Source'),
2727
max_length=128,
28-
unique=True
2928
)
3029
links = models.ManyToManyField(
3130
'base.Link',

‎src/datasources/resources.py

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
import re
2+
3+
from import_export import resources
4+
from import_export.fields import Field, widgets
5+
6+
from base.models import Link, LinkTypeChoices
7+
from datasources.models import DataSource, SourceSubdivision
8+
9+
10+
class DataSourceResource(resources.ModelResource):
11+
"""
12+
A resource for importing and exporting data sources using the Django ORM.
13+
"""
14+
name = Field(attribute='name', column_name='Name')
15+
display_name = Field(attribute='display_name', column_name='Name')
16+
description = Field(attribute='description', column_name='Description')
17+
source_license = Field(attribute='source_license', column_name='License')
18+
links = Field(
19+
attribute='links',
20+
column_name='Links',
21+
widget=widgets.ManyToManyWidget(Link, field='url', separator='|'),
22+
)
23+
source_subdivisions = Field(
24+
attribute='source_subdivisions',
25+
column_name='Name',
26+
widget=widgets.ForeignKeyWidget(DataSource, field='name'),
27+
)
28+
29+
class Meta:
30+
model = DataSource
31+
fields = ('name', 'display_name', 'description', 'source_license', 'links')
32+
import_id_fields = ['name']
33+
34+
def before_import_row(self, row, **kwargs):
35+
"""
36+
Hook called before importing each row. Modifies 'Links' column to include
37+
any additional links specified in 'DUA' or 'Link' columns.
38+
"""
39+
self.process_links(row)
40+
self.process_subdivisions(row)
41+
42+
def process_subdivisions(self, row):
43+
if row['Source Subdivision']:
44+
data_source = DataSource.objects.get(name=row['Name'])
45+
source_subdivision, created = SourceSubdivision.objects.get_or_create(
46+
name=row['Source Subdivision'],
47+
defaults={
48+
'display_name': row['Source Subdivision'],
49+
'description': row['Description'],
50+
'db_source': row['DB Source'],
51+
'data_source': data_source
52+
}
53+
)
54+
55+
def process_links(self, row):
56+
row['Links'] = ''
57+
if row['DUA']:
58+
link, created = Link.objects.get_or_create(url=row['DUA'], defaults={'link_type': LinkTypeChoices.DUA})
59+
row['Links'] += row['Links'] + f'|{link.url}'
60+
61+
if row['Link']:
62+
pattern = r'\[(.*?)\]\((.*?)\)'
63+
pattern_match = re.search(pattern, row['Link'])
64+
link_type_mapping = {choice.label: choice.value for choice in LinkTypeChoices}
65+
link_type = link_type_mapping[pattern_match.group(1)]
66+
link_url = pattern_match.group(2)
67+
link, created = Link.objects.get_or_create(url=link_url, link_type=link_type)
68+
row['Links'] += row['Links'] + f'|{link.url}'
69+
70+
71+
class SourceSubdivisionResource(resources.ModelResource):
72+
name = Field(attribute='name', column_name='Source Subdivision')
73+
display_name = Field(attribute='display_name', column_name='Source Subdivision')
74+
description = Field(attribute='description', column_name='Description')
75+
db_source = Field(attribute='db_source', column_name='DB Source')
76+
data_source = Field(
77+
attribute='data_source',
78+
column_name='Name',
79+
widget=widgets.ForeignKeyWidget(DataSource, field='name'),
80+
)
81+
links = Field(
82+
attribute='links',
83+
column_name='Links',
84+
widget=widgets.ManyToManyWidget(Link, field='url', separator='|'),
85+
)
86+
87+
class Meta:
88+
model = SourceSubdivision
89+
fields = ('name', 'display_name', 'description', 'links', 'data_source')
90+
import_id_fields = ['name']
91+
skip_unchanged = True
92+
93+
def before_import_row(self, row, **kwargs) -> None:
94+
"""
95+
Hook called before importing each row. Modifies 'Links' column to include
96+
any additional links specified in 'DUA' or 'Link' columns.
97+
"""
98+
row['Links'] = ''
99+
if row['DUA']:
100+
link, created = Link.objects.get_or_create(url=row['DUA'], link_type=LinkTypeChoices.DUA)
101+
row['Links'] += row['Links'] + f'|{link.url}'
102+
if row['Link']:
103+
pattern = r'\[(.*?)\]\((.*?)\)'
104+
pattern_match = re.search(pattern, row['Link'])
105+
link_type_mapping = {choice.label: choice.value for choice in LinkTypeChoices}
106+
link_type = link_type_mapping[pattern_match.group(1)]
107+
link_url = pattern_match.group(2)
108+
link, created = Link.objects.get_or_create(url=link_url, link_type=link_type)
109+
row['Links'] += row['Links'] + f'|{link.url}'

‎src/signal_documentation/settings.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
'health_check.cache',
6565
'health_check.storage',
6666
'health_check.contrib.migrations',
67+
'import_export',
6768
]
6869

6970
LOCAL_APPS = [

‎src/signals/admin.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
from django.contrib import admin
2+
from import_export.admin import ImportExportModelAdmin
23

34
from signals.models import (
45
Geography,
56
Pathogen,
67
Signal,
78
SignalType,
89
)
10+
from signals.resources import SignalResource
911

1012

1113
@admin.register(Geography)
@@ -36,7 +38,7 @@ class SignalTypeAdmin(admin.ModelAdmin):
3638

3739

3840
@admin.register(Signal)
39-
class SignalAdmin(admin.ModelAdmin):
41+
class SignalAdmin(ImportExportModelAdmin):
4042
"""
4143
Admin interface for managing signal objects.
4244
"""
@@ -53,3 +55,4 @@ class SignalAdmin(admin.ModelAdmin):
5355
'has_stderr',
5456
'has_sample_size',
5557
)
58+
resource_classes = [SignalResource]

‎src/signals/migrations/0002_remove_signal_pathogen_signal_pathogen.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Generated by Django 4.2.3 on 2023-07-07 11:41
2+
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
8+
dependencies = [
9+
('signals', '0001_initial'),
10+
]
11+
12+
operations = [
13+
migrations.RemoveField(
14+
model_name='signal',
15+
name='pathogen',
16+
),
17+
migrations.AddField(
18+
model_name='signal',
19+
name='pathogen',
20+
field=models.ManyToManyField(help_text='Pathogen/Disease Area', related_name='signals', to='signals.pathogen'),
21+
),
22+
]

‎src/signals/models.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -144,12 +144,10 @@ class Signal(models.Model):
144144
null=True,
145145
blank=True
146146
)
147-
pathogen = models.ForeignKey(
147+
pathogen = models.ManyToManyField(
148148
'signals.Pathogen',
149149
related_name='signals',
150150
help_text=_('Pathogen/Disease Area'),
151-
on_delete=models.SET_NULL,
152-
null=True
153151
)
154152
signal_type = models.ManyToManyField(
155153
'signals.SignalType',
@@ -232,7 +230,7 @@ class Signal(models.Model):
232230
'datasources.SourceSubdivision',
233231
related_name='signals',
234232
help_text=_('Source Subdivision'),
235-
on_delete=models.PROTECT
233+
on_delete=models.PROTECT,
236234
)
237235

238236
def __str__(self) -> str:

0 commit comments

Comments
 (0)