Skip to content

Commit d03ddb3

Browse files
dmytrotskodmytrotsko
authored
Merge pull request #143 from cmu-delphi/development
Fixed data import (some singals were not imported)
2 parents ecb1245 + 88ac639 commit d03ddb3

File tree

3 files changed

+36
-35
lines changed

3 files changed

+36
-35
lines changed

‎src/datasources/resources.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import re
22

33
from import_export import resources
4-
from import_export.fields import Field, widgets
4+
from import_export.fields import Field
5+
from import_export.widgets import ForeignKeyWidget
56

67
from base.models import Link
78
from datasources.models import DataSource, SourceSubdivision
@@ -65,7 +66,7 @@ class SourceSubdivisionResource(resources.ModelResource):
6566
data_source = Field(
6667
column_name="DB Source",
6768
attribute="data_source",
68-
widget=widgets.ForeignKeyWidget(DataSource, field="name"),
69+
widget=ForeignKeyWidget(DataSource, field="name"),
6970
)
7071

7172
class Meta:

‎src/signal_sets/resources.py

+5-4
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44

55

66
from import_export import resources
7-
from import_export.fields import Field, widgets
7+
from import_export.fields import Field
8+
from import_export.widgets import ForeignKeyWidget, ManyToManyWidget
89

910
from datasources.models import DataSource
1011
from signal_sets.models import SignalSet
@@ -68,7 +69,7 @@ def process_datasources(row) -> None:
6869
if row["Original Data Provider"]:
6970
data_source = row["Original Data Provider"]
7071
data_source_obj, _ = DataSource.objects.get_or_create(name=data_source, defaults={"display_name": data_source.capitalize()})
71-
row["Original Data Provider"] = data_source_obj
72+
row["Original Data Provider"] = data_source_obj.id
7273

7374

7475
def fix_boolean_fields(row) -> Any:
@@ -101,7 +102,7 @@ class SignalSetResource(resources.ModelResource):
101102
data_source = Field(
102103
attribute="data_source",
103104
column_name="Original Data Provider",
104-
widget=widgets.ForeignKeyWidget(DataSource, field="name"),
105+
widget=ForeignKeyWidget(DataSource),
105106
)
106107
endpoint = Field(attribute="endpoint", column_name="Endpoint")
107108
language = Field(attribute="language", column_name="Language (likely English) ")
@@ -116,7 +117,7 @@ class SignalSetResource(resources.ModelResource):
116117
geographic_scope = Field(
117118
attribute="geographic_scope",
118119
column_name="Geographic Coverage",
119-
widget=widgets.ForeignKeyWidget(GeographicScope),
120+
widget=ForeignKeyWidget(GeographicScope),
120121
)
121122
geographic_granularity = Field(
122123
attribute="geographic_granularity",

‎src/signals/resources.py

+28-29
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22

33
from import_export import resources
44
from import_export.results import RowResult
5-
from import_export.fields import Field, widgets
5+
from import_export.fields import Field
6+
from import_export.widgets import ForeignKeyWidget, ManyToManyWidget
67

78
from django.db.models import Max
89

@@ -63,7 +64,7 @@ def process_signal_type(row) -> None:
6364
if row["Indicator Type"]:
6465
signal_type = row["Indicator Type"]
6566
signal_type_obj, _ = SignalType.objects.get_or_create(name=signal_type)
66-
row["Indicator Type"] = signal_type_obj
67+
row["Indicator Type"] = signal_type_obj.id
6768

6869

6970
def process_format_type(row) -> None:
@@ -73,9 +74,7 @@ def process_format_type(row) -> None:
7374
if row["Format"]:
7475
format_type = row["Format"].strip()
7576
format_type_obj, _ = FormatType.objects.get_or_create(name=format_type)
76-
row["Format"] = format_type_obj
77-
else:
78-
row["Format"] = None
77+
row["Format"] = format_type_obj.id
7978

8079

8180
def process_severity_pyramid_rungs(row) -> None:
@@ -109,7 +108,7 @@ def process_category(row) -> None:
109108
if row["Category"]:
110109
category = row["Category"].strip()
111110
category_obj, _ = Category.objects.get_or_create(name=category)
112-
row["Category"] = category_obj
111+
row["Category"] = category_obj.id
113112

114113

115114
def process_geographic_scope(row) -> None:
@@ -131,7 +130,7 @@ def process_source(row) -> None:
131130
if row["Source Subdivision"]:
132131
source = row["Source Subdivision"]
133132
source_obj, _ = SourceSubdivision.objects.get_or_create(name=source)
134-
row["Source Subdivision"] = source_obj
133+
row["Source Subdivision"] = source_obj.id
135134

136135

137136
def process_available_geographies(row) -> None:
@@ -223,12 +222,12 @@ class SignalBaseResource(ModelResource):
223222
base = Field(
224223
attribute="base",
225224
column_name="base",
226-
widget=widgets.ForeignKeyWidget(Signal, field="id"),
225+
widget=ForeignKeyWidget(Signal, field="id"),
227226
)
228227
source = Field(
229228
attribute="source",
230229
column_name="Source Subdivision",
231-
widget=widgets.ForeignKeyWidget(SourceSubdivision, field="name"),
230+
widget=ForeignKeyWidget(SourceSubdivision),
232231
)
233232

234233
class Meta:
@@ -258,12 +257,12 @@ class SignalResource(ModelResource):
258257
pathogen = Field(
259258
attribute="pathogen",
260259
column_name="Pathogen/\nDisease Area",
261-
widget=widgets.ManyToManyWidget(Pathogen, field="name", separator=","),
260+
widget=ManyToManyWidget(Pathogen, field="name", separator=","),
262261
)
263262
signal_type = Field(
264263
attribute="signal_type",
265264
column_name="Indicator Type",
266-
widget=widgets.ForeignKeyWidget(SignalType, field="name"),
265+
widget=ForeignKeyWidget(SignalType),
267266
)
268267
active = Field(attribute="active", column_name="Active")
269268
description = Field(attribute="description", column_name="Description")
@@ -273,7 +272,7 @@ class SignalResource(ModelResource):
273272
format_type = Field(
274273
attribute="format_type",
275274
column_name="Format",
276-
widget=widgets.ForeignKeyWidget(FormatType, field="name"),
275+
widget=ForeignKeyWidget(FormatType),
277276
)
278277
time_type = Field(attribute="time_type", column_name="Time Type")
279278
time_label = Field(attribute="time_label", column_name="Time Label")
@@ -292,22 +291,22 @@ class SignalResource(ModelResource):
292291
severity_pyramid_rung = Field(
293292
attribute="severity_pyramid_rung",
294293
column_name="Surveillance Categories",
295-
widget=widgets.ForeignKeyWidget(SeverityPyramidRung),
294+
widget=ForeignKeyWidget(SeverityPyramidRung),
296295
)
297296
category = Field(
298297
attribute="category",
299298
column_name="Category",
300-
widget=widgets.ForeignKeyWidget(Category, "name"),
299+
widget=ForeignKeyWidget(Category),
301300
)
302301
geographic_scope = Field(
303302
attribute="geographic_scope",
304303
column_name="Geographic Coverage",
305-
widget=widgets.ForeignKeyWidget(GeographicScope),
304+
widget=ForeignKeyWidget(GeographicScope),
306305
)
307306
available_geographies = Field(
308307
attribute="available_geography",
309308
column_name="Geographic Levels",
310-
widget=widgets.ManyToManyWidget(Geography, field="name", separator=","),
309+
widget=ManyToManyWidget(Geography, field="name", separator=","),
311310
)
312311
temporal_scope_start = Field(
313312
attribute="temporal_scope_start", column_name="Temporal Scope Start"
@@ -330,7 +329,7 @@ class SignalResource(ModelResource):
330329
source = Field(
331330
attribute="source",
332331
column_name="Source Subdivision",
333-
widget=widgets.ForeignKeyWidget(SourceSubdivision, field="name"),
332+
widget=ForeignKeyWidget(SourceSubdivision),
334333
)
335334
data_censoring = Field(attribute="data_censoring", column_name="Data Censoring")
336335
missingness = Field(attribute="missingness", column_name="Missingness")
@@ -346,7 +345,7 @@ class SignalResource(ModelResource):
346345
signal_set = Field(
347346
attribute="signal_set",
348347
column_name="Indicator Set",
349-
widget=widgets.ForeignKeyWidget(SignalSet, field="name"),
348+
widget=ForeignKeyWidget(SignalSet, field="name"),
350349
)
351350

352351
class Meta:
@@ -429,7 +428,7 @@ def after_import_row(self, row, row_result, **kwargs):
429428
signal_obj.related_links.add(link)
430429
process_available_geographies(row)
431430
signal_obj.severity_pyramid_rung = SeverityPyramidRung.objects.get(id=row["Surveillance Categories"])
432-
signal_obj.format_type = row["Format"]
431+
signal_obj.format_type = FormatType.objects.get(id=row["Format"])
433432
signal_obj.save()
434433
except Signal.DoesNotExist as e:
435434
print(f"Signal.DoesNotExist: {e}")
@@ -452,12 +451,12 @@ class OtherEndpointSignalResource(ModelResource):
452451
pathogen = Field(
453452
attribute="pathogen",
454453
column_name="Pathogen/\nDisease Area",
455-
widget=widgets.ManyToManyWidget(Pathogen, field="name", separator=","),
454+
widget=ManyToManyWidget(Pathogen, field="name", separator=","),
456455
)
457456
signal_type = Field(
458457
attribute="signal_type",
459458
column_name="Indicator Type",
460-
widget=widgets.ForeignKeyWidget(SignalType, field="name"),
459+
widget=ForeignKeyWidget(SignalType),
461460
)
462461
active = Field(attribute="active", column_name="Active")
463462
description = Field(attribute="description", column_name="Description")
@@ -467,7 +466,7 @@ class OtherEndpointSignalResource(ModelResource):
467466
format_type = Field(
468467
attribute="format_type",
469468
column_name="Format",
470-
widget=widgets.ForeignKeyWidget(FormatType, field="name"),
469+
widget=ForeignKeyWidget(FormatType),
471470
)
472471
time_type = Field(attribute="time_type", column_name="Time Type")
473472
time_label = Field(attribute="time_label", column_name="Time Label")
@@ -486,22 +485,22 @@ class OtherEndpointSignalResource(ModelResource):
486485
severity_pyramid_rung = Field(
487486
attribute="severity_pyramid_rung",
488487
column_name="Surveillance Categories",
489-
widget=widgets.ForeignKeyWidget(SeverityPyramidRung),
488+
widget=ForeignKeyWidget(SeverityPyramidRung),
490489
)
491490
category = Field(
492491
attribute="category",
493492
column_name="Category",
494-
widget=widgets.ForeignKeyWidget(Category, "name"),
493+
widget=ForeignKeyWidget(Category),
495494
)
496495
geographic_scope = Field(
497496
attribute="geographic_scope",
498497
column_name="Geographic Coverage",
499-
widget=widgets.ForeignKeyWidget(GeographicScope),
498+
widget=ForeignKeyWidget(GeographicScope),
500499
)
501500
available_geographies = Field(
502501
attribute="available_geography",
503502
column_name="Geographic Levels",
504-
widget=widgets.ManyToManyWidget(Geography, field="name", separator=","),
503+
widget=ManyToManyWidget(Geography, field="name", separator=","),
505504
)
506505
temporal_scope_start = Field(
507506
attribute="temporal_scope_start", column_name="Temporal Scope Start"
@@ -524,7 +523,7 @@ class OtherEndpointSignalResource(ModelResource):
524523
source = Field(
525524
attribute="source",
526525
column_name="Source Subdivision",
527-
widget=widgets.ForeignKeyWidget(SourceSubdivision, field="name"),
526+
widget=ForeignKeyWidget(SourceSubdivision),
528527
)
529528
data_censoring = Field(attribute="data_censoring", column_name="Data Censoring")
530529
missingness = Field(attribute="missingness", column_name="Missingness")
@@ -540,7 +539,7 @@ class OtherEndpointSignalResource(ModelResource):
540539
signal_set = Field(
541540
attribute="signal_set",
542541
column_name="Indicator Set",
543-
widget=widgets.ForeignKeyWidget(SignalSet, field="name"),
542+
widget=ForeignKeyWidget(SignalSet, field="name"),
544543
)
545544

546545
class Meta:
@@ -623,7 +622,7 @@ def after_import_row(self, row, row_result, **kwargs):
623622
signal_obj.related_links.add(link)
624623
process_available_geographies(row)
625624
signal_obj.severity_pyramid_rung = SeverityPyramidRung.objects.get(id=row["Surveillance Categories"])
626-
signal_obj.format_type = row["Format"]
625+
signal_obj.format_type = FormatType.objects.get(id=row["Format"])
627626
signal_obj.save()
628627
except Signal.DoesNotExist as e:
629628
print(f"Signal.DoesNotExist: {e}")

0 commit comments

Comments
 (0)