Skip to content

Fixed data import (some singals were not imported) #143

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 17, 2025
Merged

Fixed data import (some singals were not imported) #143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/datasources/resources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import re

from import_export import resources
from import_export.fields import Field, widgets
from import_export.fields import Field
from import_export.widgets import ForeignKeyWidget

from base.models import Link
from datasources.models import DataSource, SourceSubdivision
Expand Down Expand Up @@ -65,7 +66,7 @@ class SourceSubdivisionResource(resources.ModelResource):
data_source = Field(
column_name="DB Source",
attribute="data_source",
widget=widgets.ForeignKeyWidget(DataSource, field="name"),
widget=ForeignKeyWidget(DataSource, field="name"),
)

class Meta:
Expand Down
9 changes: 5 additions & 4 deletions src/signal_sets/resources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@


from import_export import resources
from import_export.fields import Field, widgets
from import_export.fields import Field
from import_export.widgets import ForeignKeyWidget, ManyToManyWidget

from datasources.models import DataSource
from signal_sets.models import SignalSet
Expand Down Expand Up @@ -68,7 +69,7 @@ def process_datasources(row) -> None:
if row["Original Data Provider"]:
data_source = row["Original Data Provider"]
data_source_obj, _ = DataSource.objects.get_or_create(name=data_source, defaults={"display_name": data_source.capitalize()})
row["Original Data Provider"] = data_source_obj
row["Original Data Provider"] = data_source_obj.id


def fix_boolean_fields(row) -> Any:
Expand Down Expand Up @@ -101,7 +102,7 @@ class SignalSetResource(resources.ModelResource):
data_source = Field(
attribute="data_source",
column_name="Original Data Provider",
widget=widgets.ForeignKeyWidget(DataSource, field="name"),
widget=ForeignKeyWidget(DataSource),
)
endpoint = Field(attribute="endpoint", column_name="Endpoint")
language = Field(attribute="language", column_name="Language (likely English) ")
Expand All @@ -116,7 +117,7 @@ class SignalSetResource(resources.ModelResource):
geographic_scope = Field(
attribute="geographic_scope",
column_name="Geographic Coverage",
widget=widgets.ForeignKeyWidget(GeographicScope),
widget=ForeignKeyWidget(GeographicScope),
)
geographic_granularity = Field(
attribute="geographic_granularity",
Expand Down
57 changes: 28 additions & 29 deletions src/signals/resources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

from import_export import resources
from import_export.results import RowResult
from import_export.fields import Field, widgets
from import_export.fields import Field
from import_export.widgets import ForeignKeyWidget, ManyToManyWidget

from django.db.models import Max

Expand Down Expand Up @@ -63,7 +64,7 @@ def process_signal_type(row) -> None:
if row["Indicator Type"]:
signal_type = row["Indicator Type"]
signal_type_obj, _ = SignalType.objects.get_or_create(name=signal_type)
row["Indicator Type"] = signal_type_obj
row["Indicator Type"] = signal_type_obj.id


def process_format_type(row) -> None:
Expand All @@ -73,9 +74,7 @@ def process_format_type(row) -> None:
if row["Format"]:
format_type = row["Format"].strip()
format_type_obj, _ = FormatType.objects.get_or_create(name=format_type)
row["Format"] = format_type_obj
else:
row["Format"] = None
row["Format"] = format_type_obj.id


def process_severity_pyramid_rungs(row) -> None:
Expand Down Expand Up @@ -109,7 +108,7 @@ def process_category(row) -> None:
if row["Category"]:
category = row["Category"].strip()
category_obj, _ = Category.objects.get_or_create(name=category)
row["Category"] = category_obj
row["Category"] = category_obj.id


def process_geographic_scope(row) -> None:
Expand All @@ -131,7 +130,7 @@ def process_source(row) -> None:
if row["Source Subdivision"]:
source = row["Source Subdivision"]
source_obj, _ = SourceSubdivision.objects.get_or_create(name=source)
row["Source Subdivision"] = source_obj
row["Source Subdivision"] = source_obj.id


def process_available_geographies(row) -> None:
Expand Down Expand Up @@ -223,12 +222,12 @@ class SignalBaseResource(ModelResource):
base = Field(
attribute="base",
column_name="base",
widget=widgets.ForeignKeyWidget(Signal, field="id"),
widget=ForeignKeyWidget(Signal, field="id"),
)
source = Field(
attribute="source",
column_name="Source Subdivision",
widget=widgets.ForeignKeyWidget(SourceSubdivision, field="name"),
widget=ForeignKeyWidget(SourceSubdivision),
)

class Meta:
Expand Down Expand Up @@ -258,12 +257,12 @@ class SignalResource(ModelResource):
pathogen = Field(
attribute="pathogen",
column_name="Pathogen/\nDisease Area",
widget=widgets.ManyToManyWidget(Pathogen, field="name", separator=","),
widget=ManyToManyWidget(Pathogen, field="name", separator=","),
)
signal_type = Field(
attribute="signal_type",
column_name="Indicator Type",
widget=widgets.ForeignKeyWidget(SignalType, field="name"),
widget=ForeignKeyWidget(SignalType),
)
active = Field(attribute="active", column_name="Active")
description = Field(attribute="description", column_name="Description")
Expand All @@ -273,7 +272,7 @@ class SignalResource(ModelResource):
format_type = Field(
attribute="format_type",
column_name="Format",
widget=widgets.ForeignKeyWidget(FormatType, field="name"),
widget=ForeignKeyWidget(FormatType),
)
time_type = Field(attribute="time_type", column_name="Time Type")
time_label = Field(attribute="time_label", column_name="Time Label")
Expand All @@ -292,22 +291,22 @@ class SignalResource(ModelResource):
severity_pyramid_rung = Field(
attribute="severity_pyramid_rung",
column_name="Surveillance Categories",
widget=widgets.ForeignKeyWidget(SeverityPyramidRung),
widget=ForeignKeyWidget(SeverityPyramidRung),
)
category = Field(
attribute="category",
column_name="Category",
widget=widgets.ForeignKeyWidget(Category, "name"),
widget=ForeignKeyWidget(Category),
)
geographic_scope = Field(
attribute="geographic_scope",
column_name="Geographic Coverage",
widget=widgets.ForeignKeyWidget(GeographicScope),
widget=ForeignKeyWidget(GeographicScope),
)
available_geographies = Field(
attribute="available_geography",
column_name="Geographic Levels",
widget=widgets.ManyToManyWidget(Geography, field="name", separator=","),
widget=ManyToManyWidget(Geography, field="name", separator=","),
)
temporal_scope_start = Field(
attribute="temporal_scope_start", column_name="Temporal Scope Start"
Expand All @@ -330,7 +329,7 @@ class SignalResource(ModelResource):
source = Field(
attribute="source",
column_name="Source Subdivision",
widget=widgets.ForeignKeyWidget(SourceSubdivision, field="name"),
widget=ForeignKeyWidget(SourceSubdivision),
)
data_censoring = Field(attribute="data_censoring", column_name="Data Censoring")
missingness = Field(attribute="missingness", column_name="Missingness")
Expand All @@ -346,7 +345,7 @@ class SignalResource(ModelResource):
signal_set = Field(
attribute="signal_set",
column_name="Indicator Set",
widget=widgets.ForeignKeyWidget(SignalSet, field="name"),
widget=ForeignKeyWidget(SignalSet, field="name"),
)

class Meta:
Expand Down Expand Up @@ -429,7 +428,7 @@ def after_import_row(self, row, row_result, **kwargs):
signal_obj.related_links.add(link)
process_available_geographies(row)
signal_obj.severity_pyramid_rung = SeverityPyramidRung.objects.get(id=row["Surveillance Categories"])
signal_obj.format_type = row["Format"]
signal_obj.format_type = FormatType.objects.get(id=row["Format"])
signal_obj.save()
except Signal.DoesNotExist as e:
print(f"Signal.DoesNotExist: {e}")
Expand All @@ -452,12 +451,12 @@ class OtherEndpointSignalResource(ModelResource):
pathogen = Field(
attribute="pathogen",
column_name="Pathogen/\nDisease Area",
widget=widgets.ManyToManyWidget(Pathogen, field="name", separator=","),
widget=ManyToManyWidget(Pathogen, field="name", separator=","),
)
signal_type = Field(
attribute="signal_type",
column_name="Indicator Type",
widget=widgets.ForeignKeyWidget(SignalType, field="name"),
widget=ForeignKeyWidget(SignalType),
)
active = Field(attribute="active", column_name="Active")
description = Field(attribute="description", column_name="Description")
Expand All @@ -467,7 +466,7 @@ class OtherEndpointSignalResource(ModelResource):
format_type = Field(
attribute="format_type",
column_name="Format",
widget=widgets.ForeignKeyWidget(FormatType, field="name"),
widget=ForeignKeyWidget(FormatType),
)
time_type = Field(attribute="time_type", column_name="Time Type")
time_label = Field(attribute="time_label", column_name="Time Label")
Expand All @@ -486,22 +485,22 @@ class OtherEndpointSignalResource(ModelResource):
severity_pyramid_rung = Field(
attribute="severity_pyramid_rung",
column_name="Surveillance Categories",
widget=widgets.ForeignKeyWidget(SeverityPyramidRung),
widget=ForeignKeyWidget(SeverityPyramidRung),
)
category = Field(
attribute="category",
column_name="Category",
widget=widgets.ForeignKeyWidget(Category, "name"),
widget=ForeignKeyWidget(Category),
)
geographic_scope = Field(
attribute="geographic_scope",
column_name="Geographic Coverage",
widget=widgets.ForeignKeyWidget(GeographicScope),
widget=ForeignKeyWidget(GeographicScope),
)
available_geographies = Field(
attribute="available_geography",
column_name="Geographic Levels",
widget=widgets.ManyToManyWidget(Geography, field="name", separator=","),
widget=ManyToManyWidget(Geography, field="name", separator=","),
)
temporal_scope_start = Field(
attribute="temporal_scope_start", column_name="Temporal Scope Start"
Expand All @@ -524,7 +523,7 @@ class OtherEndpointSignalResource(ModelResource):
source = Field(
attribute="source",
column_name="Source Subdivision",
widget=widgets.ForeignKeyWidget(SourceSubdivision, field="name"),
widget=ForeignKeyWidget(SourceSubdivision),
)
data_censoring = Field(attribute="data_censoring", column_name="Data Censoring")
missingness = Field(attribute="missingness", column_name="Missingness")
Expand All @@ -540,7 +539,7 @@ class OtherEndpointSignalResource(ModelResource):
signal_set = Field(
attribute="signal_set",
column_name="Indicator Set",
widget=widgets.ForeignKeyWidget(SignalSet, field="name"),
widget=ForeignKeyWidget(SignalSet, field="name"),
)

class Meta:
Expand Down Expand Up @@ -623,7 +622,7 @@ def after_import_row(self, row, row_result, **kwargs):
signal_obj.related_links.add(link)
process_available_geographies(row)
signal_obj.severity_pyramid_rung = SeverityPyramidRung.objects.get(id=row["Surveillance Categories"])
signal_obj.format_type = row["Format"]
signal_obj.format_type = FormatType.objects.get(id=row["Format"])
signal_obj.save()
except Signal.DoesNotExist as e:
print(f"Signal.DoesNotExist: {e}")
Loading