diff --git a/go-static/files/local_units/Administrative Bulk Import Template - Local Units.xlsx b/go-static/files/local_units/Administrative Bulk Import Template - Local Units.xlsx new file mode 100644 index 000000000..4beadbb67 Binary files /dev/null and b/go-static/files/local_units/Administrative Bulk Import Template - Local Units.xlsx differ diff --git a/go-static/files/local_units/Health-Care-Bulk-Import-Template-Local-Units.xlsm b/go-static/files/local_units/Health-Care-Bulk-Import-Template-Local-Units.xlsm new file mode 100644 index 000000000..5b9033fd7 Binary files /dev/null and b/go-static/files/local_units/Health-Care-Bulk-Import-Template-Local-Units.xlsm differ diff --git a/go-static/files/local_units/local-unit-bulk-upload-template.csv b/go-static/files/local_units/local-unit-bulk-upload-template.csv deleted file mode 100644 index 58fe39a2f..000000000 --- a/go-static/files/local_units/local-unit-bulk-upload-template.csv +++ /dev/null @@ -1 +0,0 @@ -local_branch_name,english_branch_name,subtype,level,postcode,address_loc,address_en,city_loc,city_en,focal_person_loc,focal_person_en,phone,email,link,source_en,source_loc,date_of_data,visibility,longitude,latitude diff --git a/go-static/files/local_units/local-unit-health-bulk-upload-template.csv b/go-static/files/local_units/local-unit-health-bulk-upload-template.csv deleted file mode 100644 index da9593c64..000000000 --- a/go-static/files/local_units/local-unit-health-bulk-upload-template.csv +++ /dev/null @@ -1 +0,0 @@ -focal_point_email,focal_point_phone_number,focal_point_position,health_facility_type,other_facility_type,affiliation,functionality,primary_health_care_centre,speciality,hospital_type,is_teaching_hospital,is_in_patient_capacity,maximum_capacity,is_isolation_rooms_wards,number_of_isolation_rooms,is_warehousing,is_cold_chain,general_medical_services,specialized_medical_beyond_primary_level,other_services,blood_services,total_number_of_human_resource,general_practitioner,specialist,residents_doctor,nurse,dentist,nursing_aid,midwife,other_medical_heal,other_profiles,feedback,professional_training_facilities,ambulance_type_a,ambulance_type_b,ambulance_type_c,residential_long_term_care_facilities,primary_health_care_center,other_affiliation,local_branch_name,english_branch_name,subtype,level,postcode,address_loc,address_en,city_loc,city_en,focal_person_loc,focal_person_en,phone,email,link,source_en,source_loc,date_of_data,visibility,longitude,latitude diff --git a/local_units/bulk_upload.py b/local_units/bulk_upload.py index eb388c94d..4f93ad771 100644 --- a/local_units/bulk_upload.py +++ b/local_units/bulk_upload.py @@ -1,11 +1,14 @@ -import csv import io import logging from dataclasses import dataclass +from datetime import date, datetime from typing import Any, Dict, Generic, Literal, Optional, Type, TypeVar +import openpyxl from django.core.files.base import ContentFile from django.db import transaction +from openpyxl import Workbook +from openpyxl.styles import PatternFill from rest_framework.exceptions import ErrorDetail from rest_framework.serializers import Serializer @@ -15,7 +18,6 @@ logger = logging.getLogger(__name__) - ContextType = TypeVar("ContextType") @@ -26,78 +28,94 @@ class BulkUploadError(Exception): class ErrorWriter: - def __init__(self, fieldnames: list[str]): - self._fieldnames = ["upload_status"] + fieldnames + ERROR_ROW_FILL = PatternFill(start_color="FF0000", end_color="FF0000", fill_type="solid") + + def __init__(self, fieldnames: list[str], header_map: dict[str, str]): + """Initialize workbook and header row.""" + self._header_map = header_map or {} + self._reverse_header_map = {v: k for k, v in self._header_map.items()} + # Convert model field names to xlsx template headers + display_header = [self._reverse_header_map.get(field, field) for field in fieldnames] + + self._fieldnames = ["Upload Status"] + display_header + self._workbook = Workbook() + self._ws = self._workbook.active + + self._ws.append(self._fieldnames) + self._existing_error_columns = set() self._rows: list[dict[str, str]] = [] - self._output = io.StringIO() - self._writer = csv.DictWriter(self._output, fieldnames=self._fieldnames) - self._writer.writeheader() self._has_errors = False def _format_errors(self, errors: dict) -> dict[str, list[str]]: - """Format serializer errors into field_name and list of messages.""" - error = {} + """Recursively flatten DRF errors.""" + formatted = {} for key, value in errors.items(): if isinstance(value, dict): - for inner_key, inner_value in self._format_errors(value).items(): - error[inner_key] = inner_value + formatted.update(self._format_errors(value)) elif isinstance(value, list): - error[key] = [self._clean_message(v) for v in value] + header = self._reverse_header_map.get(key, key) + formatted[header] = [self._clean_message(v) for v in value] else: - error[key] = [self._clean_message(value)] - return error + header = self._reverse_header_map.get(key, key) + formatted[header] = [self._clean_message(value)] + return formatted - def _clean_message(self, msg: Any) -> str: - """Convert ErrorDetail or other objects into normal text.""" + def _clean_message(self, msg: any) -> str: if isinstance(msg, ErrorDetail): return str(msg) return str(msg) - def _update_csv_header_with_errors(self): - """Update the CSV with updated headers when new error columns are introduced.""" - self._output.seek(0) - self._output.truncate() - self._writer = csv.DictWriter(self._output, fieldnames=self._fieldnames) - self._writer.writeheader() - for row in self._rows: - self._writer.writerow(row) + def _add_error_columns(self, fields: list[str]): + """Ensure field has a matching _error column.""" + for field in fields: + col_name = f"{field}_error" + if col_name in self._existing_error_columns: + continue + self._existing_error_columns.add(col_name) + + if field in self._fieldnames: + idx = self._fieldnames.index(field) + 1 + self._fieldnames.insert(idx, col_name) + self._ws.insert_cols(idx + 1) + else: + self._fieldnames.append(col_name) + for i, col_name in enumerate(self._fieldnames, start=1): + self._ws.cell(row=1, column=i, value=col_name) def write( self, - row: dict[str, str], + row: dict[str, any], status: Literal[LocalUnitBulkUpload.Status.SUCCESS, LocalUnitBulkUpload.Status.FAILED], error_detail: dict | None = None, - ) -> None: - row_copy = {col: row.get(col, "") for col in self._fieldnames} - row_copy["upload_status"] = status.name - added_error_column = False + ): + row_display = {self._reverse_header_map.get(k, k): v for k, v in row.items()} + row_out = {col: row_display.get(col, "") for col in self._fieldnames} + row_out["Upload Status"] = status.name if status == LocalUnitBulkUpload.Status.FAILED and error_detail: - formatted_errors = self._format_errors(error_detail) - for field, messages in formatted_errors.items(): - error_col = f"{field}_error" - - if error_col not in self._fieldnames: - if field in self._fieldnames: - col_idx = self._fieldnames.index(field) - self._fieldnames.insert(col_idx + 1, error_col) - else: - self._fieldnames.append(error_col) - - added_error_column = True - row_copy[error_col] = "; ".join(messages) - self._rows.append(row_copy) - if added_error_column: - self._update_csv_header_with_errors() - else: - self._writer.writerow(row_copy) + formatted = self._format_errors(error_detail) + self._add_error_columns(list(formatted.keys())) + for field, msgs in formatted.items(): + row_out[f"{field}_error"] = "; ".join(msgs) + self._has_errors = True + + self._ws.append([row_out.get(col, "") for col in self._fieldnames]) + + if status == LocalUnitBulkUpload.Status.FAILED: + for cell in self._ws[self._ws.max_row]: + cell.fill = self.ERROR_ROW_FILL def to_content_file(self) -> ContentFile: - return ContentFile(self._output.getvalue().encode("utf-8")) + """Export workbook as Content File for Django.""" + buffer = io.BytesIO() + self._workbook.save(buffer) + buffer.seek(0) + return ContentFile(buffer.getvalue()) class BaseBulkUpload(Generic[ContextType]): serializer_class: Type[Serializer] + HEADER_MAP: dict[str, str] def __init__(self, bulk_upload: LocalUnitBulkUpload): if self.serializer_class is None: @@ -116,12 +134,15 @@ def delete_existing_local_unit(self): """Delete existing local units based on the context.""" pass - def _validate_csv(self, fieldnames) -> None: + def _validate_type(self, fieldnames) -> None: pass - def _is_csv_empty(self, csv_reader: csv.DictReader) -> tuple[bool, list[dict]]: - rows = list(csv_reader) - return len(rows) == 0, rows + def is_excel_data_empty(self, sheet, data_start_row=4): + """Check if file is empty or not""" + for row in sheet.iter_rows(values_only=True, min_row=data_start_row): + if any(cell is not None for cell in row): + return False + return True def process_row(self, data: Dict[str, Any]) -> bool: serializer = self.serializer_class(data=data) @@ -132,49 +153,72 @@ def process_row(self, data: Dict[str, Any]) -> bool: return False def run(self) -> None: - with self.bulk_upload.file.open("rb") as csv_file: - file = io.TextIOWrapper(csv_file, encoding="utf-8") - csv_reader = csv.DictReader(file) - fieldnames = csv_reader.fieldnames or [] - try: - is_empty, rows = self._is_csv_empty(csv_reader) - if is_empty: - raise BulkUploadError("The uploaded CSV file is empty or contains only blank rows.") - - csv_reader = iter(rows) - - self._validate_csv(fieldnames) - except BulkUploadError as e: - self.bulk_upload.status = LocalUnitBulkUpload.Status.FAILED - self.bulk_upload.error_message = str(e) - self.bulk_upload.save(update_fields=["status", "error_message"]) - logger.warning(f"[BulkUpload:{self.bulk_upload.pk}] Validation error: {str(e)}") - return - - context = self.get_context().__dict__ - self.error_writer = ErrorWriter(fieldnames) - try: + header_row_index = 2 + data_row_index = header_row_index + 2 + + try: + with self.bulk_upload.file.open("rb") as f: + workbook = openpyxl.load_workbook(f, data_only=True, read_only=True) + sheet = workbook.active + + headers = next(sheet.iter_rows(values_only=True, min_row=header_row_index, max_row=header_row_index)) + raw_fieldnames = [str(h).strip() for h in headers if h and str(h).strip()] + header_map = self.HEADER_MAP or {} + mapped_fieldnames = [header_map.get(h, h) for h in raw_fieldnames] + fieldnames = mapped_fieldnames + + if self.is_excel_data_empty(sheet, data_start_row=data_row_index): + raise BulkUploadError("The uploaded file is empty. Please provide at least one data row.") + + self._validate_type(fieldnames) + self.error_writer = ErrorWriter(fieldnames=raw_fieldnames, header_map=header_map) + context = self.get_context().__dict__ + with transaction.atomic(): self.delete_existing_local_unit() - for row_index, row_data in enumerate(csv_reader, start=2): - data = {**row_data, **context} - if self.process_row(data): + + for row_index, row_values in enumerate( + sheet.iter_rows(values_only=True, min_row=data_row_index), + start=data_row_index, + ): + if not any(cell is not None for cell in row_values): + continue # skip empty rows + + row_dict = dict(zip(fieldnames, row_values)) + row_dict = {**row_dict, **context} + + # Convert date/datetime to str + for key, value in row_dict.items(): + if isinstance(value, (datetime, date)): + row_dict[key] = value.strftime("%Y-%m-%d") + + if self.process_row(row_dict): self.success_count += 1 - self.error_writer.write(row_data, status=LocalUnitBulkUpload.Status.SUCCESS) + self.error_writer.write(row_dict, status=LocalUnitBulkUpload.Status.SUCCESS) else: self.failed_count += 1 self.error_writer.write( - row_data, status=LocalUnitBulkUpload.Status.FAILED, error_detail=self.error_detail + row_dict, + status=LocalUnitBulkUpload.Status.FAILED, + error_detail=self.error_detail, ) - logger.warning(f"[BulkUpload:{self.bulk_upload.pk}] Row '{row_index}' failed") + logger.warning(f"[BulkUpload:{self.bulk_upload.pk}] Row {row_index} failed") if self.failed_count > 0: - raise BulkUploadError("Bulk upload failed with some errors.") + raise BulkUploadError() self.bulk_manager.done() self._finalize_success() - except BulkUploadError: - self._finalize_failure() + + workbook.close() + + except Exception as e: + self.bulk_upload.status = LocalUnitBulkUpload.Status.FAILED + self.bulk_upload.error_message = str(e) + self.bulk_upload.save(update_fields=["status", "error_message"]) + if isinstance(e, BulkUploadError): + logger.warning(f"[BulkUpload:{self.bulk_upload.pk}] error: {e}") + self._finalize_failure() def _finalize_success(self) -> None: self.bulk_upload.success_count = self.success_count @@ -187,7 +231,7 @@ def _finalize_success(self) -> None: def _finalize_failure(self) -> None: if self.error_writer: error_file = self.error_writer.to_content_file() - self.bulk_upload.error_file.save("error_file.csv", error_file, save=True) + self.bulk_upload.error_file.save("error_file.xlsx", error_file, save=True) self.bulk_upload.success_count = self.success_count self.bulk_upload.failed_count = self.failed_count @@ -206,6 +250,28 @@ class LocalUnitUploadContext: class BaseBulkUploadLocalUnit(BaseBulkUpload[LocalUnitUploadContext]): + HEADER_MAP = { + "Date of Update": "date_of_data", + "Local Unit Name (En)": "english_branch_name", + "Local Unit Name (Local)": "local_branch_name", + "Visibility": "visibility", + "Coverage": "level", + "Sub-type": "subtype", + "Focal Person (En)": "focal_person_en", + "Source (En)": "source_en", + "Source (Local)": "source_loc", + "Focal Person (Local)": "focal_person_loc", + "Address (Local)": "address_loc", + "Address (En)": "address_en", + "Locality (Local)": "city_loc", + "Locality (En)": "city_en", + "Local Unit Post Code": "postcode", + "Local Unit Email": "email", + "Local Unit Phone Number": "phone", + "Local Unit Website": "link", + "Latitude": "latitude", + "Longitude": "longitude", + } def __init__(self, bulk_upload: LocalUnitBulkUpload): from local_units.serializers import LocalUnitBulkUploadDetailSerializer @@ -232,19 +298,66 @@ def delete_existing_local_unit(self): else: logger.info("No existing local units found for deletion.") - def _validate_csv(self, fieldnames) -> None: + def _validate_type(self, fieldnames: list[str]) -> None: + health_field_names = set(get_model_field_names(HealthData)) - present_health_fields = health_field_names & set(fieldnames) + present_health_fields = {h for h in health_field_names if h.lower() in [f.lower() for f in fieldnames]} local_unit_type = LocalUnitType.objects.filter(id=self.bulk_upload.local_unit_type_id).first() if not local_unit_type: raise ValueError("Invalid local unit type") if present_health_fields and local_unit_type.name.strip().lower() != "health care": - raise BulkUploadError(f"You cannot upload Healthcare data when the Local Unit type is set to {local_unit_type.name}.") + raise BulkUploadError( + f"You cannot upload Healthcare data when the Local Unit type is set to '{local_unit_type.name}'." + ) class BulkUploadHealthData(BaseBulkUpload[LocalUnitUploadContext]): + # Local Unit headers + Health Data headers + HEADER_MAP = { + **BaseBulkUploadLocalUnit.HEADER_MAP, + **{ + "Focal Person Email": "focal_point_email", + "Focal Person Phone Number": "focal_point_phone_number", + "Focal Person Position": "focal_point_position", + "Health Facility Type": "health_facility_type", + "Other Facility Type": "other_facility_type", + "Affiliation": "affiliation", + "Other Affiliation": "other_affiliation", + "Functionality": "functionality", + "Primary Health Care Center": "primary_health_care_center", + "Specialities": "speciality", + "Hospital Type": "hospital_type", + "Teaching Hospital": "is_teaching_hospital", + "In-patient Capacity": "is_in_patient_capacity", + "Isolation Rooms": "is_isolation_rooms_wards", + "Number of Isolation Beds": "number_of_isolation_rooms", + "Warehousing": "is_warehousing", + "Cold Chain": "is_cold_chain", + "Maximum Bed Capacity": "maximum_capacity", + "General Medical Services": "general_medical_services", + "Specialized Medical Services (beyond primary level)": "specialized_medical_beyond_primary_level", + "Blood Services": "blood_services", + "Other Services": "other_services", + "Total Number of Human Resources": "total_number_of_human_resource", + "General Practitioners": "general_practitioner", + "Resident Doctors": "residents_doctor", + "Specialists": "specialist", + "Nurses": "nurse", + "Nursing Aids": "nursing_aid", + "Dentists": "dentist", + "Midwife": "midwife", + "Pharmacists": "pharmacists", + "Other Profiles": "other_profiles", + "Other Training Facility": "other_training_facilities", + "Professional Training Facilities": "professional_training_facilities", + "Ambulance Type A": "ambulance_type_a", + "Ambulance Type B": "ambulance_type_b", + "Ambulance Type C": "ambulance_type_c", + }, + } + def __init__(self, bulk_upload: LocalUnitBulkUpload): from local_units.serializers import LocalUnitBulkUploadDetailSerializer @@ -277,7 +390,17 @@ def delete_existing_local_unit(self): logger.info("No existing local units found for deletion.") def process_row(self, data: dict[str, any]) -> bool: - health_data = {k: data.get(k) for k in list(data.keys()) if k in self.health_field_names} + from local_units.serializers import HealthDataBulkUploadSerializer + + health_data = {k: data.get(k) for k in data.keys() if k in self.health_field_names} + if health_data: - data["health"] = health_data - return super().process_row(data) + health_serializer = HealthDataBulkUploadSerializer(data=health_data) + if not health_serializer.is_valid(): + self.error_detail = health_serializer.errors + return False + health_instance = health_serializer.save() + for k in health_data.keys(): + data.pop(k, None) + data["health"] = health_instance.pk + return super().process_row(data) diff --git a/local_units/models.py b/local_units/models.py index 158c52beb..bd987df74 100644 --- a/local_units/models.py +++ b/local_units/models.py @@ -203,6 +203,7 @@ class HealthData(models.Model): null=True, blank=True, ) + ambulance_type_a = models.IntegerField(verbose_name=_("Ambulance Type A"), blank=True, null=True) ambulance_type_b = models.IntegerField(verbose_name=_("Ambulance Type B"), blank=True, null=True) ambulance_type_c = models.IntegerField(verbose_name=_("Ambulance Type C"), blank=True, null=True) diff --git a/local_units/permissions.py b/local_units/permissions.py index 1fb27e4d3..55381a9db 100644 --- a/local_units/permissions.py +++ b/local_units/permissions.py @@ -25,20 +25,20 @@ def has_object_permission(self, request, view, obj): class IsAuthenticatedForLocalUnit(permissions.BasePermission): - message = "Only Country Admins, Local Unit Validators, Region Admins, or Superusers are allowed to update Local Units." + message = ( + "Only Country Admins, Local Unit Validators, Region Admins, IFRC Admins or Superusers are allowed to update Local Units." + ) def has_object_permission(self, request, view, obj): if request.method not in ["PUT", "PATCH"]: return True # Only restrict update operations user = request.user - - if user.is_superuser: + if user.has_perm("api.ifrc_admin") or user.is_superuser: return True country_id = obj.country_id - country = Country.objects.get(id=country_id) - region_id = country.region_id + region_id = obj.country.region_id # Country admin specific permissions country_admin_ids = [ int(codename.replace("country_admin_", "")) diff --git a/local_units/serializers.py b/local_units/serializers.py index 4ab84d46a..07f039a2d 100644 --- a/local_units/serializers.py +++ b/local_units/serializers.py @@ -673,6 +673,7 @@ def update(self, instance, validated_data): class LocalUnitBulkUploadSerializer(serializers.ModelSerializer): + VALID_FILE_EXTENSIONS = (".xlsx", ".xlsm") country = serializers.PrimaryKeyRelatedField( queryset=Country.objects.filter( is_deprecated=False, independent=True, iso3__isnull=False, record_type=CountryType.COUNTRY @@ -703,8 +704,8 @@ class Meta: ) def validate_file(self, file): - if not file.name.endswith(".csv"): - raise serializers.ValidationError(gettext("File must be a CSV file.")) + if not file.name.lower().endswith(self.VALID_FILE_EXTENSIONS): + raise serializers.ValidationError(gettext("The uploaded file must be an Excel document (.xlsx or .xlsm).")) if file.size > 10 * 1024 * 1024: raise serializers.ValidationError(gettext("File must be less than 10 MB.")) return file @@ -871,7 +872,7 @@ class LocalUnitBulkUploadDetailSerializer(serializers.ModelSerializer): visibility = serializers.CharField(required=True, allow_blank=True) date_of_data = serializers.CharField(required=False, allow_null=True) level = serializers.CharField(required=False, allow_null=True) - health = HealthDataBulkUploadSerializer(required=False) + health = serializers.PrimaryKeyRelatedField(queryset=HealthData.objects.all(), required=False, allow_null=True) class Meta: model = LocalUnit @@ -952,10 +953,6 @@ def validate(self, validated_data): validated_data["status"] = LocalUnit.Status.EXTERNALLY_MANAGED # NOTE: Bulk upload doesn't call create() method - health_data = validated_data.pop("health", None) - if health_data: - health_instance = HealthData.objects.create(**health_data) - validated_data["health"] = health_instance return validated_data diff --git a/local_units/test_views.py b/local_units/test_views.py index f2d0f5b09..38e3c2ae3 100644 --- a/local_units/test_views.py +++ b/local_units/test_views.py @@ -120,7 +120,6 @@ def setUp(self): country_codename = f"local_unit_country_validator_{self.type_3.id}_{self.country2.id}" region_codename = f"local_unit_region_validator_{self.type_3.id}_{region.id}" global_codename = f"local_unit_global_validator_{self.type_3.id}" - country_permission = Permission.objects.get(codename=country_codename) region_permission = Permission.objects.get(codename=region_codename) global_permission = Permission.objects.get(codename=global_codename) @@ -882,7 +881,7 @@ def test_create_local_unit_with_externally_managed_country_and_type(self): self.assertEqual(response.status_code, 400) self.assertEqual(LocalUnitChangeRequest.objects.count(), 0) - def test_country_region_admin_permission_for_local_unit_update(self): + def test_local_unit_update(self): region1 = RegionFactory.create(name=2, label="Asia Pacific") region2 = RegionFactory.create(name=0, label="Africa") @@ -898,6 +897,7 @@ def test_country_region_admin_permission_for_local_unit_update(self): self.asia_admin = UserFactory.create(email="asia@admin.com") self.africa_admin = UserFactory.create(email="africa@admin.com") self.india_admin = UserFactory.create(email="india@admin.com") + self.ifrc_admin = UserFactory.create(email="ifrc@admin.test") # India admin setup management.call_command("make_permissions") country_admin_codename = f"country_admin_{country.id}" @@ -922,6 +922,14 @@ def test_country_region_admin_permission_for_local_unit_update(self): africa_admin_group.permissions.add(africa_admin_permission) self.africa_admin.groups.add(africa_admin_group) + # Ifrc admin + ifrc_admin_codename = "ifrc_admin" + ifrc_admin_permission = Permission.objects.get(codename=ifrc_admin_codename) + ifrc_admin_group_name = "IFRC Admins" + ifrc_admin_group = Group.objects.get(name=ifrc_admin_group_name) + ifrc_admin_group.permissions.add(ifrc_admin_permission) + self.ifrc_admin.groups.add(ifrc_admin_group) + local_unit = LocalUnitFactory.create( country=country, type=self.local_unit_type, @@ -936,6 +944,13 @@ def test_country_region_admin_permission_for_local_unit_update(self): status=LocalUnit.Status.VALIDATED, date_of_data="2023-08-08", ) + local_unit3 = LocalUnitFactory.create( + country=country, + type=self.local_unit_type, + draft=False, + status=LocalUnit.Status.VALIDATED, + date_of_data="2023-08-08", + ) url = f"/api/v2/local-units/{local_unit.id}/" data = { "local_branch_name": "Updated local branch name", @@ -968,6 +983,12 @@ def test_country_region_admin_permission_for_local_unit_update(self): response = self.client.patch(url, data=data, format="json") self.assert_200(response) self.assertEqual(response.data["local_branch_name"], "Updated local branch name") + # Test update as ifrc admin + url = f"/api/v2/local-units/{local_unit3.id}/" + self.authenticate(self.ifrc_admin) + response = self.client.patch(url, data=data, format="json") + self.assert_200(response) + self.assertEqual(response.data["local_branch_name"], "Updated local branch name") class TestExternallyManagedLocalUnit(APITestCase): @@ -1177,15 +1198,15 @@ def setUp(self): global_group.permissions.add(global_permission) self.global_validator_user.groups.add(global_group) - file_path = os.path.join(settings.TEST_DIR, "local_unit/test.csv") + file_path = os.path.join(settings.TEST_DIR, "local_unit/test-admin.xlsx") with open(file_path, "rb") as f: self._file_content = f.read() - def create_upload_file(self, filename="test.csv"): + def create_upload_file(self, filename="test-admin.xlsx"): """ Always return a new file instance to prevent stream exhaustion. """ - return SimpleUploadedFile(filename, self._file_content, content_type="text/csv") + return SimpleUploadedFile(filename, self._file_content, content_type="text/xlsx") @mock.patch("local_units.tasks.process_bulk_upload_local_unit.delay") def test_bulk_upload_local_unit(self, mock_delay): @@ -1330,16 +1351,16 @@ def setUpTestData(cls): cls.local_unit_type = LocalUnitType.objects.create(code=1, name="Administrative") cls.local_unit_type2 = LocalUnitType.objects.create(code=2, name="Health Care") cls.level = LocalUnitLevel.objects.create(level=0, name="National") - file_path = os.path.join(settings.TEST_DIR, "local_unit/test.csv") + file_path = os.path.join(settings.TEST_DIR, "local_unit/test-admin.xlsx") with open(file_path, "rb") as f: cls._file_content = f.read() - def create_upload_file(cls, filename="test.csv"): - return SimpleUploadedFile(filename, cls._file_content, content_type="text/csv") + def create_upload_file(cls, filename="test-admin.xlsx"): + return SimpleUploadedFile(filename, cls._file_content, content_type="text/xlsx") def test_bulk_upload_with_incorrect_country(cls): """ - Test bulk upload fails when the country does not match CSV data. + Test bulk upload fails when the country does not match xlsx data. """ cls.bulk_upload = LocalUnitBulkUploadFactory.create( country=cls.country1, @@ -1362,13 +1383,13 @@ def test_bulk_upload_with_incorrect_country(cls): def test_bulk_upload_with_valid_country(cls): """ - Test bulk upload succeeds when the country matches CSV data + Test bulk upload succeeds when the country matches xlsx data """ cls.bulk_upload = LocalUnitBulkUploadFactory.create( country=cls.country2, # Brazil local_unit_type=cls.local_unit_type, triggered_by=cls.user, - file=cls.create_upload_file(), # CSV with Brazil rows + file=cls.create_upload_file(), # xlsx with Brazil rows status=LocalUnitBulkUpload.Status.PENDING, ) runner = BaseBulkUploadLocalUnit(cls.bulk_upload) @@ -1381,7 +1402,7 @@ def test_bulk_upload_with_valid_country(cls): def test_bulk_upload_fails_and_delete(cls): """ - Test bulk upload fails and delete when CSV has incorrect data. + Test bulk upload fails and delete when xlsx has incorrect data. """ LocalUnitFactory.create_batch( 5, @@ -1410,7 +1431,7 @@ def test_bulk_upload_fails_and_delete(cls): def test_bulk_upload_deletes_old_and_creates_new_local_units(cls): """ - Test bulk upload with correct CSV data. + Test bulk upload with correct data. """ old_local_unit = LocalUnitFactory.create( country=cls.country2, @@ -1442,10 +1463,14 @@ def test_empty_administrative_file(cls): Test bulk upload file is empty """ - file_path = os.path.join(settings.STATICFILES_DIRS[0], "files", "local_units", "local-unit-bulk-upload-template.csv") + file_path = os.path.join( + settings.STATICFILES_DIRS[0], "files", "local_units", "Administrative Bulk Import Template - Local Units.xlsx" + ) with open(file_path, "rb") as f: file_content = f.read() - empty_file = SimpleUploadedFile(name="local-unit-bulk-upload-template.csv", content=file_content, content_type="text/csv") + empty_file = SimpleUploadedFile( + name="Administrative Bulk Import Template - Local Units.xlsx", content=file_content, content_type="text/xlsx" + ) LocalUnitFactory.create_batch( 5, country=cls.country2, @@ -1531,16 +1556,16 @@ def setUpTestData(cls): cls.professional_training_facilities = ProfessionalTrainingFacility.objects.create(code=1, name="Nurses") cls.general_medical_services = GeneralMedicalService.objects.create(code=1, name="Minor Trauma") - file_path = os.path.join(settings.TEST_DIR, "local_unit/test-health.csv") + file_path = os.path.join(settings.TEST_DIR, "local_unit/test-health.xlsm") with open(file_path, "rb") as f: cls._file_content = f.read() - def create_upload_file(cls, filename="test-health.csv"): - return SimpleUploadedFile(filename, cls._file_content, content_type="text/csv") + def create_upload_file(cls, filename="test-health.xlsm"): + return SimpleUploadedFile(filename, cls._file_content, content_type="text/xlsm") def test_bulk_upload_health_with_incorrect_country(cls): """ - Should fail when CSV rows are not equal to bulk upload country. + Should fail when rows are not equal to bulk upload country. """ cls.bulk_upload = LocalUnitBulkUploadFactory.create( country=cls.country1, @@ -1561,7 +1586,7 @@ def test_bulk_upload_health_with_incorrect_country(cls): def test_bulk_upload_health_fails_and_does_not_delete(cls): """ - Should fail and keep existing LocalUnits & HealthData when CSV invalid. + Should fail and keep existing LocalUnits & HealthData when file invalid. """ health_data = HealthDataFactory.create_batch( 5, @@ -1640,12 +1665,12 @@ def test_empty_health_template_file(cls): """ file_path = os.path.join( - settings.STATICFILES_DIRS[0], "files", "local_units", "local-unit-health-bulk-upload-template.csv" + settings.STATICFILES_DIRS[0], "files", "local_units", "Health-Care-Bulk-Import-Template-Local-Units.xlsm" ) with open(file_path, "rb") as f: file_content = f.read() empty_file = SimpleUploadedFile( - name="local-unit-health-bulk-upload-template.csv", content=file_content, content_type="text/csv" + name="Health-Care-Bulk-Import-Template-Local-Units.xlsm", content=file_content, content_type="text/xlsm" ) health_data = HealthDataFactory.create_batch( 5, diff --git a/local_units/utils.py b/local_units/utils.py index 2f2fabe1a..852d94c07 100644 --- a/local_units/utils.py +++ b/local_units/utils.py @@ -72,16 +72,13 @@ def get_model_field_names( def normalize_bool(value): - if isinstance(value, bool): - return value if not value: return False val = str(value).strip().lower() - if val in ("true", "1", "yes", "y"): + if val in ("yes"): return True - if val in ("false", "0", "no", "n"): + if val in ("no"): return False - return False def wash(string): @@ -91,4 +88,4 @@ def wash(string): def numerize(value): - return value if value.isdigit() else 0 + return value if value else 0 diff --git a/local_units/views.py b/local_units/views.py index 5e88c550c..772a3b1ae 100644 --- a/local_units/views.py +++ b/local_units/views.py @@ -476,8 +476,10 @@ class LocalUnitBulkUploadViewSet( def get_bulk_upload_template(self, request): template_type = request.query_params.get("bulk_upload_template", "local_unit") if template_type == "health_care": - file_url = request.build_absolute_uri(static("files/local_units/local-unit-health-bulk-upload-template.csv")) + file_url = request.build_absolute_uri(static("files/local_units/Health-Care-Bulk-Import-Template-Local-Units.xlsm")) else: - file_url = request.build_absolute_uri(static("files/local_units/local-unit-bulk-upload-template.csv")) + file_url = request.build_absolute_uri( + static("files/local_units/Administrative Bulk Import Template - Local Units.xlsx") + ) template = {"template_url": file_url} return response.Response(LocalUnitTemplateFilesSerializer(template).data) diff --git a/main/test_files/local_unit/test-admin.xlsx b/main/test_files/local_unit/test-admin.xlsx new file mode 100644 index 000000000..bfb2b4904 Binary files /dev/null and b/main/test_files/local_unit/test-admin.xlsx differ diff --git a/main/test_files/local_unit/test-health.csv b/main/test_files/local_unit/test-health.csv deleted file mode 100644 index 199297414..000000000 --- a/main/test_files/local_unit/test-health.csv +++ /dev/null @@ -1,4 +0,0 @@ -focal_point_email,focal_point_phone_number,focal_point_position,health_facility_type,other_facility_type,affiliation,functionality,primary_health_care_centre,speciality,hospital_type,is_teaching_hospital,is_in_patient_capacity,maximum_capacity,is_isolation_rooms_wards,number_of_isolation_rooms,is_warehousing,is_cold_chain,general_medical_services,specialized_medical_beyond_primary_level,other_services,blood_services,total_number_of_human_resource,general_practitioner,specialist,residents_doctor,nurse,dentist,nursing_aid,midwife,other_medical_heal,feedback,professional_training_facilities,ambulance_type_a,ambulance_type_b,ambulance_type_c,residential_long_term_care_facilities,primary_health_care_center,other_affiliation,local_branch_name,english_branch_name,subtype,level,postcode,address_loc,address_en,city_loc,city_en,focal_person_loc,focal_person_en,phone,email,link,source_en,source_loc,date_of_data,visibility,longitude,latitude -jele@redcross1.org.sz,26876088546,Programmes Manager,Ambulance Station,tet,Public Government Facility,Fully Functional,Medical Practices,"Initiate TB treatment, Cervical Cancer Screening and testing and diagnostic and treatment for people living with HIV and follow up care through the ART programme which the government supports very well",Mental Health Hospital,yes,No,2,yes,2,Yes,Yes,Minor Trauma,Anaesthesiology,test,Blood Collection,32,0,0,0,3,0,0,9,Yes,first question of initial question did not provide for the option to write the name of the NS. It is written LRC yet it should allow Baphalali Eswatini Red Cross Society (BERCS) to be inscribed in the box.,Nurses,2,1,1,1,Dental Practices,test,Cruz Vermelha - Órgão Central HQ - Brasília,,Office,National,70300-910,"Setor Comercial Sul (SCS), quadra 6, bloco A, nº 157, salas 502 e 503 - Edifício Bandeirantes, Asa Sul, Brasília-DF ",,Brasília - DF,,Lourenço Braga,,55 (61) 99909-0761,secretario.geral@cvb.org.br,http://www.cruzvermelha.org.br/,Brazilian Red Cross,Brazilian Red Cross Local,2024-02-07,Public,-47.88972222,-15.79638889 -jele@redcross2.org.sz,26876088546,Programmes Manager,Ambulance Station,tet,Public Government Facility,Fully Functional,"Medical Practices,Dental Practices","Initiate TB treatment, Cervical Cancer Screening and testing and diagnostic and treatment for people living with HIV and follow up care through the ART programme which the government supports very well",Mental Health Hospital,No ,No,2,yes,2,Yes,Yes,Minor Trauma,Anaesthesiology,test,Blood Collection,32,0,0,0,3,0,0,9,Yes,first question of initial question did not provide for the option to write the name of the NS. It is written LRC yet it should allow Baphalali Eswatini Red Cross Society (BERCS) to be inscribed in the box.,Nurses,1,2,3,1,Dental Practices,test,Filial Alagoas,,Office,National,57035-530,"Av. Com. Gustavo de Paiva, 2889 - Mangabeiras, Maceió - AL",,Alagoas - AL,,Agarina Mendonça,,55 (82) 3325-2430,diretoria@cvbal.org.br,https://www.cruzvermelha.org.br/pb/filiais/alagoas/,Brazilian Red Cross,Brazilian Red Cross Local,2024-02-07,Public,-35.71611111,-9.64777778 -jele@redcross.org.sz,26876088546,Programmes Manager,Ambulance Station,tet,Public Government Facility,Fully Functional,Medical Practices,"Initiate TB treatment, Cervical Cancer Screening and testing and diagnostic and treatment for people living with HIV and follow up care through the ART programme which the government supports very well",Mental Health Hospital,No ,No,2,no,2,Yes,Yes,Minor Trauma,Anaesthesiology,test,Blood Collection,32,0,0,0,3,0,0,9,Yes,first question of initial question did not provide for the option to write the name of the NS. It is written LRC yet it should allow Baphalali Eswatini Red Cross Society (BERCS) to be inscribed in the box.,Nurses,1,2,2,1,Dental Practices,test,Cruz Vermelha - Órgão Central HQ,,Office,National,20230-130,"Praça Cruz Vermelha N° 10-12, Centro, Rio de Janeiro - RJ",,Rio de Janeiro - RJ,,Thiago Quintaneiro,,55 (21) 2507-3577 / 2507-3392,secretario.cooperacao@cvb.org.br,http://www.cruzvermelha.org.br/,Brazilian Red Cross,Brazilian Red Cross Local,2024-02-07,Public,-43.1875,-22.91111111 diff --git a/main/test_files/local_unit/test-health.xlsm b/main/test_files/local_unit/test-health.xlsm new file mode 100644 index 000000000..9950bda6e Binary files /dev/null and b/main/test_files/local_unit/test-health.xlsm differ diff --git a/main/test_files/local_unit/test.csv b/main/test_files/local_unit/test.csv deleted file mode 100644 index 4860fc255..000000000 --- a/main/test_files/local_unit/test.csv +++ /dev/null @@ -1,4 +0,0 @@ -local_branch_name,english_branch_name,subtype,level,postcode,address_loc,address_en,city_loc,city_en,focal_person_loc,focal_person_en,phone,email,link,source_en,source_loc,date_of_data,visibility,longitude,latitude -Cruz Vermelha - Órgão Central HQ,,Office,National,20230-130,"Praça Cruz Vermelha N° 10-12, Centro, Rio de Janeiro - RJ",,Rio de Janeiro - RJ,,Thiago Quintaneiro,,55 (21) 2507-3577 / 2507-3392,secretario.cooperacao@cvb.org.br,http://www.cruzvermelha.org.br/,Brazilian Red Cross,Brazilian Red Cross Local,2024-02-07,Public,-43.1875,-22.91111111 -Cruz Vermelha - Órgão Central HQ - Brasília,,Office,National,70300-910,"Setor Comercial Sul (SCS), quadra 6, bloco A, nº 157, salas 502 e 503 - Edifício Bandeirantes, Asa Sul, Brasília-DF ",,Brasília - DF,,Lourenço Braga,,55 (61) 99909-0761,secretario.geral@cvb.org.br,http://www.cruzvermelha.org.br/,Brazilian Red Cross,Brazilian Red Cross Local,2024-02-07,Public,-47.88972222,-15.79638889 -Filial Alagoas,,Office,National,57035-530,"Av. Com. Gustavo de Paiva, 2889 - Mangabeiras, Maceió - AL",,Alagoas - AL,,Agarina Mendonça,,55 (82) 3325-2430,diretoria@cvbal.org.br,https://www.cruzvermelha.org.br/pb/filiais/alagoas/,Brazilian Red Cross,Brazilian Red Cross Local,2024-02-07,Public,-35.71611111,-9.64777778