diff --git a/go-static/files/local_units/Administrative Bulk Import Template - Local Units.xlsx b/go-static/files/local_units/Administrative Bulk Import Template - Local Units.xlsx new file mode 100644 index 000000000..4beadbb67 Binary files /dev/null and b/go-static/files/local_units/Administrative Bulk Import Template - Local Units.xlsx differ diff --git a/go-static/files/local_units/Health Care Bulk Import Template - Local Units.xlsx b/go-static/files/local_units/Health Care Bulk Import Template - Local Units.xlsx new file mode 100644 index 000000000..0ca799bd9 Binary files /dev/null and b/go-static/files/local_units/Health Care Bulk Import Template - Local Units.xlsx differ diff --git a/go-static/files/local_units/local-unit-bulk-upload-template.csv b/go-static/files/local_units/local-unit-bulk-upload-template.csv deleted file mode 100644 index 58fe39a2f..000000000 --- a/go-static/files/local_units/local-unit-bulk-upload-template.csv +++ /dev/null @@ -1 +0,0 @@ -local_branch_name,english_branch_name,subtype,level,postcode,address_loc,address_en,city_loc,city_en,focal_person_loc,focal_person_en,phone,email,link,source_en,source_loc,date_of_data,visibility,longitude,latitude diff --git a/go-static/files/local_units/local-unit-health-bulk-upload-template.csv b/go-static/files/local_units/local-unit-health-bulk-upload-template.csv deleted file mode 100644 index da9593c64..000000000 --- a/go-static/files/local_units/local-unit-health-bulk-upload-template.csv +++ /dev/null @@ -1 +0,0 @@ -focal_point_email,focal_point_phone_number,focal_point_position,health_facility_type,other_facility_type,affiliation,functionality,primary_health_care_centre,speciality,hospital_type,is_teaching_hospital,is_in_patient_capacity,maximum_capacity,is_isolation_rooms_wards,number_of_isolation_rooms,is_warehousing,is_cold_chain,general_medical_services,specialized_medical_beyond_primary_level,other_services,blood_services,total_number_of_human_resource,general_practitioner,specialist,residents_doctor,nurse,dentist,nursing_aid,midwife,other_medical_heal,other_profiles,feedback,professional_training_facilities,ambulance_type_a,ambulance_type_b,ambulance_type_c,residential_long_term_care_facilities,primary_health_care_center,other_affiliation,local_branch_name,english_branch_name,subtype,level,postcode,address_loc,address_en,city_loc,city_en,focal_person_loc,focal_person_en,phone,email,link,source_en,source_loc,date_of_data,visibility,longitude,latitude diff --git a/local_units/bulk_upload.py b/local_units/bulk_upload.py index eb388c94d..e70c2958c 100644 --- a/local_units/bulk_upload.py +++ b/local_units/bulk_upload.py @@ -1,11 +1,14 @@ -import csv import io import logging from dataclasses import dataclass +from datetime import date, datetime from typing import Any, Dict, Generic, Literal, Optional, Type, TypeVar +import openpyxl from django.core.files.base import ContentFile from django.db import transaction +from openpyxl import Workbook +from openpyxl.styles import PatternFill from rest_framework.exceptions import ErrorDetail from rest_framework.serializers import Serializer @@ -15,7 +18,6 @@ logger = logging.getLogger(__name__) - ContextType = TypeVar("ContextType") @@ -26,74 +28,89 @@ class BulkUploadError(Exception): class ErrorWriter: - def __init__(self, fieldnames: list[str]): - self._fieldnames = ["upload_status"] + fieldnames + ERROR_ROW_FILL = PatternFill(start_color="FF0000", end_color="FF0000", fill_type="solid") + + def __init__(self, fieldnames: list[str], header_map: dict[str, str]): + """Initialize workbook and header row.""" + self._header_map = header_map or {} + self._reverse_header_map = {v: k for k, v in self._header_map.items()} + # Convert model field names to xlsx template headers + display_header = [self._reverse_header_map.get(field, field) for field in fieldnames] + + self._fieldnames = ["Upload Status"] + display_header + self._workbook = Workbook() + self._ws = self._workbook.active + + self._ws.append(self._fieldnames) + self._existing_error_columns = set() self._rows: list[dict[str, str]] = [] - self._output = io.StringIO() - self._writer = csv.DictWriter(self._output, fieldnames=self._fieldnames) - self._writer.writeheader() self._has_errors = False def _format_errors(self, errors: dict) -> dict[str, list[str]]: - """Format serializer errors into field_name and list of messages.""" - error = {} + """Recursively flatten DRF errors.""" + formatted = {} for key, value in errors.items(): if isinstance(value, dict): - for inner_key, inner_value in self._format_errors(value).items(): - error[inner_key] = inner_value + formatted.update(self._format_errors(value)) elif isinstance(value, list): - error[key] = [self._clean_message(v) for v in value] + header = self._reverse_header_map.get(key, key) + formatted[header] = [self._clean_message(v) for v in value] else: - error[key] = [self._clean_message(value)] - return error + header = self._reverse_header_map.get(key, key) + formatted[header] = [self._clean_message(value)] + return formatted - def _clean_message(self, msg: Any) -> str: - """Convert ErrorDetail or other objects into normal text.""" + def _clean_message(self, msg: any) -> str: if isinstance(msg, ErrorDetail): return str(msg) return str(msg) - def _update_csv_header_with_errors(self): - """Update the CSV with updated headers when new error columns are introduced.""" - self._output.seek(0) - self._output.truncate() - self._writer = csv.DictWriter(self._output, fieldnames=self._fieldnames) - self._writer.writeheader() - for row in self._rows: - self._writer.writerow(row) + def _add_error_columns(self, fields: list[str]): + """Ensure field has a matching _error column.""" + for field in fields: + col_name = f"{field}_error" + if col_name in self._existing_error_columns: + continue + self._existing_error_columns.add(col_name) + + if field in self._fieldnames: + idx = self._fieldnames.index(field) + 1 + self._fieldnames.insert(idx, col_name) + self._ws.insert_cols(idx + 1) + else: + self._fieldnames.append(col_name) + for i, col_name in enumerate(self._fieldnames, start=1): + self._ws.cell(row=1, column=i, value=col_name) def write( self, - row: dict[str, str], + row: dict[str, any], status: Literal[LocalUnitBulkUpload.Status.SUCCESS, LocalUnitBulkUpload.Status.FAILED], error_detail: dict | None = None, - ) -> None: - row_copy = {col: row.get(col, "") for col in self._fieldnames} - row_copy["upload_status"] = status.name - added_error_column = False + ): + row_display = {self._reverse_header_map.get(k, k): v for k, v in row.items()} + row_out = {col: row_display.get(col, "") for col in self._fieldnames} + row_out["Upload Status"] = status.name if status == LocalUnitBulkUpload.Status.FAILED and error_detail: - formatted_errors = self._format_errors(error_detail) - for field, messages in formatted_errors.items(): - error_col = f"{field}_error" - - if error_col not in self._fieldnames: - if field in self._fieldnames: - col_idx = self._fieldnames.index(field) - self._fieldnames.insert(col_idx + 1, error_col) - else: - self._fieldnames.append(error_col) + formatted = self._format_errors(error_detail) + self._add_error_columns(list(formatted.keys())) + for field, msgs in formatted.items(): + row_out[f"{field}_error"] = "; ".join(msgs) + self._has_errors = True - added_error_column = True - row_copy[error_col] = "; ".join(messages) - self._rows.append(row_copy) - if added_error_column: - self._update_csv_header_with_errors() - else: - self._writer.writerow(row_copy) + self._ws.append([row_out.get(col, "") for col in self._fieldnames]) + + if status == LocalUnitBulkUpload.Status.FAILED: + for cell in self._ws[self._ws.max_row]: + cell.fill = self.ERROR_ROW_FILL def to_content_file(self) -> ContentFile: - return ContentFile(self._output.getvalue().encode("utf-8")) + """Export workbook as Content File for Django.""" + buffer = io.BytesIO() + self._workbook.save(buffer) + buffer.seek(0) + return ContentFile(buffer.getvalue()) class BaseBulkUpload(Generic[ContextType]): @@ -116,12 +133,15 @@ def delete_existing_local_unit(self): """Delete existing local units based on the context.""" pass - def _validate_csv(self, fieldnames) -> None: + def _validate_type(self, fieldnames) -> None: pass - def _is_csv_empty(self, csv_reader: csv.DictReader) -> tuple[bool, list[dict]]: - rows = list(csv_reader) - return len(rows) == 0, rows + def is_excel_data_empty(self, sheet, data_start_row=4): + """Check if file is empty or not""" + for row in sheet.iter_rows(values_only=True, min_row=data_start_row): + if any(cell is not None for cell in row): + return False + return True def process_row(self, data: Dict[str, Any]) -> bool: serializer = self.serializer_class(data=data) @@ -132,49 +152,71 @@ def process_row(self, data: Dict[str, Any]) -> bool: return False def run(self) -> None: - with self.bulk_upload.file.open("rb") as csv_file: - file = io.TextIOWrapper(csv_file, encoding="utf-8") - csv_reader = csv.DictReader(file) - fieldnames = csv_reader.fieldnames or [] + with self.bulk_upload.file.open("rb") as f: try: - is_empty, rows = self._is_csv_empty(csv_reader) - if is_empty: - raise BulkUploadError("The uploaded CSV file is empty or contains only blank rows.") - - csv_reader = iter(rows) - - self._validate_csv(fieldnames) - except BulkUploadError as e: + # TODO(sudip): Use read_only while reading xlsx file + workbook = openpyxl.load_workbook(f, data_only=True) + sheet = workbook.active + header_row_index = 2 + data_row_index = header_row_index + 2 + + # Read header row + headers = next(sheet.iter_rows(values_only=True, min_row=header_row_index, max_row=header_row_index)) + raw_fieldnames = [str(h).strip() for h in headers if h and str(h).strip()] + header_map = getattr(self, "HEADER_MAP", {}) or {} + mapped_fieldnames = [header_map.get(h, h) for h in raw_fieldnames] + fieldnames = mapped_fieldnames + + if self.is_excel_data_empty(sheet, data_start_row=data_row_index): + raise BulkUploadError("The uploaded Excel file is empty. Please provide at least one data row.") + + self._validate_type(fieldnames) + data_rows = ( + row + for row in sheet.iter_rows(values_only=True, min_row=4) + if any(cell is not None for cell in row) # skip the empty rows + ) + except Exception as e: self.bulk_upload.status = LocalUnitBulkUpload.Status.FAILED self.bulk_upload.error_message = str(e) self.bulk_upload.save(update_fields=["status", "error_message"]) logger.warning(f"[BulkUpload:{self.bulk_upload.pk}] Validation error: {str(e)}") return - context = self.get_context().__dict__ - self.error_writer = ErrorWriter(fieldnames) - try: - with transaction.atomic(): - self.delete_existing_local_unit() - for row_index, row_data in enumerate(csv_reader, start=2): - data = {**row_data, **context} - if self.process_row(data): - self.success_count += 1 - self.error_writer.write(row_data, status=LocalUnitBulkUpload.Status.SUCCESS) - else: - self.failed_count += 1 - self.error_writer.write( - row_data, status=LocalUnitBulkUpload.Status.FAILED, error_detail=self.error_detail - ) - logger.warning(f"[BulkUpload:{self.bulk_upload.pk}] Row '{row_index}' failed") - - if self.failed_count > 0: - raise BulkUploadError("Bulk upload failed with some errors.") - - self.bulk_manager.done() - self._finalize_success() - except BulkUploadError: - self._finalize_failure() + context = self.get_context().__dict__ + self.error_writer = ErrorWriter(fieldnames=raw_fieldnames, header_map=header_map) + + try: + with transaction.atomic(): + self.delete_existing_local_unit() + + for row_index, row_values in enumerate(data_rows, start=data_row_index): + row_dict = dict(zip(fieldnames, row_values)) + row_dict = {**row_dict, **context} + # Convert datetime objects to strings + for key, value in row_dict.items(): + if isinstance(value, (datetime, date)): + row_dict[key] = value.strftime("%Y-%m-%d") + if self.process_row(row_dict): + self.success_count += 1 + self.error_writer.write(row_dict, status=LocalUnitBulkUpload.Status.SUCCESS) + else: + self.failed_count += 1 + self.error_writer.write( + row_dict, + status=LocalUnitBulkUpload.Status.FAILED, + error_detail=self.error_detail, + ) + logger.warning(f"[BulkUpload:{self.bulk_upload.pk}] Row {row_index} failed") + + if self.failed_count > 0: + raise BulkUploadError("Bulk upload failed with some errors.") + + self.bulk_manager.done() + self._finalize_success() + + except BulkUploadError: + self._finalize_failure() def _finalize_success(self) -> None: self.bulk_upload.success_count = self.success_count @@ -187,7 +229,7 @@ def _finalize_success(self) -> None: def _finalize_failure(self) -> None: if self.error_writer: error_file = self.error_writer.to_content_file() - self.bulk_upload.error_file.save("error_file.csv", error_file, save=True) + self.bulk_upload.error_file.save("error_file.xlsx", error_file, save=True) self.bulk_upload.success_count = self.success_count self.bulk_upload.failed_count = self.failed_count @@ -206,6 +248,28 @@ class LocalUnitUploadContext: class BaseBulkUploadLocalUnit(BaseBulkUpload[LocalUnitUploadContext]): + HEADER_MAP = { + "Date of Update": "date_of_data", + "Local Unit Name (En)": "english_branch_name", + "Local Unit Name (Local)": "local_branch_name", + "Visibility": "visibility", + "Coverage": "level", + "Sub-type": "subtype", + "Focal Person (En)": "focal_person_en", + "Source (En)": "source_en", + "Source (Local)": "source_loc", + "Focal Person (Local)": "focal_person_loc", + "Address (Local)": "address_loc", + "Address (En)": "address_en", + "Locality (Local)": "city_loc", + "Locality (En)": "city_en", + "Local Unit Post Code": "postcode", + "Local Unit Email": "email", + "Local Unit Phone Number": "phone", + "Local Unit Website": "link", + "Latitude": "latitude", + "Longitude": "longitude", + } def __init__(self, bulk_upload: LocalUnitBulkUpload): from local_units.serializers import LocalUnitBulkUploadDetailSerializer @@ -232,19 +296,66 @@ def delete_existing_local_unit(self): else: logger.info("No existing local units found for deletion.") - def _validate_csv(self, fieldnames) -> None: + def _validate_type(self, fieldnames: list[str]) -> None: + health_field_names = set(get_model_field_names(HealthData)) - present_health_fields = health_field_names & set(fieldnames) + present_health_fields = {h for h in health_field_names if h.lower() in [f.lower() for f in fieldnames]} local_unit_type = LocalUnitType.objects.filter(id=self.bulk_upload.local_unit_type_id).first() if not local_unit_type: raise ValueError("Invalid local unit type") if present_health_fields and local_unit_type.name.strip().lower() != "health care": - raise BulkUploadError(f"You cannot upload Healthcare data when the Local Unit type is set to {local_unit_type.name}.") + raise BulkUploadError( + f"You cannot upload Healthcare data when the Local Unit type is set to '{local_unit_type.name}'." + ) class BulkUploadHealthData(BaseBulkUpload[LocalUnitUploadContext]): + # Local Unit headers + Health Data headers + HEADER_MAP = { + **BaseBulkUploadLocalUnit.HEADER_MAP, + **{ + "Focal Person Email": "focal_point_email", + "Focal Person Phone Number": "focal_point_phone_number", + "Focal Person Position": "focal_point_position", + "Health Facility Type": "health_facility_type", + "Other Facility Type": "other_facility_type", + "Affiliation": "affiliation", + "Other Affiliation": "other_affiliation", + "Functionality": "functionality", + "Primary Health Care Center": "primary_health_care_center", + "Specialities": "speciality", + "Hospital Type": "hospital_type", + "Teaching Hospital": "is_teaching_hospital", + "In-patient Capacity": "is_in_patient_capacity", + "Isolation Rooms": "is_isolation_rooms_wards", + "Number of Isolation Beds": "number_of_isolation_rooms", + "Warehousing": "is_warehousing", + "Cold Chain": "is_cold_chain", + "Maximum Bed Capacity": "maximum_capacity", + "General Medical Services": "general_medical_services", + "Specialized Medical Services (beyond primary level)": "specialized_medical_beyond_primary_level", + "Blood Services": "blood_services", + "Other Services": "other_services", + "Total Number of Human Resources": "total_number_of_human_resource", + "General Practitioners": "general_practitioner", + "Resident Doctors": "residents_doctor", + "Specialists": "specialist", + "Nurses": "nurse", + "Nursing Aids": "nursing_aid", + "Dentists": "dentist", + "Midwife": "midwife", + "Pharmacists": "pharmacists", + "Other Profiles": "other_profiles", + "Other Training Facility": "other_training_facilities", + "Professional Training Facilities": "professional_training_facilities", + "Ambulance Type A": "ambulance_type_a", + "Ambulance Type B": "ambulance_type_b", + "Ambulance Type C": "ambulance_type_c", + }, + } + def __init__(self, bulk_upload: LocalUnitBulkUpload): from local_units.serializers import LocalUnitBulkUploadDetailSerializer @@ -277,7 +388,17 @@ def delete_existing_local_unit(self): logger.info("No existing local units found for deletion.") def process_row(self, data: dict[str, any]) -> bool: - health_data = {k: data.get(k) for k in list(data.keys()) if k in self.health_field_names} + from local_units.serializers import HealthDataBulkUploadSerializer + + health_data = {k: data.get(k) for k in data.keys() if k in self.health_field_names} + if health_data: - data["health"] = health_data - return super().process_row(data) + health_serializer = HealthDataBulkUploadSerializer(data=health_data) + if not health_serializer.is_valid(): + self.error_detail = health_serializer.errors + return False + health_instance = health_serializer.save() + for k in health_data.keys(): + data.pop(k, None) + data["health"] = health_instance.pk + return super().process_row(data) diff --git a/local_units/models.py b/local_units/models.py index 158c52beb..bd987df74 100644 --- a/local_units/models.py +++ b/local_units/models.py @@ -203,6 +203,7 @@ class HealthData(models.Model): null=True, blank=True, ) + ambulance_type_a = models.IntegerField(verbose_name=_("Ambulance Type A"), blank=True, null=True) ambulance_type_b = models.IntegerField(verbose_name=_("Ambulance Type B"), blank=True, null=True) ambulance_type_c = models.IntegerField(verbose_name=_("Ambulance Type C"), blank=True, null=True) diff --git a/local_units/serializers.py b/local_units/serializers.py index b22172740..e31190658 100644 --- a/local_units/serializers.py +++ b/local_units/serializers.py @@ -703,8 +703,8 @@ class Meta: ) def validate_file(self, file): - if not file.name.endswith(".csv"): - raise serializers.ValidationError(gettext("File must be a CSV file.")) + if not file.name.endswith(".xlsx"): + raise serializers.ValidationError(gettext("File must be a xlsx file.")) if file.size > 10 * 1024 * 1024: raise serializers.ValidationError(gettext("File must be less than 10 MB.")) return file @@ -871,7 +871,7 @@ class LocalUnitBulkUploadDetailSerializer(serializers.ModelSerializer): visibility = serializers.CharField(required=True, allow_blank=True) date_of_data = serializers.CharField(required=False, allow_null=True) level = serializers.CharField(required=False, allow_null=True) - health = HealthDataBulkUploadSerializer(required=False) + health = serializers.PrimaryKeyRelatedField(queryset=HealthData.objects.all(), required=False, allow_null=True) class Meta: model = LocalUnit @@ -952,8 +952,4 @@ def validate(self, validated_data): validated_data["status"] = LocalUnit.Status.EXTERNALLY_MANAGED # NOTE: Bulk upload doesn't call create() method - health_data = validated_data.pop("health", None) - if health_data: - health_instance = HealthData.objects.create(**health_data) - validated_data["health"] = health_instance return validated_data diff --git a/local_units/test_views.py b/local_units/test_views.py index 768b71013..47c08ecad 100644 --- a/local_units/test_views.py +++ b/local_units/test_views.py @@ -1177,15 +1177,15 @@ def setUp(self): global_group.permissions.add(global_permission) self.global_validator_user.groups.add(global_group) - file_path = os.path.join(settings.TEST_DIR, "local_unit/test.csv") + file_path = os.path.join(settings.TEST_DIR, "local_unit/test-admin.xlsx") with open(file_path, "rb") as f: self._file_content = f.read() - def create_upload_file(self, filename="test.csv"): + def create_upload_file(self, filename="test-admin.xlsx"): """ Always return a new file instance to prevent stream exhaustion. """ - return SimpleUploadedFile(filename, self._file_content, content_type="text/csv") + return SimpleUploadedFile(filename, self._file_content, content_type="text/xlsx") @mock.patch("local_units.tasks.process_bulk_upload_local_unit.delay") def test_bulk_upload_local_unit(self, mock_delay): @@ -1330,16 +1330,16 @@ def setUpTestData(cls): cls.local_unit_type = LocalUnitType.objects.create(code=1, name="Administrative") cls.local_unit_type2 = LocalUnitType.objects.create(code=2, name="Health Care") cls.level = LocalUnitLevel.objects.create(level=0, name="National") - file_path = os.path.join(settings.TEST_DIR, "local_unit/test.csv") + file_path = os.path.join(settings.TEST_DIR, "local_unit/test-admin.xlsx") with open(file_path, "rb") as f: cls._file_content = f.read() - def create_upload_file(cls, filename="test.csv"): - return SimpleUploadedFile(filename, cls._file_content, content_type="text/csv") + def create_upload_file(cls, filename="test-admin.xlsx"): + return SimpleUploadedFile(filename, cls._file_content, content_type="text/xlsx") def test_bulk_upload_with_incorrect_country(cls): """ - Test bulk upload fails when the country does not match CSV data. + Test bulk upload fails when the country does not match xlsx data. """ cls.bulk_upload = LocalUnitBulkUploadFactory.create( country=cls.country1, @@ -1362,13 +1362,13 @@ def test_bulk_upload_with_incorrect_country(cls): def test_bulk_upload_with_valid_country(cls): """ - Test bulk upload succeeds when the country matches CSV data + Test bulk upload succeeds when the country matches xlsx data """ cls.bulk_upload = LocalUnitBulkUploadFactory.create( country=cls.country2, # Brazil local_unit_type=cls.local_unit_type, triggered_by=cls.user, - file=cls.create_upload_file(), # CSV with Brazil rows + file=cls.create_upload_file(), # xlsx with Brazil rows status=LocalUnitBulkUpload.Status.PENDING, ) runner = BaseBulkUploadLocalUnit(cls.bulk_upload) @@ -1381,7 +1381,7 @@ def test_bulk_upload_with_valid_country(cls): def test_bulk_upload_fails_and_delete(cls): """ - Test bulk upload fails and delete when CSV has incorrect data. + Test bulk upload fails and delete when xlsx has incorrect data. """ LocalUnitFactory.create_batch( 5, @@ -1410,7 +1410,7 @@ def test_bulk_upload_fails_and_delete(cls): def test_bulk_upload_deletes_old_and_creates_new_local_units(cls): """ - Test bulk upload with correct CSV data. + Test bulk upload with correct data. """ old_local_unit = LocalUnitFactory.create( country=cls.country2, @@ -1442,10 +1442,14 @@ def test_empty_administrative_file(cls): Test bulk upload file is empty """ - file_path = os.path.join(settings.STATICFILES_DIRS[0], "files", "local_units", "local-unit-bulk-upload-template.csv") + file_path = os.path.join( + settings.STATICFILES_DIRS[0], "files", "local_units", "Administrative Bulk Import Template - Local Units.xlsx" + ) with open(file_path, "rb") as f: file_content = f.read() - empty_file = SimpleUploadedFile(name="local-unit-bulk-upload-template.csv", content=file_content, content_type="text/csv") + empty_file = SimpleUploadedFile( + name="Administrative Bulk Import Template - Local Units.xlsx", content=file_content, content_type="text/xlsx" + ) LocalUnitFactory.create_batch( 5, country=cls.country2, @@ -1531,16 +1535,16 @@ def setUpTestData(cls): cls.professional_training_facilities = ProfessionalTrainingFacility.objects.create(code=1, name="Nurses") cls.general_medical_services = GeneralMedicalService.objects.create(code=1, name="Minor Trauma") - file_path = os.path.join(settings.TEST_DIR, "local_unit/test-health.csv") + file_path = os.path.join(settings.TEST_DIR, "local_unit/test-health.xlsx") with open(file_path, "rb") as f: cls._file_content = f.read() - def create_upload_file(cls, filename="test-health.csv"): - return SimpleUploadedFile(filename, cls._file_content, content_type="text/csv") + def create_upload_file(cls, filename="test-health.xlsx"): + return SimpleUploadedFile(filename, cls._file_content, content_type="text/xlsx") def test_bulk_upload_health_with_incorrect_country(cls): """ - Should fail when CSV rows are not equal to bulk upload country. + Should fail when rows are not equal to bulk upload country. """ cls.bulk_upload = LocalUnitBulkUploadFactory.create( country=cls.country1, @@ -1640,12 +1644,12 @@ def test_empty_health_template_file(cls): """ file_path = os.path.join( - settings.STATICFILES_DIRS[0], "files", "local_units", "local-unit-health-bulk-upload-template.csv" + settings.STATICFILES_DIRS[0], "files", "local_units", "Health Care Bulk Import Template - Local Units.xlsx" ) with open(file_path, "rb") as f: file_content = f.read() empty_file = SimpleUploadedFile( - name="local-unit-health-bulk-upload-template.csv", content=file_content, content_type="text/csv" + name="Health Care Bulk Import Template - Local Units.xlsx", content=file_content, content_type="text/xlsx" ) health_data = HealthDataFactory.create_batch( 5, diff --git a/local_units/utils.py b/local_units/utils.py index 2f2fabe1a..852d94c07 100644 --- a/local_units/utils.py +++ b/local_units/utils.py @@ -72,16 +72,13 @@ def get_model_field_names( def normalize_bool(value): - if isinstance(value, bool): - return value if not value: return False val = str(value).strip().lower() - if val in ("true", "1", "yes", "y"): + if val in ("yes"): return True - if val in ("false", "0", "no", "n"): + if val in ("no"): return False - return False def wash(string): @@ -91,4 +88,4 @@ def wash(string): def numerize(value): - return value if value.isdigit() else 0 + return value if value else 0 diff --git a/local_units/views.py b/local_units/views.py index 38f637a16..59a4320cf 100644 --- a/local_units/views.py +++ b/local_units/views.py @@ -436,8 +436,10 @@ class LocalUnitBulkUploadViewSet( def get_bulk_upload_template(self, request): template_type = request.query_params.get("bulk_upload_template", "local_unit") if template_type == "health_care": - file_url = request.build_absolute_uri(static("files/local_units/local-unit-health-bulk-upload-template.csv")) + file_url = request.build_absolute_uri(static("files/local_units/Health Care Bulk Import Template - Local Units.xlsx")) else: - file_url = request.build_absolute_uri(static("files/local_units/local-unit-bulk-upload-template.csv")) + file_url = request.build_absolute_uri( + static("files/local_units/Administrative Bulk Import Template - Local Units.xlsx") + ) template = {"template_url": file_url} return response.Response(LocalUnitTemplateFilesSerializer(template).data) diff --git a/main/test_files/local_unit/test-admin.xlsx b/main/test_files/local_unit/test-admin.xlsx new file mode 100644 index 000000000..bfb2b4904 Binary files /dev/null and b/main/test_files/local_unit/test-admin.xlsx differ diff --git a/main/test_files/local_unit/test-health.csv b/main/test_files/local_unit/test-health.csv deleted file mode 100644 index 199297414..000000000 --- a/main/test_files/local_unit/test-health.csv +++ /dev/null @@ -1,4 +0,0 @@ -focal_point_email,focal_point_phone_number,focal_point_position,health_facility_type,other_facility_type,affiliation,functionality,primary_health_care_centre,speciality,hospital_type,is_teaching_hospital,is_in_patient_capacity,maximum_capacity,is_isolation_rooms_wards,number_of_isolation_rooms,is_warehousing,is_cold_chain,general_medical_services,specialized_medical_beyond_primary_level,other_services,blood_services,total_number_of_human_resource,general_practitioner,specialist,residents_doctor,nurse,dentist,nursing_aid,midwife,other_medical_heal,feedback,professional_training_facilities,ambulance_type_a,ambulance_type_b,ambulance_type_c,residential_long_term_care_facilities,primary_health_care_center,other_affiliation,local_branch_name,english_branch_name,subtype,level,postcode,address_loc,address_en,city_loc,city_en,focal_person_loc,focal_person_en,phone,email,link,source_en,source_loc,date_of_data,visibility,longitude,latitude -jele@redcross1.org.sz,26876088546,Programmes Manager,Ambulance Station,tet,Public Government Facility,Fully Functional,Medical Practices,"Initiate TB treatment, Cervical Cancer Screening and testing and diagnostic and treatment for people living with HIV and follow up care through the ART programme which the government supports very well",Mental Health Hospital,yes,No,2,yes,2,Yes,Yes,Minor Trauma,Anaesthesiology,test,Blood Collection,32,0,0,0,3,0,0,9,Yes,first question of initial question did not provide for the option to write the name of the NS. It is written LRC yet it should allow Baphalali Eswatini Red Cross Society (BERCS) to be inscribed in the box.,Nurses,2,1,1,1,Dental Practices,test,Cruz Vermelha - Órgão Central HQ - Brasília,,Office,National,70300-910,"Setor Comercial Sul (SCS), quadra 6, bloco A, nº 157, salas 502 e 503 - Edifício Bandeirantes, Asa Sul, Brasília-DF ",,Brasília - DF,,Lourenço Braga,,55 (61) 99909-0761,secretario.geral@cvb.org.br,http://www.cruzvermelha.org.br/,Brazilian Red Cross,Brazilian Red Cross Local,2024-02-07,Public,-47.88972222,-15.79638889 -jele@redcross2.org.sz,26876088546,Programmes Manager,Ambulance Station,tet,Public Government Facility,Fully Functional,"Medical Practices,Dental Practices","Initiate TB treatment, Cervical Cancer Screening and testing and diagnostic and treatment for people living with HIV and follow up care through the ART programme which the government supports very well",Mental Health Hospital,No ,No,2,yes,2,Yes,Yes,Minor Trauma,Anaesthesiology,test,Blood Collection,32,0,0,0,3,0,0,9,Yes,first question of initial question did not provide for the option to write the name of the NS. It is written LRC yet it should allow Baphalali Eswatini Red Cross Society (BERCS) to be inscribed in the box.,Nurses,1,2,3,1,Dental Practices,test,Filial Alagoas,,Office,National,57035-530,"Av. Com. Gustavo de Paiva, 2889 - Mangabeiras, Maceió - AL",,Alagoas - AL,,Agarina Mendonça,,55 (82) 3325-2430,diretoria@cvbal.org.br,https://www.cruzvermelha.org.br/pb/filiais/alagoas/,Brazilian Red Cross,Brazilian Red Cross Local,2024-02-07,Public,-35.71611111,-9.64777778 -jele@redcross.org.sz,26876088546,Programmes Manager,Ambulance Station,tet,Public Government Facility,Fully Functional,Medical Practices,"Initiate TB treatment, Cervical Cancer Screening and testing and diagnostic and treatment for people living with HIV and follow up care through the ART programme which the government supports very well",Mental Health Hospital,No ,No,2,no,2,Yes,Yes,Minor Trauma,Anaesthesiology,test,Blood Collection,32,0,0,0,3,0,0,9,Yes,first question of initial question did not provide for the option to write the name of the NS. It is written LRC yet it should allow Baphalali Eswatini Red Cross Society (BERCS) to be inscribed in the box.,Nurses,1,2,2,1,Dental Practices,test,Cruz Vermelha - Órgão Central HQ,,Office,National,20230-130,"Praça Cruz Vermelha N° 10-12, Centro, Rio de Janeiro - RJ",,Rio de Janeiro - RJ,,Thiago Quintaneiro,,55 (21) 2507-3577 / 2507-3392,secretario.cooperacao@cvb.org.br,http://www.cruzvermelha.org.br/,Brazilian Red Cross,Brazilian Red Cross Local,2024-02-07,Public,-43.1875,-22.91111111 diff --git a/main/test_files/local_unit/test-health.xlsx b/main/test_files/local_unit/test-health.xlsx new file mode 100644 index 000000000..d819ddc76 Binary files /dev/null and b/main/test_files/local_unit/test-health.xlsx differ diff --git a/main/test_files/local_unit/test.csv b/main/test_files/local_unit/test.csv deleted file mode 100644 index 4860fc255..000000000 --- a/main/test_files/local_unit/test.csv +++ /dev/null @@ -1,4 +0,0 @@ -local_branch_name,english_branch_name,subtype,level,postcode,address_loc,address_en,city_loc,city_en,focal_person_loc,focal_person_en,phone,email,link,source_en,source_loc,date_of_data,visibility,longitude,latitude -Cruz Vermelha - Órgão Central HQ,,Office,National,20230-130,"Praça Cruz Vermelha N° 10-12, Centro, Rio de Janeiro - RJ",,Rio de Janeiro - RJ,,Thiago Quintaneiro,,55 (21) 2507-3577 / 2507-3392,secretario.cooperacao@cvb.org.br,http://www.cruzvermelha.org.br/,Brazilian Red Cross,Brazilian Red Cross Local,2024-02-07,Public,-43.1875,-22.91111111 -Cruz Vermelha - Órgão Central HQ - Brasília,,Office,National,70300-910,"Setor Comercial Sul (SCS), quadra 6, bloco A, nº 157, salas 502 e 503 - Edifício Bandeirantes, Asa Sul, Brasília-DF ",,Brasília - DF,,Lourenço Braga,,55 (61) 99909-0761,secretario.geral@cvb.org.br,http://www.cruzvermelha.org.br/,Brazilian Red Cross,Brazilian Red Cross Local,2024-02-07,Public,-47.88972222,-15.79638889 -Filial Alagoas,,Office,National,57035-530,"Av. Com. Gustavo de Paiva, 2889 - Mangabeiras, Maceió - AL",,Alagoas - AL,,Agarina Mendonça,,55 (82) 3325-2430,diretoria@cvbal.org.br,https://www.cruzvermelha.org.br/pb/filiais/alagoas/,Brazilian Red Cross,Brazilian Red Cross Local,2024-02-07,Public,-35.71611111,-9.64777778