Refactor bulk upload logic for XLSX file processing

local_units/models.py

@@ -203,6 +203,7 @@ class HealthData(models.Model):
         null=True,
         blank=True,
     )
+
     ambulance_type_a = models.IntegerField(verbose_name=_("Ambulance Type A"), blank=True, null=True)
     ambulance_type_b = models.IntegerField(verbose_name=_("Ambulance Type B"), blank=True, null=True)
     ambulance_type_c = models.IntegerField(verbose_name=_("Ambulance Type C"), blank=True, null=True)

local_units/serializers.py

@@ -703,8 +703,8 @@ class Meta:
         )
 
     def validate_file(self, file):
-        if not file.name.endswith(".csv"):
-            raise serializers.ValidationError(gettext("File must be a CSV file."))
+        if not file.name.endswith(".xlsx"):
+            raise serializers.ValidationError(gettext("File must be a xlsx file."))
         if file.size > 10 * 1024 * 1024:
             raise serializers.ValidationError(gettext("File must be less than 10 MB."))
         return file
@@ -871,7 +871,7 @@ class LocalUnitBulkUploadDetailSerializer(serializers.ModelSerializer):
     visibility = serializers.CharField(required=True, allow_blank=True)
     date_of_data = serializers.CharField(required=False, allow_null=True)
     level = serializers.CharField(required=False, allow_null=True)
-    health = HealthDataBulkUploadSerializer(required=False)
+    health = serializers.PrimaryKeyRelatedField(queryset=HealthData.objects.all(), required=False, allow_null=True)
 
     class Meta:
         model = LocalUnit
@@ -952,8 +952,4 @@ def validate(self, validated_data):
         validated_data["status"] = LocalUnit.Status.EXTERNALLY_MANAGED
 
         # NOTE: Bulk upload doesn't call create() method
-        health_data = validated_data.pop("health", None)
-        if health_data:
-            health_instance = HealthData.objects.create(**health_data)
-            validated_data["health"] = health_instance
         return validated_data

local_units/test_views.py

@@ -1177,15 +1177,15 @@ def setUp(self):
         global_group.permissions.add(global_permission)
         self.global_validator_user.groups.add(global_group)
 
-        file_path = os.path.join(settings.TEST_DIR, "local_unit/test.csv")
+        file_path = os.path.join(settings.TEST_DIR, "local_unit/test-admin.xlsx")
         with open(file_path, "rb") as f:
             self._file_content = f.read()
 
-    def create_upload_file(self, filename="test.csv"):
+    def create_upload_file(self, filename="test-admin.xlsx"):
         """
         Always return a new file instance to prevent stream exhaustion.
         """
-        return SimpleUploadedFile(filename, self._file_content, content_type="text/csv")
+        return SimpleUploadedFile(filename, self._file_content, content_type="text/xlsx")
 
     @mock.patch("local_units.tasks.process_bulk_upload_local_unit.delay")
     def test_bulk_upload_local_unit(self, mock_delay):
@@ -1330,16 +1330,16 @@ def setUpTestData(cls):
         cls.local_unit_type = LocalUnitType.objects.create(code=1, name="Administrative")
         cls.local_unit_type2 = LocalUnitType.objects.create(code=2, name="Health Care")
         cls.level = LocalUnitLevel.objects.create(level=0, name="National")
-        file_path = os.path.join(settings.TEST_DIR, "local_unit/test.csv")
+        file_path = os.path.join(settings.TEST_DIR, "local_unit/test-admin.xlsx")
         with open(file_path, "rb") as f:
             cls._file_content = f.read()
 
-    def create_upload_file(cls, filename="test.csv"):
-        return SimpleUploadedFile(filename, cls._file_content, content_type="text/csv")
+    def create_upload_file(cls, filename="test-admin.xlsx"):
+        return SimpleUploadedFile(filename, cls._file_content, content_type="text/xlsx")
 
     def test_bulk_upload_with_incorrect_country(cls):
         """
-        Test bulk upload fails when the country does not match CSV data.
+        Test bulk upload fails when the country does not match xlsx data.
         """
         cls.bulk_upload = LocalUnitBulkUploadFactory.create(
             country=cls.country1,
@@ -1362,13 +1362,13 @@ def test_bulk_upload_with_incorrect_country(cls):
 
     def test_bulk_upload_with_valid_country(cls):
         """
-        Test bulk upload succeeds when the country matches CSV data
+        Test bulk upload succeeds when the country matches xlsx data
         """
         cls.bulk_upload = LocalUnitBulkUploadFactory.create(
             country=cls.country2,  # Brazil
             local_unit_type=cls.local_unit_type,
             triggered_by=cls.user,
-            file=cls.create_upload_file(),  # CSV with Brazil rows
+            file=cls.create_upload_file(),  # xlsx with Brazil rows
             status=LocalUnitBulkUpload.Status.PENDING,
         )
         runner = BaseBulkUploadLocalUnit(cls.bulk_upload)
@@ -1381,7 +1381,7 @@ def test_bulk_upload_with_valid_country(cls):
 
     def test_bulk_upload_fails_and_delete(cls):
         """
-        Test bulk upload fails and delete when CSV has incorrect data.
+        Test bulk upload fails and delete when xlsx has incorrect data.
         """
         LocalUnitFactory.create_batch(
             5,
@@ -1410,7 +1410,7 @@ def test_bulk_upload_fails_and_delete(cls):
 
     def test_bulk_upload_deletes_old_and_creates_new_local_units(cls):
         """
-        Test bulk upload with correct CSV data.
+        Test bulk upload with correct data.
         """
         old_local_unit = LocalUnitFactory.create(
             country=cls.country2,
@@ -1442,10 +1442,14 @@ def test_empty_administrative_file(cls):
         Test bulk upload file is empty
         """
 
-        file_path = os.path.join(settings.STATICFILES_DIRS[0], "files", "local_units", "local-unit-bulk-upload-template.csv")
+        file_path = os.path.join(
+            settings.STATICFILES_DIRS[0], "files", "local_units", "Administrative Bulk Import Template - Local Units.xlsx"
+        )
         with open(file_path, "rb") as f:
             file_content = f.read()
-        empty_file = SimpleUploadedFile(name="local-unit-bulk-upload-template.csv", content=file_content, content_type="text/csv")
+        empty_file = SimpleUploadedFile(
+            name="Administrative Bulk Import Template - Local Units.xlsx", content=file_content, content_type="text/xlsx"
+        )
         LocalUnitFactory.create_batch(
             5,
             country=cls.country2,
@@ -1531,16 +1535,16 @@ def setUpTestData(cls):
         cls.professional_training_facilities = ProfessionalTrainingFacility.objects.create(code=1, name="Nurses")
         cls.general_medical_services = GeneralMedicalService.objects.create(code=1, name="Minor Trauma")
 
-        file_path = os.path.join(settings.TEST_DIR, "local_unit/test-health.csv")
+        file_path = os.path.join(settings.TEST_DIR, "local_unit/test-health.xlsx")
         with open(file_path, "rb") as f:
             cls._file_content = f.read()
 
-    def create_upload_file(cls, filename="test-health.csv"):
-        return SimpleUploadedFile(filename, cls._file_content, content_type="text/csv")
+    def create_upload_file(cls, filename="test-health.xlsx"):
+        return SimpleUploadedFile(filename, cls._file_content, content_type="text/xlsx")
 
     def test_bulk_upload_health_with_incorrect_country(cls):
         """
-        Should fail when CSV rows are not equal to bulk upload country.
+        Should fail when rows are not equal to bulk upload country.
         """
         cls.bulk_upload = LocalUnitBulkUploadFactory.create(
             country=cls.country1,
@@ -1640,12 +1644,12 @@ def test_empty_health_template_file(cls):
         """
 
         file_path = os.path.join(
-            settings.STATICFILES_DIRS[0], "files", "local_units", "local-unit-health-bulk-upload-template.csv"
+            settings.STATICFILES_DIRS[0], "files", "local_units", "Health Care Bulk Import Template - Local Units.xlsx"
         )
         with open(file_path, "rb") as f:
             file_content = f.read()
         empty_file = SimpleUploadedFile(
-            name="local-unit-health-bulk-upload-template.csv", content=file_content, content_type="text/csv"
+            name="Health Care Bulk Import Template - Local Units.xlsx", content=file_content, content_type="text/xlsx"
         )
         health_data = HealthDataFactory.create_batch(
             5,

local_units/utils.py

@@ -72,16 +72,13 @@ def get_model_field_names(
 
 
 def normalize_bool(value):
-    if isinstance(value, bool):
-        return value
     if not value:
         return False
     val = str(value).strip().lower()
-    if val in ("true", "1", "yes", "y"):
+    if val in ("yes"):
         return True
-    if val in ("false", "0", "no", "n"):
+    if val in ("no"):
         return False
-    return False
 
 
 def wash(string):
@@ -91,4 +88,4 @@ def wash(string):
 
 
 def numerize(value):
-    return value if value.isdigit() else 0
+    return value if value else 0

local_units/views.py

@@ -436,8 +436,10 @@ class LocalUnitBulkUploadViewSet(
     def get_bulk_upload_template(self, request):
         template_type = request.query_params.get("bulk_upload_template", "local_unit")
         if template_type == "health_care":
-            file_url = request.build_absolute_uri(static("files/local_units/local-unit-health-bulk-upload-template.csv"))
+            file_url = request.build_absolute_uri(static("files/local_units/Health Care Bulk Import Template - Local Units.xlsx"))
         else:
-            file_url = request.build_absolute_uri(static("files/local_units/local-unit-bulk-upload-template.csv"))
+            file_url = request.build_absolute_uri(
+                static("files/local_units/Administrative Bulk Import Template - Local Units.xlsx")
+            )
         template = {"template_url": file_url}
         return response.Response(LocalUnitTemplateFilesSerializer(template).data)