8972 export attendee csv error (#8952)

* Rewrite celery export and its helper task with Dictwriter, better validation

* Added new parameter for test case

* Test case logic change due to new return type

* Test case logic change due to new return type

* Recorrect test case

---------

Co-authored-by: Norbert Preining <[email protected]>

Author: catInOrbit

app/api/helpers/csv_jobs_util.py

@@ -57,72 +57,65 @@ def export_orders_csv(orders):
     return rows
 
 
-def export_attendees_csv(attendees, custom_forms):
+def export_attendees_csv(attendees, custom_forms, attendee_form_dict):
+    return_dict_list = []
 
-    headers = [
-        'Order#',
-        'Order Date',
-        'Status',
-        'Payment Type',
-        'Payment Mode',
-        'Ticket Name',
-        'Ticket Price',
-        'Ticket Type',
-        'Tax ID',
-        'Address',
-        'Company',
-        'Country',
-        'State',
-        'City',
-        'Zipcode',
-    ]
-
-    for fields in custom_forms:
-        headers.append(fields.name)
-
-    rows = [headers]
     for attendee in attendees:
-        column = [
-            str(attendee.order.get_invoice_number()) if attendee.order else '-',
-            str(attendee.order.created_at.strftime('%B %-d, %Y %H:%M %z'))
+        data = {
+            'Order#': str(attendee.order.get_invoice_number()) if attendee.order else '-',
+            'Order Date': str(attendee.order.created_at.strftime('%B %-d, %Y %H:%M %z'))
             if attendee.order and attendee.order.created_at
             else '-',
-            str(attendee.order.status)
+            'Status': str(attendee.order.status)
             if attendee.order and attendee.order.status
             else '-',
-            str(attendee.order.paid_via)
+            'Payment Type': str(attendee.order.paid_via)
             if attendee.order and attendee.order.paid_via
             else '',
-            str(attendee.order.payment_mode)
+            'Payment Mode':  str(attendee.order.payment_mode)
             if attendee.order and attendee.order.payment_mode
             else '',
-            str(attendee.ticket.name) if attendee.ticket and attendee.ticket.name else '',
-            str(attendee.ticket.price)
+            'Ticket Name':  str(attendee.ticket.name)
+            if attendee.ticket and attendee.ticket.name else '',
+            'Ticket Price': str(attendee.ticket.price)
             if attendee.ticket and attendee.ticket.price
             else '0',
-            str(attendee.ticket.type) if attendee.ticket and attendee.ticket.type else '',
-            str(attendee.order.tax_business_info)
+            'Ticket Type': str(attendee.ticket.type)
+            if attendee.ticket and attendee.ticket.type else '',
+            'Tax ID': str(attendee.order.tax_business_info)
             if attendee.order.tax_business_info
             else '',
-            str(attendee.order.address) if attendee.order.address else '',
-            str(attendee.order.company) if attendee.order.company else '',
-            str(attendee.order.country) if attendee.order.country else '',
-            str(attendee.order.state) if attendee.order.state else '',
-            str(attendee.order.city) if attendee.order.city else '',
-            str(attendee.order.zipcode) if attendee.order.zipcode else '',
-        ]
+            'Address': str(attendee.order.address) if attendee.order.address else '',
+            'Company':  str(attendee.order.company) if attendee.order.company else '',
+            'Country': str(attendee.order.country) if attendee.order.country else '',
+            'State':  str(attendee.order.state) if attendee.order.state else '',
+            'City': str(attendee.order.city) if attendee.order.city else '',
+            'Zipcode': str(attendee.order.zipcode) if attendee.order.zipcode else '',
+        }
+
         for field in custom_forms:
+            # keys don't match up, for keys like
+            # acceptVideoRecording vs accept_video_recording ..
+            key_mapping = {}
+
+            for k in attendee_form_dict.keys():
+                key_mapping[k.replace("_", "").lower()] = k
+
+            field_raw = field.identifier.replace("_", "").lower()
+            key = key_mapping.get(field_raw)
+            converted_header = attendee_form_dict.get(key)
             if field.is_complex:
                 fields_dict = attendee.complex_field_values
-                column.append(
-                    fields_dict.get(field.identifier, '') if fields_dict else ''
-                )
+                data[converted_header] = fields_dict.get(
+                    field.identifier, '') if fields_dict else ''
             else:
-                column.append(getattr(attendee, field.identifier, ''))
+                dict_value = (getattr(attendee, field.identifier, ''))
+                dict_value = "Yes" if str(dict_value) == "True" \
+                    else "No" if str(dict_value) == "False" else dict_value
+                data[converted_header] = dict_value
+        return_dict_list.append(data)
 
-        rows.append(column)
-
-    return rows
+    return return_dict_list
 
 
 def export_sessions_csv(sessions):

app/api/helpers/tasks.py

@@ -39,7 +39,7 @@
 from app.api.imports import import_event_task_base
 from app.instance import create_app
 from app.models import db
-from app.models.custom_form import CustomForms
+from app.models.custom_form import CustomForms, ATTENDEE_CUSTOM_FORM
 from app.models.discount_code import DiscountCode
 from app.models.event import Event
 from app.models.exhibitor import Exhibitor
@@ -52,8 +52,8 @@
 from app.models.user import User
 from app.models.user_follow_group import UserFollowGroup
 from app.settings import get_settings
-
 from .import_helpers import update_import_job
+from app.api.helpers.csv_jobs_util import export_attendees_csv
 
 """
 Define all API v2 celery tasks here
@@ -541,20 +541,43 @@ def export_attendees_csv_task(self, event_id):
     custom_forms = db.session.query(CustomForms).filter_by(
         event_id=event_id, form=CustomForms.TYPE.ATTENDEE, is_included=True
     )
+
+    field_headers = list(ATTENDEE_CUSTOM_FORM.keys())
+
+    def custom_form_validation(cf_orm, field_headers):
+        # set() is O(1) in membership testing
+        field_headers_set = set(field_headers)
+        forms_result = [None] * len(field_headers_set)
+
+        for row in cf_orm:
+            if row.field_identifier in field_headers_set:
+
+                field_headers_set.discard(row.field_identifier)
+                index_append = field_headers.index(row.field_identifier)
+                # forms_result.append(row)
+                forms_result.insert(index_append, row)
+
+        forms_result = [e for e in forms_result if e is not None]
+        return forms_result
+
     try:
+        custom_forms = custom_form_validation(custom_forms, field_headers)
         filedir = os.path.join(current_app.config.get('BASE_DIR'), 'static/uploads/temp/')
         if not os.path.isdir(filedir):
             os.makedirs(filedir)
         filename = f"attendees-{uuid.uuid1().hex}.csv"
         file_path = os.path.join(filedir, filename)
 
+        dict_list = export_attendees_csv(attendees, custom_forms, ATTENDEE_CUSTOM_FORM)
+        csv_headers = list(dict_list[0].keys())
+
         with open(file_path, "w") as temp_file:
-            writer = csv.writer(temp_file)
-            from app.api.helpers.csv_jobs_util import export_attendees_csv
+            writer = csv.DictWriter(temp_file, fieldnames=csv_headers)
+            writer.writeheader()
 
-            content = export_attendees_csv(attendees, custom_forms)
-            for row in content:
+            for row in dict_list:
                 writer.writerow(row)
+
         attendees_csv_file = UploadedFile(file_path=file_path, filename=filename)
         attendees_csv_url = upload(
             attendees_csv_file,

tests/all/integration/api/helpers/test_csv_jobs_util.py

@@ -11,6 +11,7 @@
 from tests.factories.order import OrderFactory
 from tests.factories.session import SessionSubFactory
 from tests.factories.speaker import SpeakerFactory
+from app.models.custom_form import ATTENDEE_CUSTOM_FORM
 
 
 class TestExportCSV(OpenEventTestCase):
@@ -32,8 +33,10 @@ def test_export_attendees_csv(self):
             test_order = OrderFactory(created_at=datetime.now())
             test_attendee.order = test_order
             custom_forms = CustomFormFactory()
-            field_data = export_attendees_csv([test_attendee], [custom_forms])
-            assert field_data[1][8] == 'tax id'
+            field_data = export_attendees_csv(
+                [test_attendee], [custom_forms], ATTENDEE_CUSTOM_FORM)
+            # new export_attendees_csv will return list of dictionary for csv_writer
+            assert field_data[0].get("Tax ID") == "tax id"
 
     def _test_export_session_csv(self, test_session=None):
         with self.app.test_request_context():