Merge pull request #272 from QuanMPhm/fix/quotechar

Fixing quotechar for names with commas

Author: knikolla

process_report/process_report.py

@@ -110,6 +110,7 @@ def merge_csv(files):
                 invoice.COST_FIELD: pandas.ArrowDtype(pyarrow.decimal128(21, 2)),
                 invoice.RATE_FIELD: str,
             },
+            quotechar="|",
         )
         dataframes.append(dataframe)
 

process_report/tests/e2e/test_data/test_invoice_openshift.csv

@@ -1,5 +1,5 @@
 Invoice Month,Project - Allocation,Project - Allocation ID,Manager (PI),Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost
-2024-01,TestProject1,TP1,testpi1@bu.edu,testpi1@bu.edu,"123 Test St, Boston MA 02101",Boston University,BU,100,OpenShift CPU,0.013,50.00
-2024-01,TestProject2,TP2,testpi2@harvard.edu,testpi2@harvard.edu,"456 Harvard Ave, Cambridge MA 02138",Harvard University,HARVARD,200,OpenShift CPU,0.013,75.00
-2024-01,TestProject3,TP3,testpi3@mit.edu,testpi3@mit.edu,"789 MIT Blvd, Cambridge MA 02139",Massachusetts Institute of Technology,MIT,150,OpenShift CPU,0.013,100.00
-2024-01,NonbillableProject,NBP1,nonbillable@example.edu,nonbillable@example.edu,"999 Test Ave, Boston MA 02101",Test Institution,TEST,50,OpenShift CPU,0.013,25.00
+2024-01,TestProject1,TP1,testpi1@bu.edu,testpi1@bu.edu,|123 Test St, Boston MA 02101|,Boston University,BU,100,OpenShift CPU,0.013,50.00
+2024-01,TestProject2,TP2,testpi2@harvard.edu,testpi2@harvard.edu,|456 Harvard Ave, Cambridge MA 02138|,Harvard University,HARVARD,200,OpenShift CPU,0.013,75.00
+2024-01,TestProject3,TP3,testpi3@mit.edu,testpi3@mit.edu,|789 MIT Blvd, Cambridge MA 02139|,Massachusetts Institute of Technology,MIT,150,OpenShift CPU,0.013,100.00
+2024-01,NonbillableProject,NBP1,nonbillable@example.edu,nonbillable@example.edu,|999 Test Ave, Boston MA 02101|,Test Institution,TEST,50,OpenShift CPU,0.013,25.00

process_report/tests/e2e/test_data/test_invoice_openstack.csv

@@ -1,4 +1,4 @@
 Invoice Month,Project - Allocation,Project - Allocation ID,Manager (PI),Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost
-2024-01,TestProject1,TP1,testpi1@bu.edu,testpi1@bu.edu,"123 Test St, Boston MA 02101",Boston University,BU,50,OpenStack CPU,0.013,30.00
-2024-01,TestProject2,TP2,testpi2@harvard.edu,testpi2@harvard.edu,"456 Harvard Ave, Cambridge MA 02138",Harvard University,HARVARD,80,OpenStack CPU,0.013,40.00
-2024-01,TestProject4,TP4,testpi4@northeastern.edu,testpi4@northeastern.edu,"360 Huntington Ave, Boston MA 02115",Northeastern University,NEU,120,OpenStack CPU,0.013,65.00
+2024-01,TestProject1,TP1,testpi1@bu.edu,testpi1@bu.edu,|123 Test St, Boston MA 02101|,Boston University,BU,50,OpenStack CPU,0.013,30.00
+2024-01,TestProject2,TP2,testpi2@harvard.edu,testpi2@harvard.edu,|456 Harvard Ave, Cambridge MA 02138|,Harvard University,HARVARD,80,OpenStack CPU,0.013,40.00
+2024-01,TestProject4,TP4,testpi4@northeastern.edu,testpi4@northeastern.edu,|360 Huntington Ave, Boston MA 02115|,Northeastern University,NEU,120,OpenStack CPU,0.013,65.00

process_report/tests/e2e/test_data/test_invoice_storage.csv

@@ -1,3 +1,3 @@
 Invoice Month,Project - Allocation,Project - Allocation ID,Manager (PI),Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost
-2024-01,TestProject1,TP1,testpi1@bu.edu,testpi1@bu.edu,"123 Test St, Boston MA 02101",Boston University,BU,1000,Storage,0.0001,20.00
-2024-01,TestProject3,TP3,testpi3@mit.edu,testpi3@mit.edu,"789 MIT Blvd, Cambridge MA 02139",Massachusetts Institute of Technology,MIT,2000,Storage,0.0001,35.00
+2024-01,TestProject1,TP1,testpi1@bu.edu,testpi1@bu.edu,|123 Test St, Boston MA 02101|,Boston University,BU,1000,Storage,0.0001,20.00
+2024-01,TestProject3,TP3,testpi3@mit.edu,testpi3@mit.edu,|789 MIT Blvd, Cambridge MA 02139|,Massachusetts Institute of Technology,MIT,2000,Storage,0.0001,35.00

process_report/tests/e2e/test_data/test_invoices/test_nerc-ocp-test 2025-04.csv

@@ -1,5 +1,5 @@
 Invoice Month,Project - Allocation,Project - Allocation ID,Manager (PI),Cluster Name,Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost
-2024-01,P4ID,P4ID,pi4@example.edu,ocp-test,pi4@example.edu,"123 Test St, Boston MA 02101",Boston University,BU,280,OpenShift CPU,0.013,100.0
-2024-01,P4ID,P4ID,pi4@example.edu,ocp-test,pi4@example.edu,"123 Test St, Boston MA 02101",Boston University,BU,280,OpenStack GPUA100SXM4,0.013,100
-2024-01,P5ID,P5ID,pi5@harvard.edu,ocp-test,pi5@harvard.edu,"456 Harvard Ave, Cambridge MA 02138",Harvard University,HARVARD,280,OpenShift CPU,0.013,200
-2024-01,P6ID,P6ID,pi6@mit.edu,ocp-test,pi6@mit.edu,"789 MIT Blvd, Cambridge MA 02139",MIT,MIT,280,OpenShift CPU,0.013,300
+2024-01,P4ID,P4ID,pi4@example.edu,ocp-test,pi4@example.edu,|123 Test St, Boston MA 02101|,Boston University,BU,280,OpenShift CPU,0.013,100.0
+2024-01,P4ID,P4ID,pi4@example.edu,ocp-test,pi4@example.edu,|123 Test St, Boston MA 02101|,Boston University,BU,280,OpenStack GPUA100SXM4,0.013,100
+2024-01,P5ID,P5ID,pi5@harvard.edu,ocp-test,pi5@harvard.edu,|456 Harvard Ave, Cambridge MA 02138|,Harvard University,HARVARD,280,OpenShift CPU,0.013,200
+2024-01,P6ID,P6ID,pi6@mit.edu,ocp-test,pi6@mit.edu,|789 MIT Blvd, Cambridge MA 02139|,MIT,MIT,280,OpenShift CPU,0.013,300

process_report/tests/unit/test_util.py

@@ -28,7 +28,7 @@ class TestMergeCSV(TestCase):
     def setUp(self):
         self.header = ["Cost", "Name", "Rate"]
         self.data = [
-            [1, "Alice", 25],
+            [1, "Alice, Allison", 25],
             [2, "Bob", 30],
             [3, "Charlie", 28],
         ]
@@ -41,7 +41,7 @@ def setUp(self):
             )
             self.csv_files.append(csv_file)
             dataframe = pandas.DataFrame(self.data, columns=self.header)
-            dataframe.to_csv(csv_file, index=False)
+            dataframe.to_csv(csv_file, index=False, quotechar="|")
             csv_file.close()
 
     def tearDown(self):
@@ -61,6 +61,8 @@ def test_merge_csv(self):
         # Assert that the headers in the merged DataFrame match the expected headers
         assert merged_dataframe.columns.tolist() == self.header
 
+        assert merged_dataframe["Name"].iloc[0] == "Alice, Allison"
+
 
 class TestTimedProjects(TestCase):
     def setUp(self):