To_csv works, need to implement read_csv

Author: Jaspvr

pandas/core/generic.py

@@ -3939,6 +3939,7 @@ def to_csv(
             doublequote=doublequote,
             escapechar=escapechar,
             storage_options=storage_options,
+            preserve_complex=preserve_complex,
         )
 
     # ----------------------------------------------------------------------

pandas/io/formats/csvs.py

@@ -100,24 +100,23 @@ def __init__(
         self.cols = self._initialize_columns(cols)
         self.chunksize = self._initialize_chunksize(chunksize)
 
+        print("output preserve var: ", self.preserve_complex)
         if self.preserve_complex:
+            print("here1")
             import json
             import numpy as np
 
             for col in self.obj.columns:
                 if self.obj[col].dtype == "O":
-                    try:
-                        first_val = self.obj[col].iloc[0]
-                        if isinstance(first_val, (np.ndarray, list)):
-                            self.obj[col] = self.obj[col].apply(
-                                lambda x: json.dumps(x.tolist())
-                                if isinstance(x, np.ndarray)
-                                else json.dumps(x)
-                                if isinstance(x, list)
-                                else x
-                            )
-                    except Exception:
-                        continue
+                    first_val = self.obj[col].iloc[0]
+                    if isinstance(first_val, (np.ndarray, list)):
+                        self.obj[col] = self.obj[col].apply(
+                            lambda x: json.dumps(x.tolist())
+                            if isinstance(x, np.ndarray)
+                            else json.dumps(x) if isinstance(x, list)
+                            else x
+                        )
+
 
 
     @property

pandas/io/formats/format.py

@@ -1000,6 +1000,7 @@ def to_csv(
             doublequote=doublequote,
             escapechar=escapechar,
             storage_options=storage_options,
+            preserve_complex=preserve_complex,
             formatter=self.fmt,
         )
         csv_formatter.save()

scripts/tests/test_csv.py

@@ -1,28 +1,60 @@
+# import pandas as pd
+# import numpy as np
+
+# # Create a DataFrame with NumPy arrays
+# df = pd.DataFrame({
+#     'id': [1, 2],
+#     'embedding': [np.array([0.1, 0.2, 0.3]), np.array([0.4, 0.5, 0.6])]
+# })
+
+# # Save to CSV
+# csv_file = "test_numpy_array.csv"
+# df.to_csv(csv_file, index=False, preserve_complex=True)
+# print(f"Saved CSV:\n{open(csv_file).read()}")
+
+# # Read back the CSV
+# df_loaded = pd.read_csv(csv_file)
+
+# # Print results
+# print("\nLoaded DataFrame:")
+# print(df_loaded)
+
+# # ✅ **Make the test fail by checking if we correctly load NumPy arrays**
+# try:
+#     assert isinstance(df_loaded["embedding"][0], np.ndarray), "Test Failed: Embeddings were not preserved as NumPy arrays!"
+#     print("\nTest Passed: Embeddings were correctly preserved as NumPy arrays")
+# except AssertionError as e:
+#     print("\nTest Failed: Pandas does not preserve NumPy arrays in CSV, needs improvement!")
+#     raise e
+
 import pandas as pd
+print(pd.__file__)
+print(pd.__version__)
+
 import numpy as np
+import os
 
 # Create a DataFrame with NumPy arrays
 df = pd.DataFrame({
     'id': [1, 2],
     'embedding': [np.array([0.1, 0.2, 0.3]), np.array([0.4, 0.5, 0.6])]
 })
 
-# Save to CSV
+# Save to CSV (where your custom preserve_complex logic resides)
 csv_file = "test_numpy_array.csv"
 df.to_csv(csv_file, index=False, preserve_complex=True)
-print(f"Saved CSV:\n{open(csv_file).read()}")
 
-# Read back the CSV
-df_loaded = pd.read_csv(csv_file)
+# Read back the raw CSV content (as text only)
+with open(csv_file, "r") as f:
+    csv_content = f.read()
 
-# Print results
-print("\nLoaded DataFrame:")
-print(df_loaded)
+print(f"Saved CSV:\n{csv_content}")
 
-# ✅ **Make the test fail by checking if we correctly load NumPy arrays**
+# Simple test: check that our JSON-ified arrays are present in the CSV text
 try:
-    assert isinstance(df_loaded["embedding"][0], np.ndarray), "Test Failed: Embeddings were not preserved as NumPy arrays!"
-    print("\nTest Passed: Embeddings were correctly preserved as NumPy arrays")
-except AssertionError as e:
-    print("\nTest Failed: Pandas does not preserve NumPy arrays in CSV, needs improvement!")
-    raise e
+    assert "[0.1, 0.2, 0.3]" in csv_content
+    assert "[0.4, 0.5, 0.6]" in csv_content
+    print("\nTest Passed: The CSV output includes JSON-serialized arrays for 'embedding'.")
+except AssertionError:
+    print("\nTest Failed: The CSV does not appear to have JSON-serialized arrays as expected!")
+    raise

test_numpy_array.csv

@@ -1,3 +1,3 @@
 id,embedding
-1,[0.1 0.2 0.3]
-2,[0.4 0.5 0.6]
+1,"[0.1, 0.2, 0.3]"
+2,"[0.4, 0.5, 0.6]"