PERF: fix performance regression in memory_usage(deep=True) for object dtype

asv_bench/benchmarks/frame_methods.py

@@ -619,4 +619,17 @@ def time_select_dtypes(self, n):
         self.df.select_dtypes(include="int")
 
 
+class MemoryUsage:
+    def setup(self):
+        self.df = DataFrame(np.random.randn(100000, 2), columns=list("AB"))
+        self.df2 = self.df.copy()
+        self.df2["A"] = self.df2["A"].astype("object")
+
+    def time_memory_usage(self):
+        self.df.memory_usage(deep=True)
+
+    def time_memory_usage_object_dtype(self):
+        self.df2.memory_usage(deep=True)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

pandas/core/base.py

@@ -1387,7 +1387,7 @@ def memory_usage(self, deep=False):
 
         v = self.array.nbytes
         if deep and is_object_dtype(self) and not PYPY:
-            v += lib.memory_usage_of_objects(self.array)
+            v += lib.memory_usage_of_objects(self._values)
         return v
 
     @doc(