Update materials post SG update (pre-DR)

Author: stephengruppetta

polars-vs-pandas/DataFrame_Plots.ipynb

@@ -1,17 +1,17 @@
 import numpy as np
 
 
-def data_generation(number_of_rows):
+def generate_data(number_of_rows):
+    rng = np.random.default_rng()
+
     return {
         "order_id": range(1, number_of_rows + 1),
-        "region": np.random.choice(
-            ["North", "South", "East", "West"], size=number_of_rows
-        ),
-        "sales_person": np.random.choice(
+        "region": rng.choice(["North", "South", "East", "West"], size=number_of_rows),
+        "sales_person": rng.choice(
             ["Armstrong", "Aldrin", "Collins"], size=number_of_rows
         ),
-        "product": np.random.choice(
+        "product": rng.choice(
             ["Helmet", "Oxygen", "Boots", "Gloves"], size=number_of_rows
         ),
-        "sales_income": np.random.randint(1, 5001, size=number_of_rows),
+        "sales_income": rng.integers(1, 5001, size=number_of_rows),
     }

polars-vs-pandas/code_speed_test.py

@@ -1,82 +1,88 @@
+import functools
 import sys
-import time
+from timeit import Timer
 
 import pandas as pd
 import polars as pl
-from data_generation import data_generation
 
-# Data Generation
+from data_generation import generate_data
 
-test_data = data_generation(int(sys.argv[1]))
 
-# Polars DataFrame Test
+def create_pandas_dataframe(test_data):
+    return pd.DataFrame(test_data).convert_dtypes(dtype_backend="pyarrow")
 
-overall_time_start = time.time()
 
-polars_dataframe = pl.DataFrame(test_data)
+def create_polars_dataframe(test_data):
+    return pl.DataFrame(test_data)
 
-processing_time_start = time.time()
 
-(
-    polars_dataframe.group_by(["region", "product", "sales_person"]).agg(
-        total_sales=pl.col("sales_income").sum()
-    )
-)
-
-end_time = time.time()
-
-del polars_dataframe
-
-print(
-    f"Polars DataFrame creation: {processing_time_start - overall_time_start}"
-)
-print(f"Polars DataFrame query runtime: {end_time - processing_time_start}")
-print(f"Polars DataFrame overall time: {end_time - overall_time_start}")
-print()
-
-# Polars LazyFrame Test
+def create_polars_lazyframe(test_data):
+    return pl.LazyFrame(test_data)
 
-overall_time_start = time.time()
 
-polars_lazyframe = pl.LazyFrame(test_data)
+def analyze_pandas_dataframe(pandas_df):
+    return pandas_df.groupby(["region", "product", "sales_person"])[
+        "sales_income"
+    ].sum()
 
-processing_time_start = time.time()
 
-(
-    polars_lazyframe.group_by(["region", "product", "sales_person"]).agg(
+def analyze_polars_dataframe(polars_df):
+    return polars_df.group_by(["region", "product", "sales_person"]).agg(
         total_sales=pl.col("sales_income").sum()
     )
-).collect()
 
-end_time = time.time()
 
-del polars_lazyframe
+def analyze_polars_lazyframe(polars_lf):
+    return (
+        polars_lf.group_by(["region", "product", "sales_person"])
+        .agg(total_sales=pl.col("sales_income").sum())
+        .collect()
+    )
 
-print(
-    f"Polars LazyFrame creation: {processing_time_start - overall_time_start}"
-)
-print(f"Polars LazyFrame query runtime: {end_time - processing_time_start}")
-print(f"Polars LazyFrame overall time: {end_time - overall_time_start}")
-print()
 
-# Pandas DataFrame Test
+test_data = generate_data(int(sys.argv[1]))
+
+print("Creating Dataframes...")
+print(f"Pandas dataframe creation time for {int(sys.argv[1])} rows:")
+print(Timer(functools.partial(create_pandas_dataframe, test_data)).timeit(100))
+print(f"\nPolars dataframe creation time for {int(sys.argv[1])} rows:")
+print(Timer(functools.partial(create_polars_dataframe, test_data)).timeit(100))
+print(f"\nPolars lazyframe creation time for {int(sys.argv[1])} rows:")
+print(Timer(functools.partial(create_polars_lazyframe, test_data)).timeit(100))
 
-overall_time_start = time.time()
+pandas_df = create_pandas_dataframe(test_data)
+polars_df = create_polars_dataframe(test_data)
+polars_lf = create_polars_lazyframe(test_data)
 
-pandas_dataframe = pd.DataFrame(test_data)
+print("\nAnalyzing Dataframes...")
+print(f"Pandas dataframe analysis time for {int(sys.argv[1])} rows:")
+print(Timer(functools.partial(analyze_pandas_dataframe, pandas_df)).timeit(100))
 
-processing_time_start = time.time()
+print(f"\nPolars dataframe analysis time for {int(sys.argv[1])} rows:")
+print(Timer(functools.partial(analyze_polars_dataframe, polars_df)).timeit(100))
 
-pandas_dataframe.groupby(["region", "product", "sales_person"])[
-    "sales_income"
-].sum()
+print(f"\nPolars lazyframe analysis time for {int(sys.argv[1])} rows:")
+print(Timer(functools.partial(analyze_polars_lazyframe, polars_lf)).timeit(100))
 
-end_time = time.time()
+print("\nShow Boots sales in the East region for pandas DataFrame")
+print(analyze_pandas_dataframe(pandas_df)["East"]["Boots"])
 
-del pandas_dataframe
+print("\nShow Boots sales in the East region for polars DataFrame")
+print(
+    (
+        analyze_polars_dataframe(polars_df).filter(
+            pl.col("region") == "East",
+            pl.col("product") == "Boots",
+        )
+    )
+)
 
+print("\nShow Boots sales in the East region for pandas LazyFrame")
 print(
-    f"Pandas DataFrame creation: {processing_time_start - overall_time_start}"
+    (
+        analyze_polars_lazyframe(polars_lf).filter(
+            pl.col("region") == "East",
+            pl.col("product") == "Boots",
+        )
+    )
 )
-print(f"Pandas DataFrame query runtime: {end_time - processing_time_start}")
-print(f"Pandas DataFrame overall time: {end_time - overall_time_start}")

polars-vs-pandas/data_generation.py

@@ -1,22 +1,30 @@
 import pandas as pd
 import polars as pl
 
-orders_pandas = pd.read_parquet("Online_Retail.parquet")
+# Pandas index-based syntax
+orders_pandas = pd.read_parquet("online_retail.parquet")
 
 orders_pandas["Total"] = orders_pandas["Quantity"] * orders_pandas["UnitPrice"]
 
 orders_pandas[["InvoiceNo", "Quantity", "UnitPrice", "Total"]][
-    orders_pandas["Total"] > 10
+    orders_pandas["Total"] > 100
 ].head(3)
 
+# Pandas method chaining syntax
+orders_pandas = pd.read_parquet("online_retail.parquet")
 
-orders_polars = pl.read_csv("online_retail.csv")
+(
+    orders_pandas.assign(Total=orders_pandas["Quantity"] * orders_pandas["UnitPrice"])
+    .filter(["InvoiceNo", "Quantity", "UnitPrice", "Total"])
+    .query("Total > 100")
+).head(3)
 
+# Polars method chaining syntax
 orders_polars = pl.read_parquet("online_retail.parquet")
 
 (
     orders_polars.select(
         pl.col(["InvoiceNo", "Quantity", "UnitPrice"]),
         total=pl.col("Quantity") * pl.col("UnitPrice"),
-    ).filter(pl.col("total") > 10)
+    ).filter(pl.col("total") > 100)
 ).head(3)

polars-vs-pandas/dataframe_and_lazyframe_time_tests.py

@@ -0,0 +1,34 @@
+import functools
+import sys
+from timeit import Timer
+
+import polars as pl
+
+from data_generation import generate_data
+
+
+def create_polars_lazyframe(test_data):
+    return pl.LazyFrame(test_data)
+
+
+def analyze_polars_lazyframe(polars_lf):
+    polars_lf.group_by(["region", "product", "sales_person"]).agg(
+        total_sales=pl.col("sales_income").sum()
+    ).collect()
+
+
+def analyze_polars_streaming(polars_lf):
+    polars_lf.group_by(["region", "product", "sales_person"]).agg(
+        total_sales=pl.col("sales_income").sum()
+    ).collect(engine="streaming")
+
+
+test_data = generate_data(int(sys.argv[1]))
+
+polars_lf = create_polars_lazyframe(test_data)
+
+print(f"Polars lazyframe analysis time for {int(sys.argv[1])} rows:")
+print(Timer(functools.partial(analyze_polars_lazyframe, polars_lf)).timeit(100))
+
+print(f"\nPolars streaming analysis time for {int(sys.argv[1])} rows:")
+print(Timer(functools.partial(analyze_polars_streaming, polars_lf)).timeit(100))