Skip to content

Commit 90e1597

Browse files
committed
Update all tpch examples to use select instead of select_columns
1 parent 94eb233 commit 90e1597

22 files changed

+82
-94
lines changed

examples/tpch/convert_data_to_parquet.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,6 @@
138138

139139
df = ctx.read_csv(source_file, schema=schema, has_header=False, delimiter="|")
140140

141-
df = df.select_columns(*output_cols)
141+
df = df.select(*output_cols)
142142

143143
df.write_parquet(dest_file, compression="snappy")

examples/tpch/q02_minimum_cost_supplier.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,10 @@
4343

4444
ctx = SessionContext()
4545

46-
df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns(
46+
df_part = ctx.read_parquet(get_data_path("part.parquet")).select(
4747
"p_partkey", "p_mfgr", "p_type", "p_size"
4848
)
49-
df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns(
49+
df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select(
5050
"s_acctbal",
5151
"s_name",
5252
"s_address",
@@ -55,13 +55,13 @@
5555
"s_nationkey",
5656
"s_suppkey",
5757
)
58-
df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select_columns(
58+
df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select(
5959
"ps_partkey", "ps_suppkey", "ps_supplycost"
6060
)
61-
df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns(
61+
df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select(
6262
"n_nationkey", "n_regionkey", "n_name"
6363
)
64-
df_region = ctx.read_parquet(get_data_path("region.parquet")).select_columns(
64+
df_region = ctx.read_parquet(get_data_path("region.parquet")).select(
6565
"r_regionkey", "r_name"
6666
)
6767

@@ -115,7 +115,7 @@
115115

116116
# From the problem statement, these are the values we wish to output
117117

118-
df = df.select_columns(
118+
df = df.select(
119119
"s_acctbal",
120120
"s_name",
121121
"n_name",

examples/tpch/q03_shipping_priority.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,13 @@
3737

3838
ctx = SessionContext()
3939

40-
df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns(
40+
df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select(
4141
"c_mktsegment", "c_custkey"
4242
)
43-
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns(
43+
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select(
4444
"o_orderdate", "o_shippriority", "o_custkey", "o_orderkey"
4545
)
46-
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns(
46+
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select(
4747
"l_orderkey", "l_extendedprice", "l_discount", "l_shipdate"
4848
)
4949

@@ -80,7 +80,7 @@
8080

8181
# Change the order that the columns are reported in just to match the spec
8282

83-
df = df.select_columns("l_orderkey", "revenue", "o_orderdate", "o_shippriority")
83+
df = df.select("l_orderkey", "revenue", "o_orderdate", "o_shippriority")
8484

8585
# Show result
8686

examples/tpch/q04_order_priority_checking.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,10 @@
3939

4040
ctx = SessionContext()
4141

42-
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns(
42+
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select(
4343
"o_orderdate", "o_orderpriority", "o_orderkey"
4444
)
45-
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns(
45+
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select(
4646
"l_orderkey", "l_commitdate", "l_receiptdate"
4747
)
4848

@@ -54,7 +54,7 @@
5454
# Limit results to cases where commitment date before receipt date
5555
# Aggregate the results so we only get one row to join with the order table.
5656
# Alternately, and likely more idiomatic is instead of `.aggregate` you could
57-
# do `.select_columns("l_orderkey").distinct()`. The goal here is to show
57+
# do `.select("l_orderkey").distinct()`. The goal here is to show
5858
# multiple examples of how to use Data Fusion.
5959
df_lineitem = df_lineitem.filter(col("l_commitdate") < col("l_receiptdate")).aggregate(
6060
[col("l_orderkey")], []

examples/tpch/q05_local_supplier_volume.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,22 +47,22 @@
4747

4848
ctx = SessionContext()
4949

50-
df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns(
50+
df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select(
5151
"c_custkey", "c_nationkey"
5252
)
53-
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns(
53+
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select(
5454
"o_custkey", "o_orderkey", "o_orderdate"
5555
)
56-
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns(
56+
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select(
5757
"l_orderkey", "l_suppkey", "l_extendedprice", "l_discount"
5858
)
59-
df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns(
59+
df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select(
6060
"s_suppkey", "s_nationkey"
6161
)
62-
df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns(
62+
df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select(
6363
"n_nationkey", "n_regionkey", "n_name"
6464
)
65-
df_region = ctx.read_parquet(get_data_path("region.parquet")).select_columns(
65+
df_region = ctx.read_parquet(get_data_path("region.parquet")).select(
6666
"r_regionkey", "r_name"
6767
)
6868

examples/tpch/q06_forecasting_revenue_change.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151

5252
ctx = SessionContext()
5353

54-
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns(
54+
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select(
5555
"l_shipdate", "l_quantity", "l_extendedprice", "l_discount"
5656
)
5757

examples/tpch/q07_volume_shipping.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,19 +49,19 @@
4949

5050
ctx = SessionContext()
5151

52-
df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns(
52+
df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select(
5353
"s_suppkey", "s_nationkey"
5454
)
55-
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns(
55+
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select(
5656
"l_shipdate", "l_extendedprice", "l_discount", "l_suppkey", "l_orderkey"
5757
)
58-
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns(
58+
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select(
5959
"o_orderkey", "o_custkey"
6060
)
61-
df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns(
61+
df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select(
6262
"c_custkey", "c_nationkey"
6363
)
64-
df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns(
64+
df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select(
6565
"n_nationkey", "n_name"
6666
)
6767

examples/tpch/q08_market_share.py

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -47,25 +47,23 @@
4747

4848
ctx = SessionContext()
4949

50-
df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns(
51-
"p_partkey", "p_type"
52-
)
53-
df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns(
50+
df_part = ctx.read_parquet(get_data_path("part.parquet")).select("p_partkey", "p_type")
51+
df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select(
5452
"s_suppkey", "s_nationkey"
5553
)
56-
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns(
54+
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select(
5755
"l_partkey", "l_extendedprice", "l_discount", "l_suppkey", "l_orderkey"
5856
)
59-
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns(
57+
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select(
6058
"o_orderkey", "o_custkey", "o_orderdate"
6159
)
62-
df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns(
60+
df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select(
6361
"c_custkey", "c_nationkey"
6462
)
65-
df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns(
63+
df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select(
6664
"n_nationkey", "n_name", "n_regionkey"
6765
)
68-
df_region = ctx.read_parquet(get_data_path("region.parquet")).select_columns(
66+
df_region = ctx.read_parquet(get_data_path("region.parquet")).select(
6967
"r_regionkey", "r_name"
7068
)
7169

@@ -133,7 +131,7 @@
133131

134132
# When we join to the customer dataframe, we don't want to confuse other columns, so only
135133
# select the supplier key that we need
136-
df_national_suppliers = df_national_suppliers.select_columns("s_suppkey")
134+
df_national_suppliers = df_national_suppliers.select("s_suppkey")
137135

138136

139137
# Part 3: Combine suppliers and customers and compute the market share

examples/tpch/q09_product_type_profit_measure.py

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,27 +39,25 @@
3939

4040
ctx = SessionContext()
4141

42-
df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns(
43-
"p_partkey", "p_name"
44-
)
45-
df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns(
42+
df_part = ctx.read_parquet(get_data_path("part.parquet")).select("p_partkey", "p_name")
43+
df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select(
4644
"s_suppkey", "s_nationkey"
4745
)
48-
df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select_columns(
46+
df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select(
4947
"ps_suppkey", "ps_partkey", "ps_supplycost"
5048
)
51-
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns(
49+
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select(
5250
"l_partkey",
5351
"l_extendedprice",
5452
"l_discount",
5553
"l_suppkey",
5654
"l_orderkey",
5755
"l_quantity",
5856
)
59-
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns(
57+
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select(
6058
"o_orderkey", "o_custkey", "o_orderdate"
6159
)
62-
df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns(
60+
df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select(
6361
"n_nationkey", "n_name", "n_regionkey"
6462
)
6563

examples/tpch/q10_returned_item_reporting.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444

4545
ctx = SessionContext()
4646

47-
df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns(
47+
df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select(
4848
"c_custkey",
4949
"c_nationkey",
5050
"c_name",
@@ -53,13 +53,13 @@
5353
"c_phone",
5454
"c_comment",
5555
)
56-
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns(
56+
df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select(
5757
"l_extendedprice", "l_discount", "l_orderkey", "l_returnflag"
5858
)
59-
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns(
59+
df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select(
6060
"o_orderkey", "o_custkey", "o_orderdate"
6161
)
62-
df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns(
62+
df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select(
6363
"n_nationkey", "n_name", "n_regionkey"
6464
)
6565

@@ -87,7 +87,7 @@
8787
df = df.join(df_nation, (["c_nationkey"], ["n_nationkey"]), how="inner")
8888

8989
# These are the columns the problem statement requires
90-
df = df.select_columns(
90+
df = df.select(
9191
"c_custkey",
9292
"c_name",
9393
"revenue",

0 commit comments

Comments
 (0)