Skip to content

Commit 55e2b32

Browse files
OlegWockOlegWock
committed
Merge Deepnote changes into upstream VF 2.0.3
2 parents 4103005 + 9795a91 commit 55e2b32

File tree

89 files changed

+6261
-1316
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+6261
-1316
lines changed

‎.github/workflows/build_test.yml‎

Lines changed: 10 additions & 352 deletions
Large diffs are not rendered by default.

‎Cargo.lock‎

Lines changed: 473 additions & 292 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 17 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ version = "1.36.0"
6868

6969
[workspace.dependencies.pyo3]
7070
version = "0.25.0"
71+
default-features = true
72+
features = ["serde"]
7173

7274
[workspace.dependencies.pythonize]
7375
version = "0.25.0"
@@ -99,46 +101,50 @@ version = "1.0.140"
99101
default-features = false
100102

101103
[workspace.dependencies.datafusion]
102-
version = "48.0.0"
104+
version = "49.0.0"
103105
default-features = false
104106
features = [ "nested_expressions",]
105107

106108
[workspace.dependencies.datafusion-common]
107-
version = "48.0.0"
109+
version = "49.0.0"
108110
default-features = false
109111

110112
[workspace.dependencies.datafusion-expr]
111-
version = "48.0.0"
113+
version = "49.0.0"
112114

113115
[workspace.dependencies.datafusion-proto]
114-
version = "48.0.0"
116+
version = "49.0.0"
115117
default-features = false
116118
features = [ "parquet",]
117119

118120
[workspace.dependencies.datafusion-proto-common]
119-
version = "48.0.0"
121+
version = "49.0.0"
120122
default-features = false
121123

122124
[workspace.dependencies.datafusion-physical-expr]
123-
version = "48.0.0"
125+
version = "49.0.0"
124126
default-features = false
125127

126128
[workspace.dependencies.datafusion-optimizer]
127-
version = "48.0.0"
129+
version = "49.0.0"
128130
default-features = false
129131

130132
[workspace.dependencies.datafusion-functions]
131-
version = "48.0.0"
133+
version = "49.0.0"
132134
default-features = false
133135

134136
[workspace.dependencies.datafusion-functions-nested]
135-
version = "48.0.0"
137+
version = "49.0.0"
136138

137139
[workspace.dependencies.datafusion-functions-aggregate]
138-
version = "48.0.0"
140+
version = "49.0.0"
139141

140142
[workspace.dependencies.datafusion-functions-window]
141-
version = "48.0.0"
143+
version = "49.0.0"
144+
145+
[workspace.dependencies.datafusion-sql]
146+
version = "49.0.0"
147+
default-features = false
142148

143149
[workspace.dependencies.thiserror]
144150
version = "1.0.69"
@@ -179,51 +185,3 @@ version = "0.2.19"
179185

180186
[workspace.dependencies.getrandom]
181187
version = "0.3"
182-
183-
[patch.crates-io.datafusion]
184-
git = "https://github.com/apache/datafusion"
185-
rev = "87218a1e656f05da4c8374010b3ae550842606b8"
186-
187-
[patch.crates-io.datafusion-common]
188-
git = "https://github.com/apache/datafusion"
189-
rev = "87218a1e656f05da4c8374010b3ae550842606b8"
190-
191-
[patch.crates-io.datafusion-expr]
192-
git = "https://github.com/apache/datafusion"
193-
rev = "87218a1e656f05da4c8374010b3ae550842606b8"
194-
195-
[patch.crates-io.datafusion-proto]
196-
git = "https://github.com/apache/datafusion"
197-
rev = "87218a1e656f05da4c8374010b3ae550842606b8"
198-
199-
[patch.crates-io.datafusion-proto-common]
200-
git = "https://github.com/apache/datafusion"
201-
rev = "87218a1e656f05da4c8374010b3ae550842606b8"
202-
203-
[patch.crates-io.datafusion-physical-expr]
204-
git = "https://github.com/apache/datafusion"
205-
rev = "87218a1e656f05da4c8374010b3ae550842606b8"
206-
207-
[patch.crates-io.datafusion-optimizer]
208-
git = "https://github.com/apache/datafusion"
209-
rev = "87218a1e656f05da4c8374010b3ae550842606b8"
210-
211-
[patch.crates-io.datafusion-functions]
212-
git = "https://github.com/apache/datafusion"
213-
rev = "87218a1e656f05da4c8374010b3ae550842606b8"
214-
215-
[patch.crates-io.datafusion-functions-nested]
216-
git = "https://github.com/apache/datafusion"
217-
rev = "87218a1e656f05da4c8374010b3ae550842606b8"
218-
219-
[patch.crates-io.datafusion-functions-aggregate]
220-
git = "https://github.com/apache/datafusion"
221-
rev = "87218a1e656f05da4c8374010b3ae550842606b8"
222-
223-
[patch.crates-io.datafusion-functions-window]
224-
git = "https://github.com/apache/datafusion"
225-
rev = "87218a1e656f05da4c8374010b3ae550842606b8"
226-
227-
[patch.crates-io.datafusion-sql]
228-
git = "https://github.com/apache/datafusion"
229-
rev = "87218a1e656f05da4c8374010b3ae550842606b8"

‎examples/python-examples/custom_executor.py‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
import vegafusion as vf
2+
import pyarrow as pa
3+
4+
5+
def simple_logging_executor(logical_plan_json: str) -> pa.Table:
6+
"""A minimal executor that logs and throws unimplemented error."""
7+
# TODO: Update signature once we pass logical plan as proper structure instead of string
8+
print("📋 Custom executor received logical plan:")
9+
print(f" Plan length: {len(logical_plan_json)} characters")
10+
print(f" Plan preview: {logical_plan_json[:100]}...")
11+
12+
raise NotImplementedError("Custom executor is not implemented yet!")
13+
14+
15+
# This example shows the simplest possible custom executor that logs
16+
# received plans and throws an unimplemented error.
17+
def main():
18+
"""Demonstrate the minimal custom executor."""
19+
20+
spec = get_spec()
21+
22+
schema = pa.schema(
23+
[
24+
pa.field("Title", pa.string()),
25+
pa.field("US Gross", pa.int64()),
26+
pa.field("Worldwide Gross", pa.int64()),
27+
pa.field("US DVD Sales", pa.int64()),
28+
pa.field("Production Budget", pa.int64()),
29+
pa.field("Release Date", pa.date32()),
30+
pa.field("MPAA Rating", pa.string()),
31+
pa.field("Running Time min", pa.int32()),
32+
pa.field("Distributor", pa.string()),
33+
pa.field("Source", pa.string()),
34+
pa.field("Major Genre", pa.string()),
35+
pa.field("Creative Type", pa.string()),
36+
pa.field("Director", pa.string()),
37+
pa.field("Rotten Tomatoes Rating", pa.int8()),
38+
pa.field("IMDB Rating", pa.float32()),
39+
pa.field("IMDB Votes", pa.int64()),
40+
]
41+
)
42+
43+
print("Testing custom executor with VegaFusion...")
44+
45+
# Create a DataFusion runtime that uses a custom Python executor
46+
runtime = vf.VegaFusionRuntime(
47+
executor=simple_logging_executor,
48+
)
49+
50+
try:
51+
runtime.pre_transform_spec(
52+
spec=spec,
53+
local_tz="UTC",
54+
inline_datasets={"movies": schema},
55+
)
56+
except ValueError as e:
57+
print(f"✅ Expected error caught: {e}")
58+
59+
print("Done!")
60+
61+
62+
def get_spec():
63+
"""
64+
Based on https://vega.github.io/editor/#/examples/vega/histogram-null-values
65+
"""
66+
return {
67+
"$schema": "https://vega.github.io/schema/vega/v5.json",
68+
"description": "A histogram of film ratings, modified to include null values.",
69+
"width": 400,
70+
"height": 200,
71+
"padding": 5,
72+
"autosize": {"type": "fit", "resize": True},
73+
"signals": [
74+
{
75+
"name": "maxbins",
76+
"value": 10,
77+
"bind": {"input": "select", "options": [5, 10, 20]},
78+
},
79+
{"name": "binCount", "update": "(bins.stop - bins.start) / bins.step"},
80+
{"name": "nullGap", "value": 10},
81+
{"name": "barStep", "update": "(width - nullGap) / (1 + binCount)"},
82+
],
83+
"data": [
84+
{
85+
"name": "table",
86+
"url": "vegafusion+dataset://movies",
87+
"transform": [
88+
{"type": "extent", "field": "IMDB Rating", "signal": "extent"},
89+
{
90+
"type": "bin",
91+
"signal": "bins",
92+
"field": "IMDB Rating",
93+
"extent": {"signal": "extent"},
94+
"maxbins": {"signal": "maxbins"},
95+
},
96+
],
97+
},
98+
{
99+
"name": "counts",
100+
"source": "table",
101+
"transform": [
102+
{"type": "filter", "expr": "datum['IMDB Rating'] != null"},
103+
{"type": "aggregate", "groupby": ["bin0", "bin1"]},
104+
],
105+
},
106+
{
107+
"name": "nulls",
108+
"source": "table",
109+
"transform": [
110+
{"type": "filter", "expr": "datum['IMDB Rating'] == null"},
111+
{"type": "aggregate", "groupby": []},
112+
],
113+
},
114+
],
115+
"scales": [
116+
{
117+
"name": "yscale",
118+
"type": "linear",
119+
"range": "height",
120+
"round": True,
121+
"nice": True,
122+
"domain": {
123+
"fields": [
124+
{"data": "counts", "field": "count"},
125+
{"data": "nulls", "field": "count"},
126+
]
127+
},
128+
},
129+
{
130+
"name": "xscale",
131+
"type": "linear",
132+
"range": [{"signal": "barStep + nullGap"}, {"signal": "width"}],
133+
"round": True,
134+
"domain": {"signal": "[bins.start, bins.stop]"},
135+
"bins": {"signal": "bins"},
136+
},
137+
{
138+
"name": "xscale-null",
139+
"type": "band",
140+
"range": [0, {"signal": "barStep"}],
141+
"round": True,
142+
"domain": [None],
143+
},
144+
],
145+
"axes": [
146+
{"orient": "bottom", "scale": "xscale", "tickMinStep": 0.5},
147+
{"orient": "bottom", "scale": "xscale-null"},
148+
{"orient": "left", "scale": "yscale", "tickCount": 5, "offset": 5},
149+
],
150+
"marks": [
151+
{
152+
"type": "rect",
153+
"from": {"data": "counts"},
154+
"encode": {
155+
"update": {
156+
"x": {"scale": "xscale", "field": "bin0", "offset": 1},
157+
"x2": {"scale": "xscale", "field": "bin1"},
158+
"y": {"scale": "yscale", "field": "count"},
159+
"y2": {"scale": "yscale", "value": 0},
160+
"fill": {"value": "steelblue"},
161+
},
162+
"hover": {"fill": {"value": "firebrick"}},
163+
},
164+
},
165+
{
166+
"type": "rect",
167+
"from": {"data": "nulls"},
168+
"encode": {
169+
"update": {
170+
"x": {"scale": "xscale-null", "value": None, "offset": 1},
171+
"x2": {"scale": "xscale-null", "band": 1},
172+
"y": {"scale": "yscale", "field": "count"},
173+
"y2": {"scale": "yscale", "value": 0},
174+
"fill": {"value": "#aaa"},
175+
},
176+
"hover": {"fill": {"value": "firebrick"}},
177+
},
178+
},
179+
],
180+
}
181+
182+
183+
if __name__ == "__main__":
184+
main()

0 commit comments

Comments
 (0)