vgplot (#1015)

* document Mosaic

* taxi data loader

Note: it's not active, only here for reference. To make this data loader work in CI, you have to install the proper duckdb binary on the $PATH, and I'd recommend to use a dedicated $TMPDIR rather than write in the root folder.

* Apply suggestions from code review

Co-authored-by: Mike Bostock <[email protected]>

* vgplot

* s/boilerplate//

* s/upvote//

* vg live update

* doc polish

* fix transitive preload

* re-resolve npm imports

* test getDependencyResolver

* test import resolutions; comment

* optimize resolution; apache-arrow 15

* duckdb needs latest apache-arrow, too

---------

Co-authored-by: Philippe Rivière <[email protected]>

Author: mbostock

docs/lib/mosaic.md

@@ -0,0 +1,81 @@
+---
+sql:
+  trips: nyc-taxi.parquet
+---
+
+# Mosaic vgplot
+
+[Mosaic](https://uwdata.github.io/mosaic/) is a system for linking data visualizations, tables, and inputs, leveraging [DuckDB](./duckdb) for scalable processing. Mosaic includes an interactive grammar of graphics, [Mosaic vgplot](https://uwdata.github.io/mosaic/vgplot/), built on [Observable Plot](./plot). With vgplot, you can interactively visualize and explore millions — even billions — of data points.
+
+The example below shows the pickup and dropoff locations of one million taxi rides in New York City from Jan 1–3, 2010. The dataset is stored in a 8MB [Apache Parquet](./arrow#apache-parquet) file, generated with a [data loader](../loaders).
+
+${maps}
+
+${histogram}
+
+The views above are coordinated: brushing a time window in the histogram, or a region in either map, will filter both maps. _What spatial patterns can you find?_
+
+The code below creates three views, coordinated by Mosaic’s [crossfilter](https://uwdata.github.io/mosaic/api/core/selection.html#selection-crossfilter) helper.
+
+```js echo
+// Create a shared filter
+const $filter = vg.Selection.crossfilter();
+
+// Shared attributes for the maps
+const attributes = [
+  vg.width(315),
+  vg.height(550),
+  vg.margin(0),
+  vg.xAxis(null),
+  vg.yAxis(null),
+  vg.xDomain([297000, 297000 + 28.36 * 315]),
+  vg.yDomain([57900, 57900 + 28.36 * 550]), // ensure aspect ratio of 1
+  vg.colorScale("symlog")
+];
+
+// Create two side-by-side maps
+const maps = vg.hconcat(
+  vg.plot(
+    vg.raster(vg.from("trips", {filterBy: $filter}), {x: "px", y: "py", imageRendering: "pixelated"}),
+    vg.intervalXY({as: $filter}),
+    vg.text([{label: "Taxi pickups"}], {
+      dx: 10,
+      dy: 10,
+      text: "label",
+      fill: "white",
+      frameAnchor: "top-left"
+    }),
+    ...attributes,
+    vg.colorScheme("turbo"),
+    vg.frame({stroke: "black"})
+  ),
+  vg.hspace(10),
+  vg.plot(
+    vg.raster(vg.from("trips", {filterBy: $filter}), {x: "dx", y: "dy", imageRendering: "pixelated"}),
+    vg.intervalXY({as: $filter}),
+    vg.text([{label: "Taxi dropoffs"}], {
+      dx: 10,
+      dy: 10,
+      text: "label",
+      fill: "white",
+      frameAnchor: "top-left"
+    }),
+    ...attributes,
+    vg.colorScheme("turbo"),
+    vg.frame({stroke: "black"})
+  )
+);
+
+// Create the histogram
+const histogram = vg.plot(
+  vg.rectY(vg.from("trips"), {x: vg.bin("time"), y: vg.count(), insetLeft: 0.5, insetRight: 0.5}),
+  vg.intervalX({as: $filter}),
+  vg.yTickFormat("s"),
+  vg.xLabel("Hour of pickup"),
+  vg.yLabel("Number of rides"),
+  vg.width(640),
+  vg.height(100)
+);
+```
+
+For more Mosaic examples, see the [Mosaic + Framework](https://uwdata.github.io/mosaic-framework-example/) website.

docs/lib/nyc-taxi.parquet

@@ -0,0 +1,25 @@
+duckdb :memory: << EOF
+-- Load spatial extension
+INSTALL spatial; LOAD spatial;
+
+-- Project coordinates, following the example at https://github.com/duckdb/duckdb_spatial
+CREATE TEMP TABLE rides AS SELECT
+  pickup_datetime::TIMESTAMP AS datetime,
+  ST_Transform(ST_Point(pickup_latitude, pickup_longitude), 'EPSG:4326', 'EPSG:32118') AS pick,
+  ST_Transform(ST_Point(dropoff_latitude, dropoff_longitude), 'EPSG:4326', 'EPSG:32118') AS drop
+FROM 'https://uwdata.github.io/mosaic-datasets/data/nyc-rides-2010.parquet';
+
+-- Write output parquet file
+COPY (SELECT
+  HOUR(datetime) + MINUTE(datetime) / 60 AS time,
+  ST_X(pick)::INTEGER AS px, -- extract pickup x-coord
+  ST_Y(pick)::INTEGER AS py, -- extract pickup y-coord
+  ST_X(drop)::INTEGER AS dx, -- extract dropff x-coord
+  ST_Y(drop)::INTEGER AS dy  -- extract dropff y-coord
+FROM rides
+ORDER BY 2,3,4,5,1 -- optimize output size by sorting
+) TO 'trips.parquet' (COMPRESSION 'ZSTD', row_group_size 10000000);
+EOF
+
+cat trips.parquet >&1  # Copy payload to stdout
+rm trips.parquet       # Clean up

docs/lib/nyc-taxi.parquet.sh

@@ -70,6 +70,7 @@ export default {
         {name: "Mapbox GL JS", path: "/lib/mapbox-gl"},
         {name: "Mermaid", path: "/lib/mermaid"},
         {name: "Microsoft Excel (XLSX)", path: "/lib/xlsx"},
+        {name: "Mosaic vgplot", path: "/lib/mosaic"},
         {name: "Observable Generators", path: "/lib/generators"},
         {name: "Observable Inputs", path: "/lib/inputs"},
         {name: "Observable Plot", path: "/lib/plot"},

observablehq.config.ts

@@ -15,7 +15,7 @@ const library = {
   ...sampleDatasets
 };
 
-const runtime = new Runtime(library);
+export const runtime = new Runtime(library);
 export const main = runtime.module();
 
 const cellsById = new Map();

src/client/main.js

@@ -1,6 +1,6 @@
 import {registerTable} from "npm:@observablehq/duckdb";
 import {FileAttachment, registerFile} from "npm:@observablehq/stdlib";
-import {main, undefine} from "./main.js";
+import {main, runtime, undefine} from "./main.js";
 import {enableCopyButtons} from "./pre.js";
 
 export * from "./index.js";
@@ -92,7 +92,10 @@ export function open({hash, eval: compile} = {}) {
         }
         if (message.tables.removed.length || message.tables.added.length) {
           const sql = main._resolve("sql");
-          sql.define(sql._promise); // re-evaluate sql code
+          runtime._updates.add(sql); // re-evaluate sql code
+          runtime._compute();
+          const vg = runtime._builtin._resolve("vg");
+          vg.define("vg", [], vg._definition); // reload vgplot, then re-evaluate vg code
         }
         if (message.stylesheets.added.length === 1 && message.stylesheets.removed.length === 1) {
           const [newHref] = message.stylesheets.added;

src/client/preview.js

@@ -59,8 +59,12 @@ export function registerTable(name, source) {
 }
 
 export async function sql(strings, ...args) {
+  return (await getDefaultClient()).query(strings.join("?"), args);
+}
+
+export async function getDefaultClient() {
   await Promise.all(inserts);
-  return (await (db ??= DuckDBClient.of())).query(strings.join("?"), args);
+  return await (db ??= DuckDBClient.of());
 }
 
 export class DuckDBClient {

src/client/stdlib/duckdb.js

@@ -19,4 +19,5 @@ export const SQLite = () => import("observablehq:stdlib/sqlite").then((sqlite) =
 export const SQLiteDatabaseClient = () => import("observablehq:stdlib/sqlite").then((sqlite) => sqlite.SQLiteDatabaseClient); // prettier-ignore
 export const tex = () => import("observablehq:stdlib/tex").then((tex) => tex.default);
 export const topojson = () => import("npm:topojson-client");
+export const vg = () => import("observablehq:stdlib/vgplot").then((vg) => vg.default());
 export const vl = () => import("observablehq:stdlib/vega-lite").then((vl) => vl.default);

src/client/stdlib/recommendedLibraries.js

@@ -0,0 +1,10 @@
+import * as vgplot from "npm:@uwdata/vgplot";
+import {getDefaultClient} from "observablehq:stdlib/duckdb";
+
+export default async function vg() {
+  const coordinator = new vgplot.Coordinator();
+  const api = vgplot.createAPIContext({coordinator});
+  const duckdb = (await getDefaultClient())._db;
+  coordinator.databaseConnector(vgplot.wasmConnector({duckdb}));
+  return api;
+}

src/client/stdlib/vgplot.js

@@ -31,6 +31,7 @@ export function getImplicitInputImports(inputs: Iterable<string>): Set<string> {
   if (set.has("tex")) implicits.add("npm:@observablehq/tex");
   if (set.has("topojson")) implicits.add("npm:topojson-client");
   if (set.has("vl")) implicits.add("observablehq:stdlib/vega-lite");
+  if (set.has("vg")) implicits.add("observablehq:stdlib/vgplot");
   return implicits;
 }
 
@@ -159,6 +160,7 @@ export function getImplicitDependencies(imports: Iterable<string>): Set<string>
   if (set.has("npm:@observablehq/xlsx")) implicits.add("npm:exceljs");
   if (set.has("npm:@observablehq/zip")) implicits.add("npm:jszip");
   if (set.has("observablehq:stdlib/vega-lite")) implicits.add("npm:vega-lite-api").add("npm:vega-lite").add("npm:vega");
+  if (set.has("observablehq:stdlib/vgplot")) implicits.add("npm:@uwdata/vgplot").add("npm:@observablehq/duckdb").add("npm:@duckdb/duckdb-wasm"); // prettier-ignore
   return implicits;
 }