New tip for demand graph in EIA (US Energy) example (#1041)

* 1. Remove unused columns from data loader
2. Move row filtering from page to data loader
3. Change tooltip in line chart to show all values for a given date so values are not obscured
4. Add comments and minor code decluttering

* Use default colors for hover tip

* Rolled up data in a separate table for tips

* Rollup to tip (cont'd)

* Hover feedback to dashed line

* git hygiene — rename this file instead of delete+new (pt 1)

* git hygiene pt 2 — Delete us-demand.zip.js

* git hygiene pt 3 — rename us-demand.csv.js to us-demand.zip.js

* git hygiene pt 4 — trying CLI mv instead of github UI

* git hygiene pt 5 — CLI mv

* Remove redundant data type labels

* Remove chart junk

* Remove second "date" from tip

* Remove option that does nothing

* Add semicolons

* Add semicolons

* Revert to single csv data loader instead of zip

* Change date to more global friendly format

* minimize diff & tighter loop

* adopt d3.rollup

---------

Co-authored-by: Philippe Rivière <[email protected]>

Author: pettiross

examples/eia/docs/components/charts.js

@@ -1,5 +1,16 @@
 import * as Plot from "npm:@observablehq/plot";
-import {extent} from "npm:d3";
+import {extent, format, rollup, timeFormat} from "npm:d3";
+
+function friendlyTypeName(d) {
+  switch (d) {
+    case "demandActual":
+      return "Demand (actual)";
+    case "demandForecast":
+      return "Demand (forecast)";
+    case "netGeneration":
+      return "Net generation";
+  }
+}
 
 // Top 5 balancing authorities chart
 export function top5BalancingAuthoritiesChart(width, height, top5Demand, maxDemand) {
@@ -17,29 +28,67 @@ export function top5BalancingAuthoritiesChart(width, height, top5Demand, maxDema
         fill: "#9498a0",
         sort: {y: "x", reverse: true, limit: 10},
         tip: true,
-        title: ({ name, value }) => `name: ${name}\ndemand: ${value / 1000} GWh`
+        title: ({name, value}) => `name: ${name}\ndemand: ${value / 1000} GWh`
       })
     ]
   });
 }
 
 // US electricity demand, generation and forecasting chart
-export function usGenDemandForecastChart(width, height, usDemandGenForecast, currentHour) {
+export function usGenDemandForecastChart(width, height, data, currentHour) {
+  // Roll up each hour's values into a single row for a cohesive tip
+  const compoundTips = rollup(
+    data,
+    (v) => ({...v[0], ...Object.fromEntries(v.map(({name, value}) => [name, value]))}),
+    (d) => d.date
+  ).values();
+
   return Plot.plot({
     width,
     marginTop: 0,
     height: height - 50,
-    y: {label: null},
-    x: {type: "time", tickSize: 0, tickPadding: 3},
+    y: {tickSize: 0, label: null},
+    x: {type: "time", tickSize: 0, tickPadding: 3, label: null},
     color: {
       legend: true,
-      domain: ["Day-ahead demand forecast", "Demand", "Net generation"],
-      range: ["#6cc5b0", "#ff8ab7", "#a463f2"]
+      domain: ["demandActual", "demandForecast", "netGeneration"],
+      tickFormat: friendlyTypeName,
+      range: ["#ff8ab7", "#6cc5b0", "#a463f2"]
     },
     grid: true,
     marks: [
       Plot.ruleX([currentHour], {strokeOpacity: 0.5}),
-      Plot.line(usDemandGenForecast, {x: "date", y: (d) => d.value / 1000, stroke: "name", strokeWidth: 1.2, tip: true})
+      Plot.line(data, {
+        x: "date",
+        y: (d) => d.value / 1000,
+        stroke: "name",
+        strokeWidth: 1.2
+      }),
+      Plot.ruleX(
+        compoundTips,
+        Plot.pointerX({
+          x: "date",
+          strokeDasharray: [2, 2],
+          channels: {
+            date: {value: "date", label: "Time"},
+            demandActual: {value: "demandActual", label: friendlyTypeName("demandActual")},
+            demandForecast: {value: "demandForecast", label: friendlyTypeName("demandForecast")},
+            netGeneration: {value: "netGeneration", label: friendlyTypeName("netGeneration")}
+          },
+          tip: {
+            format: {
+              date: (d) => timeFormat("%-d %b %-I %p")(d),
+              demandActual: (d) => `${format(".1f")(d / 1000)} GWh`,
+              demandForecast: (d) => `${format(".1f")(d / 1000)} GWh`,
+              netGeneration: (d) => `${format(".1f")(d / 1000)} GWh`,
+              x: false
+            },
+            fontSize: 12,
+            anchor: "bottom",
+            frameAnchor: "top"
+          }
+        })
+      )
     ]
   });
 }

examples/eia/docs/data/us-demand.csv.js

@@ -1,33 +1,31 @@
 import * as d3 from "d3";
 
-const end = d3.timeDay.offset(d3.timeHour(new Date()), 1)
+// Construct web API call for last 7 days of hourly demand data in MWh
+// Types: DF = forecasted demand, D = demand (actual), NG = net generation
+const end = d3.timeDay.offset(d3.timeHour(new Date()), 1);
+const start = d3.timeHour(d3.utcDay.offset(end, -7));
+const convertDate = d3.timeFormat("%m%d%Y %H:%M:%S");
+const usDemandUrl = `https://www.eia.gov/electricity/930-api/region_data/series_data?type[0]=DF&type[1]=D&type[2]=NG&start=${convertDate(start)}&end=${convertDate(end)}&frequency=hourly&timezone=Eastern&limit=10000&respondent[0]=US48`;
 
-const start = d3.timeHour(d3.utcDay.offset(end, -7))
+const datetimeFormat = d3.utcParse("%m/%d/%Y %H:%M:%S");
+const dateFormat = d3.utcParse("%m/%d/%Y");
+const typeNameRemap = {DF: "demandForecast", D: "demandActual", NG: "netGeneration"};
 
-const convertDate = d3.timeFormat("%m%d%Y %H:%M:%S")
-
-const usDemandUrl = `https://www.eia.gov/electricity/930-api/region_data/series_data?type[0]=D&type[1]=DF&type[2]=NG&type[3]=TI&start=${convertDate(start)}&end=${convertDate(end)}&frequency=hourly&timezone=Eastern&limit=10000&respondent[0]=US48`
-
-const tidySeries = (response, id, name) => {
-    let series = response[0].data
-    let datetimeFormat = d3.utcParse("%m/%d/%Y %H:%M:%S")
-    let dateFormat = d3.utcParse("%m/%d/%Y")
-    return series.flatMap(s => {
-      return s.VALUES.DATES.map((d,i) => {
-        return {
-          id: s[id],
-          name: s[name],
-          date: datetimeFormat(d) ? datetimeFormat(d) : dateFormat(d),
-          value: s.VALUES.DATA[i],
-          reported: s.VALUES.DATA_REPORTED[i],
-          imputed: s.VALUES.DATA_IMPUTED[i]
-        }
-      })
+// Flatten JSON from date / type / value hierarchy to a tidy array
+const jsonToTidy = (data, id) => {
+  let series = data[0].data
+  return series.flatMap(s => {
+    return s.VALUES.DATES.map((d, i) => {
+      return {
+        name: typeNameRemap[s[id]],
+        date: datetimeFormat(d) ?? dateFormat(d),
+        value: s.VALUES.DATA[i]
+      }
     })
-  }
+  })
+};
 
-const usOverviewSeries = await d3.json(usDemandUrl).then(response => {
-    return tidySeries(response, "TYPE_ID", "TYPE_NAME")
-  });
+const jsonData = await d3.json(usDemandUrl);
+const tidySeries = jsonToTidy(jsonData, "TYPE_ID");
 
-process.stdout.write(d3.csvFormat(usOverviewSeries));
+process.stdout.write(d3.csvFormat(tidySeries));

examples/eia/docs/index.md

@@ -14,11 +14,6 @@ import {balancingAuthoritiesLegend, balancingAuthoritiesMap} from "./components/
 const countryInterchangeSeries = FileAttachment("data/country-interchange.csv").csv({typed: true});
 ```
 
-```js
-// US overall demand, generation, forecast
-const usOverview = FileAttachment("data/us-demand.csv").csv({typed: true});
-```
-
 ```js
 const baHourlyDemand = FileAttachment("data/eia-ba-hourly.csv").csv({typed: true});
 ```
@@ -50,8 +45,8 @@ const eiaPoints = FileAttachment("data/eia-system-points.json").json().then(d =>
 ```
 
 ```js
-// US total demand, generation and forecast excluding total (sum)
-const usDemandGenForecast = usOverview.filter(d => d.name != "Total interchange");
+// US total demand, generation and forecast
+const usDemandGenForecast = FileAttachment("data/us-demand.csv").csv({typed: true});
 ```
 
 ```js
@@ -176,7 +171,7 @@ function centerResize(render) {
     ${resize((width, height) => top5BalancingAuthoritiesChart(width, height, top5LatestDemand, maxDemand))}
   </div>
   <div class="card grid-colspan-2">
-    <h2>US electricity generation, demand, and demand forecast (GWh)</h2>
+    <h2>US electricity generation demand vs. day-ahead forecast (GWh)</h2>
     ${resize((width, height) => usGenDemandForecastChart(width, height, usDemandGenForecast, currentHour))}
   </div>
   <div class="card grid-colspan-2">