analysis: improve analysis scripts (#250)

do not update data for compare-with-entsoe-stats
do not overwrite built-in map command
create required output folder for figures

Co-authored-by: Fabian Neumann <fabian.neumann@outlook.de>

Author: maurerle

analysis/compare-with-entsoe-stats.py

@@ -20,12 +20,13 @@
 root = pathlib.Path(__file__).parent.absolute()
 figpath = root / "figures"
 
-UPDATE = True
+# whether to update the used dataset or use the precalculated data
+UPDATE = False
 
 
 config = pm.get_config()
 
-powerplants = pm.powerplants(update=UPDATE)
+powerplants = pm.powerplants(update=UPDATE, from_url=not UPDATE)
 
 
 powerplants = powerplants.powerplant.convert_country_to_alpha2()
@@ -47,7 +48,7 @@ def parse(c):
                 rename.get(c, c), **kwargs
             ).iloc[0]
         except Exception as e:
-            print(f"Country {c} failed with {e}")
+            print(f"Country {c} failed with {repr(e)}")
             time.sleep(3)
     return np.nan
 
@@ -68,14 +69,17 @@ def parse(c):
 
 sources = [s if isinstance(s, str) else list(s)[0] for s in config["matching_sources"]]
 
-input_dbs = {
-    s.title(): getattr(pm.data, s)()
-    .powerplant.convert_country_to_alpha2()
-    .query(query)
-    .powerplant.lookup()
-    .fillna(0)
-    for s in sources
-}
+input_dbs = {}
+for s in sources:
+    print(s.title())
+    input_dbs[s.title()] = (
+        getattr(pm.data, s)()
+        .powerplant.convert_country_to_alpha2()
+        .query(query)
+        .powerplant.lookup()
+        .fillna(0)
+    )
+
 output_dbs = {
     s.title(): powerplants[
         powerplants.projectID.apply(lambda ds: s in ds)
@@ -103,7 +107,9 @@ def parse(c):
     diff[s.title() + " (%)"] = ds.fillna(0)
 
 diff = diff[out_compare.Statistics != 0]
-diff = diff.loc[:, list(set(out_compare.index.unique(1)) - {"Wind", "Solar"}), :]
+diff = diff.loc[
+    :, list(set(out_compare.index.unique(1)) - {"Biogas", "Wind", "Solar"}), :
+]
 diff.index = diff.index.get_level_values(0) + " " + diff.index.get_level_values(1)
 
 df = (diff[diff.Difference > 1]).sort_values("Difference", ascending=False)
@@ -125,6 +131,8 @@ def parse(c):
 #                                country figures                               #
 # ---------------------------------------------------------------------------- #
 
+(figpath / "country-comparison").mkdir(parents=True, exist_ok=True)
+
 fig, ax = plt.subplots(figsize=(5, 20))
 diff[diff.abs() > 2].plot.barh(ax=ax, zorder=3)
 ax.set_xlabel("Capacity difference (stats - ppm) [GW]")

analysis/danish-powerplants.py

@@ -16,7 +16,7 @@
 config_da2020 = copy.copy(config)
 config_da2020["main_query"] = "Country == 'Denmark'"
 config_da2020["hash"] = "DA-2020"
-config_da2020["matching_sources"].remove("WIKIPEDIA")
+config_da2020["matching_sources"].remove({"WIKIPEDIA": "Fueltype != 'Solar'"})
 config_da2020["matching_sources"].remove("JRC")
 
 
@@ -42,7 +42,7 @@
         name="Select", options=["Fueltype", "Country"]
     )
 
-    map = df.hvplot.points(
+    point_map = df.hvplot.points(
         "lon",
         "lat",
         color="Fueltype",
@@ -74,6 +74,6 @@
     )
     # bars.opts(opts.Overlay(title=None))
 
-    plot = map + bars
+    plot = point_map + bars
 
     hvplot.save(plot, "figures/" + label + ".html")

analysis/german-powerplants.py

@@ -4,6 +4,7 @@
 
 import copy
 
+import geoviews  # noqa
 import hvplot
 import hvplot.pandas  # noqa
 import hvplot.xarray  # noqa
@@ -25,7 +26,7 @@
 
 config_de2030 = copy.copy(config)
 config_de2030["main_query"] = query.format(year=2030)
-config_de2030["matching_sources"].remove("WIKIPEDIA")
+config_de2030["matching_sources"].remove({"WIKIPEDIA": "Fueltype != 'Solar'"})
 config_de2030["hash"] = "DE-2030"
 
 
@@ -51,7 +52,7 @@
         name="Select", options=["Fueltype", "Country"]
     )
 
-    map = df.hvplot.points(
+    point_map = df.hvplot.points(
         "lon",
         "lat",
         color="Fueltype",
@@ -83,6 +84,6 @@
     )
     # bars.opts(opts.Overlay(title=None))
 
-    plot = map + bars
+    plot = point_map + bars
 
     hvplot.save(plot, "figures/" + label + ".html")

analysis/interactive-map-with-widgets.py

@@ -16,7 +16,7 @@
 cmap = {k: v for k, v in config["fuel_to_color"].items() if k in df.Fueltype.unique()}
 checkbox = pn.widgets.CheckButtonGroup(name="Select", options=["Fueltype", "Country"])
 
-map = df.hvplot.points(
+point_map = df.hvplot.points(
     "lon",
     "lat",
     color="Fueltype",
@@ -38,5 +38,5 @@
 )
 
 
-panel = pn.Column(checkbox, map)
+panel = pn.Column(checkbox, point_map)
 panel.show()

analysis/interactive-map.py

@@ -23,7 +23,7 @@
 cmap = {k: v for k, v in config["fuel_to_color"].items() if k in df.Fueltype.unique()}
 # checkbox = pn.widgets.CheckButtonGroup(name="Select", options=["Fueltype", "Country"])
 
-map = df.hvplot.points(
+point_map = df.hvplot.points(
     "lon",
     "lat",
     color="Fueltype",
@@ -44,7 +44,7 @@
     # features={"rivers": "10m", "lakes": "10m"},
 )
 
-hvplot.save(map, "figures/powerplant-map.html")
+hvplot.save(point_map, "figures/powerplant-map.html")
 
 bars = grouped.hvplot.barh(
     by="Fueltype",