Merge branch 'master' of github.com:pypsa/pypsa-eur-sec

Author: fneum

.github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2021 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 

LICENSE.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright 2017-2021 The PyPSA-Eur Authors
+Copyright 2017-2023 The PyPSA-Eur Authors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

config.default.yaml

@@ -1,4 +1,4 @@
-version: 0.6.0
+version: 0.7.0
 
 logging_level: INFO
 
@@ -510,6 +510,7 @@ plotting:
     natural gas: '#e05b09'
     CCGT: '#a85522'
     CCGT marginal: '#a85522'
+    allam: '#B98F76'
     gas for industry co2 to atmosphere: '#692e0a'
     gas for industry co2 to stored: '#8a3400'
     gas for industry: '#853403'

doc/conf.py

@@ -63,17 +63,17 @@
 
 # General information about the project.
 project = u'PyPSA-Eur-Sec'
-copyright = u'2019-2021 Tom Brown (KIT, TUB), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB), Fabian Neumann (TUB)'
-author = u'2019-2021 Tom Brown (KIT, TUB), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB), Fabian Neumann (TUB)'
+copyright = u'2019-2023 Tom Brown (KIT, TUB), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB), Fabian Neumann (TUB)'
+author = u'2019-2023 Tom Brown (KIT, TUB), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB), Fabian Neumann (TUB)'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = u'0.6'
+version = u'0.7'
 # The full version, including alpha/beta/rc tags.
-release = u'0.6.0'
+release = u'0.7.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/data.csv

@@ -16,6 +16,7 @@ existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unk
 IRENA existing VRE capacities,existing_infrastructure/{solar|onwind|offwind}_capcity_IRENA.csv,unknown,https://www.irena.org/Statistics/Download-Data
 USGS ammonia production,myb1-2017-nitro.xls,unknown,https://www.usgs.gov/centers/nmic/nitrogen-statistics-and-information
 hydrogen salt cavern potentials,h2_salt_caverns_GWh_per_sqkm.geojson,CC BY 4.0,https://doi.org/10.1016/j.ijhydene.2019.12.161 https://doi.org/10.20944/preprints201910.0187.v1
+international port trade volumes,attributed_ports.json,CC BY 4.0,https://datacatalog.worldbank.org/search/dataset/0038118/Global---International-Ports
 hotmaps industrial site database,Industrial_Database.csv,CC BY 4.0,https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database
 Hotmaps building stock data,data_building_stock.csv,CC BY 4.0,https://gitlab.com/hotmaps/building-stock
 U-values Poland,u_values_poland.csv,unknown,https://data.europa.eu/euodp/de/data/dataset/building-stock-observatory

doc/index.rst

@@ -97,6 +97,15 @@ Ministry for Education and Research (BMBF) <https://www.bmbf.de/en/index.html>`_
 as part of the `Stromnetze Research Initiative
 <http://forschung-stromnetze.info/projekte/grundlagen-und-konzepte-fuer-effiziente-dezentrale-stromnetze/>`_.
 
+Workflow Outline
+================
+
+.. image:: ../graphics/workflow.png
+
+.. note::
+    The graph above was generated using
+    ``snakemake --rulegraph -F | sed -n "/digraph/,/}/p" | dot -Tpng -o workflow.png``
+
 
 Documentation
 =============

doc/release_notes.rst

@@ -8,31 +8,49 @@ Future release
 .. note::
   This unreleased version currently may require the master branches of PyPSA, PyPSA-Eur, and the technology-data repository.
 
-This release includes the addition of the European gas transmission network and
-incorporates retrofitting options to hydrogen.
+* new feature
+
+
+PyPSA-Eur-Sec 0.7.0 (16th February 2023)
+========================================
+
+This release includes many new features. Highlights include new gas
+infrastructure data with retrofitting options for hydrogen transport, improved
+carbon management and infrastructure planning, regionalised potentials for
+hydrogen underground storage and carbon sequestration, new applications for
+biomass, and explicit modelling of methanol and ammonia as separate energy
+carriers.
+
+This release is known to work with `PyPSA-Eur
+<https://github.com/PyPSA/pypsa-eur>`_ Version 0.7.0 and `Technology Data
+<https://github.com/PyPSA/technology-data>`_ Version 0.5.0.
 
 **Gas Transmission Network**
 
 * New rule ``retrieve_gas_infrastructure_data`` that downloads and extracts the
-  SciGRID_gas `IGGIELGN <https://zenodo.org/record/4767098>`_ dataset from zenodo.
-  It includes data on the transmission routes, pipe diameters, capacities, pressure,
-  and whether the pipeline is bidirectional and carries H-Gas or L-Gas.
+  SciGRID_gas `IGGIELGN <https://zenodo.org/record/4767098>`_ dataset from
+  zenodo. It includes data on the transmission routes, pipe diameters,
+  capacities, pressure, and whether the pipeline is bidirectional and carries
+  H-Gas or L-Gas.
 
-* New rule ``build_gas_network`` processes and cleans the pipeline data from SciGRID_gas.
-  Missing or uncertain pipeline capacities can be inferred by diameter.
+* New rule ``build_gas_network`` processes and cleans the pipeline data from
+  SciGRID_gas. Missing or uncertain pipeline capacities can be inferred by
+  diameter.
 
 * New rule ``build_gas_input_locations`` compiles the LNG import capacities
-  (including planned projects from gem.wiki), pipeline entry capacities and
-  local production capacities for each region of the model. These are the
-  regions where fossil gas can eventually enter the model.
+  (from the Global Energy Monitor's `Europe Gas Tracker
+  <https://globalenergymonitor.org/projects/europe-gas-tracker/>`_, pipeline
+  entry capacities and local production capacities for each region of the model.
+  These are the regions where fossil gas can eventually enter the model.
 
 * New rule ``cluster_gas_network`` that clusters the gas transmission network
-  data to the model resolution. Cross-regional pipeline capacities are aggregated
-  (while pressure and diameter compatibility is ignored), intra-regional pipelines
-  are dropped. Lengths are recalculated based on the regions' centroids.
+  data to the model resolution. Cross-regional pipeline capacities are
+  aggregated (while pressure and diameter compatibility is ignored),
+  intra-regional pipelines are dropped. Lengths are recalculated based on the
+  regions' centroids.
 
-* With the option ``sector: gas_network:``, the existing gas network is
-  added with a lossless transport model. A length-weighted `k-edge augmentation
+* With the option ``sector: gas_network:``, the existing gas network is added
+  with a lossless transport model. A length-weighted `k-edge augmentation
   algorithm
   <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_
   can be run to add new candidate gas pipelines such that all regions of the
@@ -41,92 +59,167 @@ incorporates retrofitting options to hydrogen.
   the gas network is activated, all the gas demands are regionally disaggregated
   as well.
 
-* New constraint allows endogenous retrofitting of gas pipelines to hydrogen pipelines.
-  This option is activated via the setting ``sector: H2_retrofit:``. For every
-  unit of gas pipeline capacity dismantled, ``sector:
+* New constraint allows endogenous retrofitting of gas pipelines to hydrogen
+  pipelines. This option is activated via the setting ``sector: H2_retrofit:``.
+  For every unit of gas pipeline capacity dismantled, ``sector:
   H2_retrofit_capacity_per_CH4`` units are made available as hydrogen pipeline
   capacity in the corresponding corridor. These repurposed hydrogen pipelines
-  have lower costs than new hydrogen pipelines. Both new and repurposed pipelines
-  can be built simultaneously. The retrofitting option ``sector: H2_retrofit:`` also works
-  with a copperplated methane infrastructure, i.e. when ``sector: gas_network: false``.
+  have lower costs than new hydrogen pipelines. Both new and repurposed
+  pipelines can be built simultaneously. The retrofitting option ``sector:
+  H2_retrofit:`` also works with a copperplated methane infrastructure, i.e.
+  when ``sector: gas_network: false``.
 
 * New hydrogen pipelines can now be built where there are already power or gas
   transmission routes. Previously, only the electricity transmission routes were
   considered.
 
-**New features and functionality**
+**Carbon Management and Biomass**
 
+* Add option to spatially resolve carrier representing stored carbon dioxide
+  (``co2_spatial``). This allows for more detailed modelling of CCUTS, e.g.
+  regarding the capturing of industrial process emissions, usage as feedstock
+  for electrofuels, transport of carbon dioxide, and geological sequestration
+  sites.
+
+* Add option for regionally-resolved geological carbon dioxide sequestration
+  potentials through new rule ``build_sequestration_potentials`` based on
+  `CO2StoP <https://setis.ec.europa.eu/european-co2-storage-database_en>`_. This
+  can be controlled in the section ``regional_co2_sequestration_potential`` of
+  the ``config.yaml``. It includes options to select the level of conservatism,
+  whether onshore potentials should be included, the respective upper and lower
+  limits per region, and an annualisation parameter for the cumulative
+  potential. The defaults are preliminary and will be validated the next
+  release.
 
-* Add option to aggregate network temporally using representative snapshots or segments (with tsam package)
+* Add option to sweep the global CO2 sequestration potentials with keyword
+  ``seq200`` in the ``{sector_opts}`` wildcard (for limit of 200 Mt CO2).
 
-* Add option for biomass boilers (wood pellets) for decentral heating
+* Add option to include `Allam cycle gas power plants
+  <https://en.wikipedia.org/wiki/Allam_power_cycle>`_ (``allam_cycle``).
 
-* Add option for BioSNG (methane from biomass) with and without CC
+* Add option for planning a new carbon dioxide network (``co2network``).
 
-* Add option for BtL (Biomass to liquid fuel/oil) with and without CC
+* Separate option to regionally resolve biomass (``biomass_spatial``) from
+  option to allow biomass transport (``biomass_transport``).
 
-* Add option for minimum part load for Fischer-Tropsch plants (default: 90%) and methanolisation plants (default: 50%).
+* Add option for biomass boilers (wood pellets) for decentral heating.
 
-* Units are assigned to the buses. These only provide a better understanding. The specifications of the units are not taken into account in the optimisation, which means that no automatic conversion of units takes place.
+* Add option for BioSNG (methane from biomass) with and without carbon capture.
 
-* Option ``retrieve_sector_databundle`` to automatically retrieve and extract data bundle.
+* Add option for BtL (biomass to liquid fuel/oil) with and without carbon
+  capture.
 
-* Add option to use waste heat of electrolysis in district heating networks (``use_electrolysis_waste_heat``).
 
-* Add regionalised hydrogen salt cavern storage potentials from `Technical Potential of Salt Caverns for Hydrogen Storage in Europe <https://doi.org/10.20944/preprints201910.0187.v1>`_.
+**Other new features**
 
-* Add option to sweep the global CO2 sequestration potentials with keyword ``seq200`` in the ``{sector_opts}`` wildcard (for limit of 200 Mt CO2).
+* Add regionalised hydrogen salt cavern storage potentials from `Technical
+  Potential of Salt Caverns for Hydrogen Storage in Europe
+  <https://doi.org/10.20944/preprints201910.0187.v1>`_. This data is compiled in
+  a new rule ``build_salt_cavern_potentials``.
 
 * Add option to resolve ammonia as separate energy carrier with Haber-Bosch
   synthesis, ammonia cracking, storage and industrial demand. The ammonia
-  carrier can be nodally resolved or copperplated across Europe. This feature is
-  controlled by ``sector: ammonia:``.
+  carrier can be nodally resolved or copperplated across Europe (see
+  ``ammonia``).
+
+* Add methanol as energy carrier, methanolisation as process, and option for
+  methanol demand in shipping sector.
+
+* Shipping demand now defaults to methanol rather than liquefied hydrogen
+  until 2050.
 
-* Add methanol as energy carrier, methanolisation as process, and option for methanol demand in shipping sector.
+* Demand for liquid hydrogen in international shipping is now geographically
+  distributed by port trade volumes in a new rule ``build_shipping_demand``
+  using data from the `World Bank Data Catalogue
+  <https://datacatalog.worldbank.org/search/dataset/0038118/Global---International-Ports>`_.
+  Domestic shipping remains distributed by population.
 
-* Updated `data bundle <https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz>`_ that includes the hydrogan salt cavern storage potentials.
+* Add option to aggregate network temporally using representative snapshots or
+  segments (with `tsam <https://github.com/FZJ-IEK3-VSA/tsam>`_).
 
-* Updated and extended documentation in <https://pypsa-eur-sec.readthedocs.io/en/latest/>
+* Add option for minimum part load for Fischer-Tropsch plants (default: 90%) and
+  methanolisation plants (default: 50%).
 
-* Shipping demand now defaults to (synthetic) oil rather than liquefied hydrogen until 2050.
+* Add option to use waste heat of electrolysis in district heating networks
+  (``use_electrolysis_waste_heat``).
 
-* Improved network plots including better legends, hydrogen retrofitting network display, and change to EqualEarth projection.
+* Add option for coal CHPs with carbon capture (see ``coal_cc``).
+
+* In overnight optimisation, it is now possible to specify a year for the
+  technology cost projections separate from the planning horizon.
 
 * New config options for changing energy demands in aviation
   (``aviation_demand_factor``) and HVC industry (``HVC_demand_factor``), as well
   as explicit ICE shares for land transport (``land_transport_ice_share``) and
   agriculture machinery (``agriculture_machinery_oil_share``).
 
-* Add option to spatially resolve carrier representing stored carbon dioxide
-  (``co2_spatial``). This allows for more detailed modelling of CCUTS, e.g.
-  regarding the capturing of industrial process emissions, usage as feedstock
-  for electrofuels, transport of carbon dioxide, and geological sequestration sites.
+* It is now possible to merge residential and services heat buses to reduce the
+  problem size (see ``cluster_heat_nodes``).
 
-* Add option for planning a new carbon dioxide network (``co2network``).
+* Added option to tweak (almost) any configuration parameter through the
+  ``{sector_opts}`` wildcard. The regional_co2_sequestration_potential is
+  triggered by the prefix ``CF+`` after which it is possible to pipe to any
+  setting that does not contain underscores (``_``). Example:
+  ``CF+sector+v2g+false`` disables vehicle-to-grid flexibility.
 
+* Option ``retrieve_sector_databundle`` to automatically retrieve and extract
+  data bundle.
 
-* Add option for regionally-resolved geological carbon dioxide sequestration
-  potentials through new rule ``build_sequestration_potentials`` based on
-  `CO2StoP <https://setis.ec.europa.eu/european-co2-storage-database_en>`_. This
-  can be controlled in the section ``regional_co2_sequestration_potential`` of
-  the ``config.yaml``. It includes options to select the level of conservatism,
-  whether onshore potentials should be included, the respective upper and lower
-  limits per region, and an annualisation parameter for the cumulative
-  potential. The defaults are preliminary and will be validated the next
-  release.
+* Removed the need to clone ``technology-data`` repository in a parallel
+  directory. The new approach automatically retrieves the technology data from
+  remote in the rule ``retrieve_cost_data``.
 
-* Separate option to regionally resolve biomass (``biomass_spatial``) from
-  option to allow biomass transport (``biomass_transport``).
+* Improved network plots including better legends, hydrogen retrofitting network
+  display, and change to EqualEarth projection. A new color scheme for
+  technologies was also introduced.
 
-* Add option to include `Allam cycle gas power plants
-  <https://en.wikipedia.org/wiki/Allam_power_cycle>`_ (``allam_cycle``).
+* Add two new rules ``build_transport_demand`` and
+  ``build_population_weighted_energy_totals`` using code previously contained in
+  ``prepare_sector_network``.
+
+* Rules that convert weather data with ``atlite`` now largely run separately for
+  categories residential, rural and total.
+
+* Units are assigned to the buses. These only provide a better understanding.
+  The specifications of the units are not taken into account in the
+  optimisation, which means that no automatic conversion of units takes place.
+
+* Configuration file and wildcards are now stored under ``n.meta`` in every
+  PyPSA network.
+
+* Updated `data bundle
+  <https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz>`_
+  that includes the hydrogan salt cavern storage potentials.
+
+* Updated and extended documentation in
+  <https://pypsa-eur-sec.readthedocs.io/en/latest/>
+
+* Added new rule ``copy_conda_env`` that exports a list of packages with which
+  the workflow was executed.
+
+* Add basic continuous integration using Github Actions.
+
+* Add basic ``rsync`` setup.
 
 **Bugfixes**
 
-* The CO2 sequestration limit implemented as GlobalConstraint (introduced in the previous version)
-  caused a failure to read in the shadow prices of other global constraints.
+* The CO2 sequestration limit implemented as GlobalConstraint (introduced in the
+  previous version) caused a failure to read in the shadow prices of other
+  global constraints.
+
+* Correct capital cost of Fischer-Tropsch according to new units in
+  ``technology-data`` repository.
+
+* Fix unit conversion error for thermal energy storage.
+
+* For myopic pathway optimisation, set optimised capacities of power grid
+  expansion of previous iteration as minimum capacity for next iteration.
 
-* Correct capital cost of Fischer-Tropsch according to new units in ``technology-data``.
+* Further rather minor bugfixes for myopic optimisation code (see `#256
+  <https://github.com/PyPSA/pypsa-eur-sec/pull/256>`_).
+
+
+Many thanks to all who contributed to this release!
 
 
 PyPSA-Eur-Sec 0.6.0 (4 October 2021)

graphics/workflow.png

@@ -178,7 +178,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
     busmap = pd.read_csv(snakemake.input.busmap, index_col=0).squeeze()
 
     inv_busmap = {}
-    for k, v in busmap.iteritems():
+    for k, v in busmap.items():
         inv_busmap[v] = inv_busmap.get(v, []) + [k]
 
     clustermaps = busmap_s.map(busmap)

scripts/add_existing_baseyear.py

@@ -40,7 +40,7 @@
     reference = ["RS", "BA"]
     average = urban_fraction[reference].mean()
     fill_values = pd.Series({ct: average for ct in missing})
-    urban_fraction = urban_fraction.append(fill_values)
+    urban_fraction = pd.concat([urban_fraction, fill_values])
 
     # population in each grid cell
     pop_cells = pd.Series(I.dot(nuts3['pop']))