diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 641a2829..1f9ff734 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -5,7 +5,7 @@ Closes # (if applicable). ## Checklist -- [ ] Code changes are sufficiently documented; i.e. new functions contain docstrings and further explanations may be given in `docs`. +- [ ] Code changes are sufficiently documented; i.e. new functions contain docstrings and further explanations may be given in `doc`. - [ ] Unit tests for new features were added (if applicable). -- [ ] A note for the release notes `doc/release_notes.md` of the upcoming release is included. +- [ ] A note for the release notes `doc/release_notes.rst` of the upcoming release is included. - [ ] I consent to the release of this PR's code under the MIT license and have added my name to the `doc/contributors.md`. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d6e8c236..a64c44d6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,6 @@ repos: rev: v5.0.0 hooks: - id: check-yaml - args: ['--unsafe'] # required for mkdocs.yaml -> pymdownx.emoji (see https://github.com/pre-commit/pre-commit-hooks/issues/552) - id: end-of-file-fixer - id: trailing-whitespace - id: check-merge-conflict diff --git a/.readthedocs.yaml b/.readthedocs.yaml index efdd086b..f448ac10 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,20 +9,16 @@ version: 2 - -mkdocs: - configuration: mkdocs.yaml - build: - os: ubuntu-24.04 + os: ubuntu-22.04 tools: - python: "3.13" + python: "3.11" + + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: doc/conf.py - jobs: - pre_system_dependencies: - - git fetch --unshallow # Needed to get version tags - pre_install: - - pip install mkdocs-material python: install: diff --git a/README.md b/README.md index 19ad53a7..728dc49b 100644 --- a/README.md +++ b/README.md @@ -104,13 +104,10 @@ and/or the current release stored on Zenodo with a release-specific DOI: ## Licence `powerplantmatching` is released as free software under the [MIT](LICENSES/MIT.txt) license. -The default output data [powerplants.csv](powerplants.csv) generated by the package is released under [CC BY 4.0](LICENSES/CC-BY-4.0.txt). -Parts of the repository may be licensed under different licenses, especially dependent package binaries for `duke` being licensed under [Apache 2.0 license](https://github.com/PyPSA/powerplantmatching/tree/master/LICENSES/Apache-2.0.txt). +The default output data generated by the package is released under [CC BY 4.0](LICENSES/CC-BY-4.0.txt). +Parts of the repository may be licensed under different licenses, please see the header information of the respective files and [REUSE.toml](REUSE.toml) for details. This repository uses the [REUSE](https://reuse.software/) conventions to indicate the licenses that apply to individual files and parts of the repository. -For details on the licenses that apply, see the the header information of the respective files and [REUSE.toml](REUSE.toml) for details. -Copyright 2018-2024 Fabian Gotzens (FZ Jülich), Jonas Hörsch (KIT), Fabian Hofmann (FIAS) -Copyright 2025- Contributors to powerplantmatching - -You can find a list of contributors in the [contributors page](https://github.com/PyPSA/powerplantmatching/graphs/contributors) and in the [contributors file](docs/contributors.md). +(c) Contributors to powerplantmatching +You can find a list of contributors in the [contributors page](https://github.com/PyPSA/powerplantmatching/graphs/contributors). diff --git a/REUSE.toml b/REUSE.toml index 5bd0e304..830c7043 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -6,8 +6,7 @@ SPDX-PackageDownloadLocation = "https://github.com/pypsa/powerplantmatching" [[annotations]] path = [ "powerplants.csv", - "docs/assets/images/powerplants.png", - "docs/examples/example.ipynb", + "doc/**", ] SPDX-FileCopyrightText = "Contributors to powerplantmatching " SPDX-License-Identifier = "CC-BY-4.0" diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc/api-core.rst b/doc/api-core.rst new file mode 100644 index 00000000..a954bf2b --- /dev/null +++ b/doc/api-core.rst @@ -0,0 +1,9 @@ +============ +Core Modules +============ + +.. automodapi:: powerplantmatching.core + +.. automodapi:: powerplantmatching.duke + +.. automodapi:: powerplantmatching.accessor diff --git a/doc/api-data.rst b/doc/api-data.rst new file mode 100644 index 00000000..2551384b --- /dev/null +++ b/doc/api-data.rst @@ -0,0 +1,14 @@ +======================= +Data Processing Modules +======================= + +.. automodapi:: powerplantmatching.data + + +.. automodapi:: powerplantmatching.cleaning + + +.. automodapi:: powerplantmatching.matching + + +.. automodapi:: powerplantmatching.collection diff --git a/doc/api-utils.rst b/doc/api-utils.rst new file mode 100644 index 00000000..8fda772b --- /dev/null +++ b/doc/api-utils.rst @@ -0,0 +1,14 @@ +=============== +Utility Modules +=============== + +.. automodapi:: powerplantmatching.utils + + +.. automodapi:: powerplantmatching.export + + +.. automodapi:: powerplantmatching.heuristics + + +.. automodapi:: powerplantmatching.plot diff --git a/doc/basics.rst b/doc/basics.rst new file mode 100644 index 00000000..7e91618b --- /dev/null +++ b/doc/basics.rst @@ -0,0 +1,199 @@ +================= +General Structure +================= + +The dataset combines the data of all the data sources listed in +`Data-Sources <#Data-Sources>`__ and provides the following information: + +- **Power plant name** - claim of each database +- **Fueltype** - {Solid Biomass, Biogas, Geothermal, Hard Coal, Hydro, Lignite, + Nuclear, Natural Gas, Oil, Solar, Wind, Other} +- **Technology** - {CCGT, OCGT, Steam Turbine, Combustion Engine, + Run-Of-River, Pumped Storage, Reservoir} +- **Set** - {Power Plant (PP), Combined Heat and Power (CHP), Storages + (Stores)} +- **Capacity** - [MW] +- **Duration** - Maximum state of charge capacity in terms of hours at + full output capacity +- **Dam Information** - Dam volume [Mm^3] and Dam Height [m] +- **Geo-position** - Latitude, Longitude +- **Country** - EU-27 + CH + NO (+ UK) minus Cyprus and Malta +- **YearCommissioned** - Commmisioning year of the powerplant +- **RetroFit** - Year of last retrofit +- **projectID** - Immutable identifier of the power plant + + +All data files of the package will be stored in the folder given by +``pm.core.package_config['data_dir']`` + + +Data Sources +------------ + +- OPSD - `Open Power System + Data `__ publish their + `data `__ + under a free license +- GEO - `Global Energy + Observatory `__, the data is not + directly available on the website, but can be obtained from an + `sqlite + scraper `__ +- GPD - `Global Power Plant + Database `__ + provide their data under a free license +- GBPT - `Global Bioenergy Powerplant Tracker by Global Energy Monitor`__ +- GCPT - `Global Coal Powerplant Tracker by Global Energy Monitor `__ +- GGPT - `Global Gas Powerplant Tracker by Global Energy Monitor `__ +- GGTPT - `Global Geothermal Powerplant Tracker by Global Energy Monitor `__ +- GNPT - `Global Nuclear Powerplant Tracker by Global Energy Monitor `__ +- GSPT - `Global Solar Powerplant Tracker by Global Energy Monitor `__ +- GWPT - `Global Wind Powerplant Tracker by Global Energy Monitor `__ +- CARMA - `Carbon Monitoring for Action `__ +- ENTSOe - `European Network of Transmission System Operators for + Electricity `__, annually provides statistics + about aggregated power plant capacities. Their data can be used as a + validation reference. We further use their `annual energy generation + report from + 2010 `__ + as an input for the hydro power plant classification. The `power + plant + dataset `__ + on the ENTSO-E transparency website is downloaded using the `ENTSO-E + Transparency + API `__. +- JRC - `Joint Research Centre Hydro-power plants + database `__ +- IRENA - `International Renewable Energy + Agency `__ open + available statistics on power plant capacities. +- BNETZA - + `Bundesnetzagentur `__ + open available data source for Germany’s power plants +- UBA (Umwelt Bundesamt Datenbank “Kraftwerke in Deutschland) + +Not available but supported sources: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- IWPDCY (International Water Power & Dam Country Yearbook) +- WEPP (Platts, World Elecrtric Power Plants Database) + + +Reliabilty Score +---------------- + +When the matched power plant entries from different sources are combined, the resulting value per column is determined by the most reliable source. The corresponding reliability scores +are: + +======= ================ +Dataset Reliabilty score +======= ================ +JRC 6 +ESE 6 +UBA 5 +OPSD 5 +OPSD_EU 5 +OPSD_DE 5 +WEPP 4 +ENTSOE 4 +IWPDCY 3 +GPD 3 +GEO 3 +BNETZA 3 +CARMA 1 +======= ================ + + + +How it works +------------ + +Whereas single databases as the CARMA, GEO or the OPSD database provide +non standardized and incomplete information, the datasets can complement +each other and improve their reliability. In a first step, +powerplantmatching converts all powerplant dataset into a standardized +format with a defined set of columns and values. The second part +consists of aggregating power plant blocks together into units. Since +some of the datasources provide their powerplant records on unit level, +without detailed information about lower-level blocks, comparing with +other sources is only possible on unit level. In the third and +name-giving step the tool combines (or matches)different, standardized +and aggregated input sources keeping only powerplants units which appear +in more than one source. The matched data afterwards is complemented by +data entries of reliable sources which have not matched. + +The aggregation and matching process heavily relies on +`DUKE `__, a java application +specialized for deduplicating and linking data. It provides many +built-in comparators such as numerical, string or geoposition +comparators. The engine does a detailed comparison for each single +argument (power plant name, fuel-type etc.) using adjusted comparators +and weights. From the individual scores for each column it computes a +compound score for the likeliness that the two powerplant records refer +to the same powerplant. If the score exceeds a given threshold, the two +records of the power plant are linked and merged into one data set. + +Let’s make that a bit more concrete by giving a quick example. Consider +the following two data sets + +Dataset 1: +~~~~~~~~~~ + ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ +| | Name | Fueltype | Classification | Country | Capacity | lat | lon | File | ++===+==========+==========+================+=============+==========+=========+============+======+ +| 0 | Aarberg | Hydro | nan | Switzerland | 14.609 | 47.0444 | 7.27578 | nan | ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ +| 1 | Abbey | Oil | nan | United | 6.4 | 51.687 | -0.0042057 | nan | +| | mills | | | Kingdom | | | | | +| | pumping | | | | | | | | ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ +| 2 | Abertay | Other | nan | United | 8 | 57.1785 | -2.18679 | nan | +| | | | | Kingdom | | | | | ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ +| 3 | Aberthaw | Coal | nan | United | 1552.5 | 51.3875 | -3.40675 | nan | +| | | | | Kingdom | | | | | ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ +| 4 | Ablass | Wind | nan | Germany | 18 | 51.2333 | 12.95 | nan | ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ +| 5 | Abono | Coal | nan | Spain | 921.7 | 43.5588 | -5.72287 | nan | ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ + +and + +Dataset 2: +~~~~~~~~~~ + ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ +| | Name | Fueltype | Classification | Country | Capacity | lat | lon | File | ++===+==========+==========+================+=============+==========+=========+============+======+ +| 0 | Aarberg | Hydro | nan | Switzerland | 15.5 | 47.0378 | 7.272 | nan | ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ +| 1 | Aberthaw | Coal | Thermal | United | 1500 | 51.3873 | -3.4049 | nan | +| | | | | Kingdom | | | | | ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ +| 2 | Abono | Coal | Thermal | Spain | 921.7 | 43.5528 | -5.7231 | nan | ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ +| 3 | Abwinden | Hydro | nan | Austria | 168 | 48.248 | 14.4305 | nan | +| | asten | | | | | | | | ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ +| 4 | Aceca | Oil | CHP | Spain | 629 | 39.941 | -3.8569 | nan | ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ +| 5 | Aceca | Natural | CCGT | Spain | 400 | 39.9427 | -3.8548 | nan | +| | fenosa | gas | | | | | | | ++---+----------+----------+----------------+-------------+----------+---------+------------+------+ + +where Dataset 2 has the higher reliability score. Apparently entries 0, +3 and 5 of Dataset 1 relate to the same power plants as the entries 0, 1 +and 2 of Dataset 2. The toolset detects those similarities and combines +them into the following set, but prioritising the values of Dataset 2: + ++---+----------+----------------+----------+----------------+----------+---------+---------+------+ +| | Name | Country | Fueltype | Classification | Capacity | lat | lon | File | ++===+==========+================+==========+================+==========+=========+=========+======+ +| 0 | Aarberg | Switzerland | Hydro | nan | 15.5 | 47.0378 | 7.272 | nan | ++---+----------+----------------+----------+----------------+----------+---------+---------+------+ +| 1 | Aberthaw | United Kingdom | Coal | Thermal | 1500 | 51.3873 | -3.4049 | nan | ++---+----------+----------------+----------+----------------+----------+---------+---------+------+ +| 2 | Abono | Spain | Coal | Thermal | 921.7 | 43.5528 | -5.7231 | nan | ++---+----------+----------------+----------+----------------+----------+---------+---------+------+ diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 00000000..65e01667 --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,85 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +from importlib.metadata import version as get_version + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +# -- Project information ----------------------------------------------------- + +project = "powerplantmatching" +copyright = "2021-2024, Fabian Hofmann, Fabian Gotzens, Jonas Hörsch, Martha Frysztacki" +author = "Fabian Hofmann, Fabian Gotzens, Jonas Hörsch, Martha Frysztacki" + +# -- Version information ------------------------------------------------- + +# The short X.Y version. +release: str = get_version("powerplantmatching") +version: str = ".".join(release.split(".")[:2]) + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx_automodapi.automodapi", # for api reference + "nbsphinx", + "sphinx.ext.imgconverter", # for SVG conversion +] +numpydoc_show_class_members = False + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"] + +source_suffix = { + ".rst": "restructuredtext", + ".txt": "markdown", + ".md": "markdown", +} + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "sphinx_book_theme" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + "repository_url": "https://github.com/pypsa/powerplantmatching", + "use_repository_button": True, + "show_navbar_depth": 1, +} + + +# These folders are copied to the documentation's HTML output +# html_static_path = ["_static"] + +# These paths are either relative to html_static_path +# or fully qualified paths (eg. https://...) +# html_css_files = ["theme_overrides.css"] + + +# -- Options for nbsphinx ------------------------------------------------- +nbsphinx_execute = "never" diff --git a/doc/contributing.rst b/doc/contributing.rst new file mode 100644 index 00000000..5615fe2c --- /dev/null +++ b/doc/contributing.rst @@ -0,0 +1,143 @@ +============ +Contributing +============ + +We welcome anyone interested in contributing to this project, +be it with new ideas, suggestions, by filing bug reports or +contributing code. + +You are invited to submit pull requests / issues to our +`Github repository `_. + +Therefore, we encourage to install ``powerplantmatching`` together with packages used for developing: + +.. code:: bash + + pip install powerplantmatching[dev] + + +This will also install the ``pre-commit`` package which checks that new changes are aligned with the guidelines. +To automatically activate ``pre-commit`` on every ``git commit``, run ``pre-commit install``. +To manually run it, execute ``pre-commit run --all``. + +To double-check that your code is working, we welcome you to write a unit test. Run all tests with + +.. code:: bash + + pytest + + + +Integrating new Data-Sources +---------------------------- + +Let’s say you have a new dataset “FOO.csv” which you want to combine +with the other data bases. Follow these steps to properly integrate it. +Please, before starting, make sure that you’ve installed +``powerplantmatching`` from your downloaded local repository (link). + +1. Look where powerplantmatching stores all data files + + .. code:: python + + import powerplantmatching as pm + pm.core.package_config['data_dir'] + +2. Store FOO.csv in this directory under the subfolder ``data/in``. So + on Linux machines the total path under which you store your data file + would be: + ``/home//.local/share/powerplantmatching/data/in/FOO.csv`` + +3. Look where powerplantmatching looks for a custom configuration file + + .. code:: python + + pm.core.package_config["custom_config"] + + + If this file does not yet exist on your machine, download the + `standard + configuration `__ + and store it under the given path as + ``.powerplantmatching_config.yaml``. + +4. Open the yaml file and add a new entry under the section + ``#data config``. The new entry should look like this + + .. code:: yaml + + FOO: + reliability_score: 4 + fn: FOO.csv + + The ``reliability_score`` indicates the reliability of your data, choose + a number between 1 (low quality data) and 7 (high quality data). If + the data is openly available, you can add an ``url`` argument linking + directly to the .csv file, which will enable automatic downloading. + + Add the name of the new entry to the ``matching_sources`` in your + yaml file like shown below + + .. code:: yaml + + #matching config + matching_sources: + ... + - OPSD + - FOO + +5. Add a function ``FOO()`` to the data.py in the powerplantmatching + source code. You find the file in your local repository under + ``powerplantmatching/data.py``. The function should be structured + like this: + + .. code:: python + + def FOO(raw=False, config=None): + """ + Importer for the FOO database. + + + Parameters + ---------- + raw : Boolean, default False + Whether to return the original dataset + config : dict, default None + Add custom specific configuration, + e.g. powerplantmatching.config.get_config(target_countries='Italy'), + defaults to powerplantmatching.config.get_config() + """ + + config = get_config() if config is None else config + df = pd.read_csv(get_raw_file("FOO")) + if raw: + return foo + df = (df + .rename(columns){'Latitude': 'lat', + 'Longitude': 'lon'}) + .loc[lambda df: df.Country.isin(config['target_countries'])] + .pipe(set_column_name, 'FOO') + ) + + return df + + Note that the code given after ``df =`` is just a placeholder for anything necessary to turn the raw data into the standardized format. You should ensure that the data gets the appropriate column names and that any attributes are in the correct format (all of the standard labels can be found in the yaml or by ``pm.get_config()[‘target_x’]`` + when replacing x by``\ columns, countries, fueltypes, sets or technologies`. + +6. Make sure the FOO entry is given in the configuration + + .. code:: python + + pm.get_config() + + and load the file + + .. code:: python + + pm.data.FOO() + +7. If everything works fine, you can run the whole matching process with + + .. code:: python + + pm.powerplants(update=True) diff --git a/doc/custom-config.rst b/doc/custom-config.rst new file mode 100644 index 00000000..6e8ef823 --- /dev/null +++ b/doc/custom-config.rst @@ -0,0 +1,39 @@ +Make your own configuration +--------------------------- + +You have the option to easily manipulate the resulting data modifying +the global configuration. Just save the `config.yaml +file `__ +as **~/.powerplantmatching_config.yaml** manually or for linux users + +.. code:: bash + + wget -O ~/.powerplantmatching_config.yaml https://raw.githubusercontent.com/PyPSA/powerplantmatching/master/powerplantmatching/package_data/config.yaml + +and change the **.powerplantmaching_config.yaml** file according to your +wishes. Thereby you can + +- determine the global set of **countries** and **fueltypes** + +- determine which data sources to combine and which data sources should + completely be contained in the final dataset + +- individually filter data sources via + `pandas.DataFrame.query `__ + statements set as an argument of data source name. See the default + `config.yaml + file `__ + as an example + +Optionally you can: + +- add your ENTSOE security token to the + **.powerplantmaching_config.yaml** file. To enable updating the + ENTSOE data by yourself. The token can be obtained by following + section 2 of the `RESTful API + documentation `__ + of the ENTSOE-E Transparency platform. + +- add your Google API key to the config.yaml file to enable geoparsing. + The key can be obtained by following the + `instructions `__. diff --git a/docs/examples/example.ipynb b/doc/example.ipynb similarity index 100% rename from docs/examples/example.ipynb rename to doc/example.ipynb diff --git a/doc/getting-started.rst b/doc/getting-started.rst new file mode 100644 index 00000000..42a84c57 --- /dev/null +++ b/doc/getting-started.rst @@ -0,0 +1,40 @@ +Get the Data +------------ + +In order to directly load the already build data into a pandas dataframe +just call + +.. code:: python + + import powerplantmatching as pm + pm.powerplants(from_url=True) + +which will parse and store the `actual dataset of powerplants of this +repository `__. +Setting ``from_url=False`` (default) will load all the necessary data +files and combine them. Note that this might take some minutes. + +The resulting dataset compared with the capacity statistics provided by +the `ENTSOE +SO&AF `__: + +.. figure:: https://raw.githubusercontent.com/PyPSA/powerplantmatching/master/matching_analysis/factor_plot_Matched%20Data.png + :alt: Capacity statistics comparison + + +The merged data is also available as a bigger dataset which provides the original data entries + +.. code:: python + + pm.powerplants(reduced=False) + +It links the entries of the matched power plants and lists all the related +properties given by the different data-sources. + +.. note:: + + Calling the function with ``reduced=False`` will trigger the matching process which requires **Java** to be installed on your system. Please ensure that you have Java installed and that it is in your system's PATH. + + You can download and install Java from the official website: https://www.java.com/en/download/ + + If Java is not installed or not in your system's PATH, you may encounter runtime errors. diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 00000000..5ddcf4df --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,191 @@ +.. powerplantmatching documentation master file, created by + sphinx-quickstart on Tue Jul 20 21:33:24 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to powerplantmatching's documentation! +============================================== + +.. image:: https://img.shields.io/pypi/v/powerplantmatching.svg + :target: https://pypi.org/project/powerplantmatching/ + :alt: pypi + +.. image:: https://img.shields.io/conda/vn/conda-forge/powerplantmatching.svg + :target: https://anaconda.org/conda-forge/powerplantmatching + :alt: conda + +.. image:: https://img.shields.io/python/required-version-toml?tomlFilePath=https%3A%2F%2Fraw.githubusercontent.com%2FPyPSA%2Fpowerplantmatching%2Fmaster%2Fpyproject.toml + :alt: pythonversion + +.. image:: https://github.com/PyPSA/powerplantmatching/actions/workflows/test.yml/badge.svg + :target: https://github.com/PyPSA/powerplantmatching/actions/workflows/test.yml + :alt: Tests + +.. image:: https://readthedocs.org/projects/powerplantmatching/badge/?version=latest + :target: https://powerplantmatching.readthedocs.io/en/latest/ + :alt: doc + +.. image:: https://results.pre-commit.ci/badge/github/PyPSA/powerplantmatching/master.svg + :target: https://results.pre-commit.ci/latest/github/PyPSA/powerplantmatching/master + :alt: pre-commit.ci status + +.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json + :target: https://github.com/astral-sh/ruff + :alt: Ruff + +.. image:: https://img.shields.io/pypi/l/powerplantmatching.svg + :alt: LICENSE + +.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3358985.svg + :target: https://zenodo.org/record/3358985#.XUReFPxS_MU + :alt: DOI + +.. image:: https://img.shields.io/stackexchange/stackoverflow/t/pypsa + :target: https://stackoverflow.com/questions/tagged/pypsa + :alt: Stack Exchange questions + +A toolset for cleaning, standardizing and combining multiple power plant +databases. + +This package provides ready-to-use power plant data for the European +power system. Starting from openly available power plant datasets, the +package cleans, standardizes and merges the input data to create a new +combining dataset, which includes all the important information. The +package allows to easily update the combined data as soon as new input +datasets are released. + +.. figure:: powerplants.png + :alt: Map of power plants in Europe + + Map of power plants in Europe + +powerplantmatching was initially developed by the `Renewable Energy +Group `__ +at `FIAS `__ to build power plant data +inputs to `PyPSA `__-based models for carrying +out simulations for the `CoNDyNet project `__, +financed by the `German Federal Ministry for Education and Research +(BMBF) `__ as part of the `Stromnetze Research +Initiative `__. + +Main Features +~~~~~~~~~~~~~~ + +- clean and standardize power plant data sets +- aggregate power plants units which belong to the same plant +- compare and combine different data sets +- create lookups and give statistical insight to power plant goodness +- provide cleaned data from different sources +- choose between gros/net capacity +- provide an already merged data set of six different data-sources +- scale the power plant capacities in order to match country specific + statistics about total power plant capacities +- visualize the data +- export your powerplant data to a + `PyPSA `__ or + `TIMES `__ + model + + +Installation +------------ + +Install the package via pip + +.. code:: bash + + pip install powerplantmatching + +or conda + +.. code:: bash + + conda install -c conda-forge powerplantmatching + + +Citing powerplantmatching +------------------------- + +If you want to cite powerplantmatching, use the following paper + +- F. Gotzens, H. Heinrichs, J. Hörsch, and F. Hofmann, `Performing + energy modelling exercises in a transparent way - The issue of data + quality in power plant + databases `__, + Energy Strategy Reviews, vol. 23, pp. 1–12, Jan. 2019. + +with bibtex + +:: + + @article{gotzens_performing_2019, + title = {Performing energy modelling exercises in a transparent way - {The} issue of data quality in power plant databases}, + volume = {23}, + issn = {2211467X}, + url = {https://linkinghub.elsevier.com/retrieve/pii/S2211467X18301056}, + doi = {10.1016/j.esr.2018.11.004}, + language = {en}, + urldate = {2018-12-03}, + journal = {Energy Strategy Reviews}, + author = {Gotzens, Fabian and Heinrichs, Heidi and Hörsch, Jonas and Hofmann, Fabian}, + month = jan, + year = {2019}, + pages = {1--12} + } + +and/or the current release stored on Zenodo with a release-specific DOI: + +|image1| + +Acknowledgements +---------------- + +The development of powerplantmatching was helped considerably by +in-depth discussions and exchanges of ideas and code with + +- Tom Brown from Karlsruhe Institute for Technology +- Chris Davis from University of Groningen and +- Johannes Friedrich, Roman Hennig and Colin McCormick of the World + Resources Institute + + + +License +------- + +Copyright 2018-2024 Fabian Gotzens (FZ Jülich), Jonas Hörsch (KIT), Fabian Hofmann (FIAS) +Copyright 2025- The powerplantmatching contributors. + +powerplantmatching is released as free software, mostly under the `MIT license `__, +with some dependent package binaries being licensed under `Apache 2.0 license `__. +The data output `powerplants.csv` is released under the `CC BY 4.0 license `__. + + +.. toctree:: + :hidden: + :maxdepth: 2 + :caption: Getting Started + + getting-started + basics + example.ipynb + custom-config + +.. toctree:: + :hidden: + :maxdepth: 2 + :caption: API Reference + + api-data + api-utils + api-core + + +.. toctree:: + :hidden: + :maxdepth: 2 + :caption: References + + release-notes + contributing + support diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 00000000..2119f510 --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/assets/images/powerplants.png b/doc/powerplants.png similarity index 100% rename from docs/assets/images/powerplants.png rename to doc/powerplants.png diff --git a/doc/release-notes.rst b/doc/release-notes.rst new file mode 100644 index 00000000..0430a104 --- /dev/null +++ b/doc/release-notes.rst @@ -0,0 +1,373 @@ +Release Notes +============== + +Upcoming Version +---------------- + +* Update Marktstammdatenregister data for Germany from `open-MaStR (February 25, 2025) `__. + +* Drop support for Python 3.9, add support for Python 3.13. Minimum required Python version is now 3.10. + + +`v0.7.1 `__ (30th January 2024) +================================================================================================= + +Bug fixes +--------- + +* Patch for a bug in matching caused by faulty names for BNA hydro powerplants + in the OPSD_EU input dataset. (https://github.com/PyPSA/powerplantmatching/pull/217) + +* Minor manual correction for CCGT powerplant. (https://github.com/PyPSA/powerplantmatching/pull/221) + +`v0.7.0 `__ (23rd January 2025) +================================================================================================= + +Features +-------- + +* Add "Marktstammdatenregister" (MaStR) data source for Germany (https://github.com/PyPSA/powerplantmatching/pull/165). + + +`v0.6.1 `__ (15th January 2025) +================================================================================================= + +Bug fixes +--------- + +* Remove duplicate conventional power plants coming from different OPSD input files (https://github.com/PyPSA/powerplantmatching/pull/213). + +`v0.6.0 `__ (18th September 2024) +================================================================================================= + +Features +-------- + +* BREAKING: Split "Bioenergy" into "Biogas" and "Solid Biomass" (https://github.com/PyPSA/powerplantmatching/pull/201). + +Bug fixes +--------- + +* Bugfix: Consistently rename countries, affecting power plants in Czech Republic and + Moldova (https://github.com/PyPSA/powerplantmatching/pull/204). + + +Version 0.5.19 (16.09.2024) +=========================== + +* Bugfix: Fix retrieval of Global Energy Monitor data. + + +Version 0.5.18 (30.08.2024) +=========================== + +* Treat Kosovo as separate country and do not assign powerplants to Serbia. + +* Correctly reference North Macedonia and Moldova for ``pycountry`` matching. + + +Version 0.5.17 (23.08.2024) +=========================== + +* Update Global Energy Monitor data to latest versions as of 2024-08-23. + +* Adjusted reliability scores in favour of GEM data. + + +Version 0.5.16 (31.07.2024) +=========================== + +* Dependency updates: Ensure compatibility with pandas 2.0, numpy 2.0 and cartopy. + +* Bugfix: Fix retrieving data without aggregation (https://github.com/PyPSA/powerplantmatching/pull/180) + + +Version 0.5.15 (12.05.2024) +=========================== + +* Updated the JRC hydro power plant database to the latest version. + (https://github.com/PyPSA/powerplantmatching/pull/123) + +* Corrected capacity of some hydro powerplants in the United Kingdom contained + in the ENTSOE data based on manual search. The error originated from the use + of the value 999 in the capacity column to represent NaN values. The changes + accumulate to a reduction of approximately 5 GW in capacity. + (https://github.com/PyPSA/powerplantmatching/pull/141) + +* Remove use of ``distutils`` package for compatibility with python 3.12. + (https://github.com/PyPSA/powerplantmatching/pull/159) + +Version 0.5.14 (30.04.2024) +=========================== + +* Bugfix to ensure compatibility with python 3.12. + +Version 0.5.13 (25.04.2024) +=========================== + +* Update IRENASTAT to include data up to 2023 (``ppm.data.IRENASTAT()``). + +Version 0.5.12 (07.04.2024) +=========================== + +* fix the filtering of GCPT/GEM coal database by Status. +* add technology renaming for GWPT/GEM wind dataset. +* improve performances of GEM data processing. + +Version 0.5.11 (05.02.2024) +=========================== + +* The default configuration no longer filters out retired power plants. This filtering should be done by the user afterwards. + +Version 0.5.10 (30.01.2024) +=========================== + +* fix deprecation warnings for new pandas version for inplace operations +* fix bus mapping in ``export`` module + +Version 0.5.9 (16.01.2024) +=========================== + +* fix deprecation warnings for new pandas version +* lower GEM reliability score to fix DateIn and DateOut for some powerplants + + +Version 0.5.8 (30.10.2023) +------------------------- + +**New Features** + +* Datasets from the Global Energy Monitor are now combined into one dataset called `GEM`. This is per default used in the matching process. + +* Updates the following Global Energy Monitor data according to latest May 2023 release: + * `GSPT`, solar power plant + * `GWPT`, wind power plant +* Changing Global Energy Monitor dataset name to -latest to avoid data update PRs in powerplantmatching + but rather encourage updates in `Global Energy Monitor data repo `__. + +**Fixes** + +* In the aggregation and merge process, the `DateIn` and `DateOut` columns are now aggregated by taking the minimum and maximum value respectively. + + +Version 0.5.7 (30.05.2023) +------------------------- + + +**Bug fixes** + +* Replace unstable FIAS and OPSD download links by new provider +* Replace ValueError when no matches found to logged warning. + +**Other Changes** + +* adds the following new Global Energy Monitor data including: + * `GBPT`, bioenergy power plant + * `GGPT`, gas power plant update + * `GCPT`, coal power plant + * `GGTPT`, geothermal power plant + * `GNPT`, nuclear power plant + * `GSPT`, solar power plant + * `GWPT`, wind power plant + * `GHPT`, hydro power plant + + +Version 0.5.6 (13.02.2023) +------------------------- + + +**Bug fixes** + +* Set `DateIn` column as a float +* Remove special characters from hash path for custom configs +* Add conversion of country name into GEO and GPD data sources + +**Other Changes** + +* Revise table in documentation and update links to powerplant database + + +Version 0.5.5 (21.12.2022) +------------------------- + + +**New Features** + +* New `EXTERNAL_DATABASE` interface to integrate additional custom data of raw data matching the powerplantmatching format. +* The example notebook was updated to the new version of the package. +* Rename the `matching_analysis` directory to `analysis`. +* Harmonize the analysis scripts with the `pm.powerlants(update=True)` functionality. + +**Bug fixes** + +* Fix `GGPT `_ (formerly `GEM_GGPT`) interface. +* Fix missing column names in dataframes +* Spanish hydro stores with an capacity larger than 50GWh in the `JRC` data base are assumed to be reservoirs even if stated differently. +* add geolocations for Italian geothermal units and adjust filters in config.yaml to bring geothermal to powerplants.csv + +**Other Changes** + +* Rename the `matching_analysis` directory to `analysis`. +* Rename `GEM_GGPT` to `GGPT` and add `GEM_GGPT` as an deprecated alias. +* Rename `matched_data` to `powerplants` in `collection.py` and add `matched_data` as an deprecated alias. +* Fueltype "Other" was remove from the Capacity_Stats function. + + +Version 0.5.4 (02.08.2022) +------------------------- + +**New Features** + +* The `GEM_GGPT `_ data containing global gas power plant data for all countries is now available. + +**Bug fixes** + +* Fix capacity-weighted mean calculation of efficiencies. The previous implementation lead to underestimated efficiencies for power plants consisting of multiple power units. + +Version 0.5.3 (08.04.2022) +------------------------- + +* Bug fix for renewable power plants from OPSD + +Version 0.5.2 (07.04.2022) +------------------------- + +* The overall config setting was fine-tuned in order to improve the matching results. +* New scripts were added to the folder `matching_analysis` + +Version 0.5.1 (04.04.2022) +------------------------- + +**New Features** + +* The `IRENASTATS `_ data is now available containing powerplant capacities for all countries from 2000 to 2020. + +**Bug fix** + +* The url of the ``powerplants`` function was fixed. + + +**Other** + +* The removal of the column ``DateMothBall`` was caught up on. +* The manual corrections were reactivated. +* Improved country code and name conversion by using ``country_converter``. + + +Version 0.5 (04.04.2022) +------------------------ + + +This release contains many breaking changes. Due to time-constraints we cannot ensure a smooth transition to the new release. If you are using a custom config file (e.g. ``~/powerplantmatching_config.yaml``) please be aware of the following config changes: + +**Configuration Changes** + +* The custom configuration now only updates the package default configuration, which makes the compatibility of configuration updates much easier. So, instead of replacing the whole package configuration (the default config provided by powerplantmatching), the new purpose of the custom config is to adjust individual values. So, please make sure to only add keys to the custom config which you want to change in comparison to the default config. +* The following sections of the configuration file ``~/powerplantmatching_config.yaml`` changed: + * the ``target_fueltypes`` section is now mapping the representative fueltypes to the regular expressions that are used in order to determine them. + * the ``target_technologies`` section is now mapping the representative technologies to the regular expressions that are used in order to determine them. + * the ``target_set`` section is now mapping the representative sets to the regular expressions that are used in order to determine them. + * a section ``clean_name`` was added. This section contains the regular expressions and lists of words that are used to clean the names of the plants. +In order to ensure compatibility with the new code, please delete these sections in your custom config. + +**Deprecations** + +* The ``CARMA`` dataset was deprecated as the data is not publicly available anymore. +* The ``IWPDCY`` dataset was deprecated in the favor of the `JRC` data. +* The ``WEPP`` dataset was deprecated due to restrictive license. +* The ``UBA`` dataset was deprecated in the favor of the ``OPSD`` data. +* The ``BENTZA`` dataset was deprecated in the favor of the ``OPSD`` data. +* The ``IRENA_stats`` dataset was deprecated as the data is not publicly available anymore. +* The following functions were deprecated and will be removed in ``v0.6``: + * ``powerplantmatching.export.to_TIMES`` + * ``powerplantmatching.export.store_open_dataset`` + * ``powerplantmatching.export.fueltype_to_abbrev` ` + * ``powerplantmatching.heuristics.set_denmark_region_id`` + * ``powerplantmatching.heuristics.remove_oversea_areas`` + * ``powerplantmatching.heuristics.set_known_retire_years`` +* The argument ``extendby_kwargs`` in the function ``powerplantmatching.collection.matched_data`` was deprecated in the favor of ``extend_by_kwargs``. + + +**New Features** + +* The `BEYOND COAL `_ data is now available as an data source. +* A new dataset ``WIKIPEDIA`` on nuclear powerplants in europe from wikipedia was added. +* The ``GEO`` dataset returns powerplant blocks instead of whole plants. +* All scripts were aligned with the ``black`` coding style. +* A documentation on readthedocs was added. +* The config has now a key `main_query` which is applied to all datasets. +* A CI was added. +* A new function ``powerplantmatching.heuristics.isin`` was added. It checks which data entries of a non-matched dataset is included in a matched dataset. + +**Breaking Code Changes:** + +* The argument `rawDE` and `rawEU` in ``powerplantmatching.data.OPSD`` was deprecated in the favor of `raw`. If ``True`` the function returns a dictionary with the raw datasets. +* All keyword arguments of the data functions in ``powerplantmatching.data`` were sorted according to ``raw``, ``update``, ``config``. This lead to some breaking changes of the arguments order. +* The Fueltype `Other` was replaced by NaN. +* The `GEO` data now returns a dataset containing power plant units. +* The ``ESE`` dataset was removed due the hosting website taken down. +* The argument ``subsume_uncommon_fueltypes_to_other`` in ``powerplantmatching.collection.matched_data`` was removed. +* The function ``powerplantmatching.cleaning.aggregate_units`` does not support the arguments `use_saved_aggregation` and `save_aggregation` anymore due to it's insecure behavior. +* The function ``powerplantmatching.matching.compare_two_datasets`` does not support the arguments `use_saved_matches` anymore due to it's insecure behavior. + + +Version 0.4.6 (25.11.2020) +=========================== + +| Triggered by the ongoing phase-outs of especially nuclear, coal and + lignite plants in many countries, we acknowledge that time-related + data of power stations and their single blocks becomes increasingly + important. +| Therefore, we decided to - adapt the columns: - rename + ``YearCommissioned`` to ``DateIn`` (reflects when a station/block had + initially started operation) - rename ``Retrofit`` to ``DateRetrofit`` + (reflects when a station/block has been retrofitted) - add + ``DateMothball`` (reflects when a station/block has been mothballed) - + add ``DateOut`` (reflects when a station/block has been finally + decommissioned) +| **Please note:** Currently, these columns only contain the year, but + we aim in future to provide exact dates (i.e. including day and month) + wherever possible. + +| Further changes: - new target_columns: - add ``EIC`` (the European + *Energy Identification Code*) - add ``StorageCapacity_MWh`` - update + `JRC Hydro + Database `__ + to v5 and add quick workaround so that pm can deal with non-unique + identifiers - replace deprecated by current pandas functions - custom + configuration and package configuration are now merged when calling + ``get_config()`` (values in the custom configuration are prioritized) + - fix retrieving BNETZA data +| - export.py has now a function for mapping bus coordinates to the + power plant list + +Version 0.4.1 (02.08.2019) +=========================== + +Data structure +~~~~~~~~~~~~~~ + +- abolish git lfs in the favour of direct url parsing +- store data in user folders + + - Linux ``~/.local/share/powerplantmatching`` + - Windows ``C:\Users\\AppData\Roaming\powerplantmatching`` + +- move necessary files to package_data in powerplantmatching folder + (such as duke binaries, xml files etc.) +- include `JRC Hydro + Database `__ + +Code +~~~~ + +- get rid of mutual module imports +- speed up grouping (cleaning.py, matching.py) +- revise/rewrite code in data.py +- enable switch for matching powerplants of the same country only (is + now default, speeds up the matching and aggregation process + significantly) +- boil down plot.py which caused long import times +- get rid of config.py in the favour of core.py and accessor.py +- drop deprecated functions in collection.py which now only includes + collect() and matched_data() diff --git a/doc/support.rst b/doc/support.rst new file mode 100644 index 00000000..11ab27a5 --- /dev/null +++ b/doc/support.rst @@ -0,0 +1,13 @@ + +.. + SPDX-FileCopyrightText: 2019-2023 The Powerplantmatching Authors + SPDX-License-Identifier: CC-BY-4.0 +####################### +Support +####################### + +* In case of code-related **questions**, please post on `stack overflow `_. +* For non-programming related and more general questions please refer to the `pypsa mailing list `_. +* To **discuss** with other PyPSA and powerplantmatching users, organise projects, share news, and get in touch with the community you can use the `discord server `_. +* For **bugs and feature requests**, please use the `issue tracker `_. +* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to :ref:`contributing`. diff --git a/docs/api-core.md b/docs/api-core.md deleted file mode 100644 index 8f13f3f7..00000000 --- a/docs/api-core.md +++ /dev/null @@ -1,13 +0,0 @@ - - -# Core Modules - -::: powerplantmatching.core - -::: powerplantmatching.duke - -::: powerplantmatching.accessor diff --git a/docs/api-data.md b/docs/api-data.md deleted file mode 100644 index ec0ec81f..00000000 --- a/docs/api-data.md +++ /dev/null @@ -1,15 +0,0 @@ - - -# Data Processing Modules - -::: powerplantmatching.data - -::: powerplantmatching.cleaning - -::: powerplantmatching.matching - -::: powerplantmatching.collection diff --git a/docs/api-utils.md b/docs/api-utils.md deleted file mode 100644 index 72998df8..00000000 --- a/docs/api-utils.md +++ /dev/null @@ -1,15 +0,0 @@ - - -# Utility Modules - -::: powerplantmatching.utils - -::: powerplantmatching.export - -::: powerplantmatching.heuristics - -::: powerplantmatching.plot diff --git a/docs/basics.md b/docs/basics.md deleted file mode 100644 index c3f3b0c5..00000000 --- a/docs/basics.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# General Structure - -The dataset combines the data of all the data sources listed in [Data-Sources](#data-sources) and provides the following information: - -- **Power plant name** - claim of each database -- **Fueltype** - {Solid Biomass, Biogas, Geothermal, Hard Coal, Hydro, Lignite, Nuclear, Natural Gas, Oil, Solar, Wind, Other} -- **Technology** - {CCGT, OCGT, Steam Turbine, Combustion Engine, Run-Of-River, Pumped Storage, Reservoir} -- **Set** - {Power Plant (PP), Combined Heat and Power (CHP), Storages (Stores)} -- **Capacity** - [MW] -- **Duration** - Maximum state of charge capacity in terms of hours at full output capacity -- **Dam Information** - Dam volume [Mm^3] and Dam Height [m] -- **Geo-position** - Latitude, Longitude -- **Country** - EU-27 + CH + NO (+ UK) minus Cyprus and Malta -- **YearCommissioned** - Commmisioning year of the powerplant -- **RetroFit** - Year of last retrofit -- **projectID** - Immutable identifier of the power plant - -All data files of the package will be stored in the folder given by `pm.core.package_config['data_dir']` - -## Data Sources - -- OPSD - [Open Power System Data](http://data.open-power-system-data.org/) publish their [data](http://data.open-power-system-data.org/conventional_power_plants/) under a free license -- GEO - [Global Energy Observatory](http://globalenergyobservatory.org/), the data is not directly available on the website, but can be obtained from an sqlite diff --git a/docs/citing.md b/docs/citing.md deleted file mode 100644 index 54f22cec..00000000 --- a/docs/citing.md +++ /dev/null @@ -1,30 +0,0 @@ - - -# Citing powerplantmatching - -If you use `powerplantmatching` in your work, please consider citing the following paper: - -> F. Gotzens, H. Heinrichs, J. Hörsch, and F. Hofmann, "Performing energy modelling exercises in a transparent way - The issue of data quality in power plant databases", Energy Strategy Reviews, vol. 23, pp. 1–12, Jan. 2019, doi:[10.1016/j.esr.2018.11.004](https://www.sciencedirect.com/science/article/pii/S2211467X18301056). - -## BibTeX entry - -```bibtex -@article{gotzens_performing_2019, - title = {Performing energy modelling exercises in a transparent way - {The} issue of data quality in power plant databases}, - volume = {23}, - issn = {2211467X}, - url = {https://linkinghub.elsevier.com/retrieve/pii/S2211467X18301056}, - doi = {10.1016/j.esr.2018.11.004}, - language = {en}, - urldate = {2018-12-03}, - journal = {Energy Strategy Reviews}, - author = {Gotzens, Fabian and Heinrichs, Heidi and Hörsch, Jonas and Hofmann, Fabian}, - month = jan, - year = {2019}, - pages = {1--12} -} -``` diff --git a/docs/contributors.md b/docs/contributors.md deleted file mode 100644 index eb964292..00000000 --- a/docs/contributors.md +++ /dev/null @@ -1,99 +0,0 @@ - - -# Contributors - -The **original authors** were: - -- Fabian Gotzens (:octicons-mark-github-16: [@fgotzens](https://github.com/fgotzens)) -- Heidi Heinrichs (:house: [website](https://www.fz-juelich.de/profile/heinrichs_h)) -- Jonas Hörsch (:octicons-mark-github-16: [@coroa](https://github.com/coroa)) -- Fabian Hofmann (:octicons-mark-github-16: [@FabianHofmann](https://github.com/FabianHofmann)) - -The development of `powerplantmatching` was **helped considerably** by in-depth discussions and exchanges of ideas and code with: - -- Tom Brown (:octicons-mark-github-16: [@nworbmot](https://github.com/nworbmot)) -- Chris Davis (University of Groningen) -- Johannes Friedrich, Roman Hennig, Colin McCormick (World Resources Institute) - -**All contributors:** - -As of 2025-08-06, the following people have contributed to `powerplantmatching`: - -- Carlos Gaete (:octicons-mark-github-16: [@cdgaete](https://github.com/cdgaete)) -- Daniel Rüdt (:octicons-mark-github-16: [@daniel-rdt](https://github.com/daniel-rdt)) -- Davide Fioriti (:octicons-mark-github-16: [@davide-f](https://github.com/davide-f)) -- enioarz (:octicons-mark-github-16: [@enioarz](https://github.com/enioarz)) -- Fabian Hofmann (:octicons-mark-github-16: [@FabianHofmann](https://github.com/FabianHofmann)) -- Fabian Neumann (:octicons-mark-github-16: [@fneum](https://github.com/fneum)) -- Febin Kachirayil (:octicons-mark-github-16: [@FebinKa](https://github.com/FebinKa)) -- Fabian Gotzens (:octicons-mark-github-16: [@fgotzens](https://github.com/fgotzens)) -- Heinz-Alexander Fuetterer (:octicons-mark-github-16: [@afuetterer](https://github.com/afuetterer)) -- Iegor Riepin (:octicons-mark-github-16: [@Irieo](https://github.com/Irieo)) -- jensch-dlr (:octicons-mark-github-16: [@jensch-dlr](https://github.com/jensch-dlr)) -- Johannes HAMPP (:octicons-mark-github-16: [@euronion](https://github.com/euronion)) -- Jonas Hörsch (:octicons-mark-github-16: [@coroa](https://github.com/coroa)) -- Leon Schumm (:octicons-mark-github-16: [@energyLS](https://github.com/energyLS)) -- Lukas Franken (:octicons-mark-github-16: [@LukasFrankenQ](https://github.com/LukasFrankenQ)) -- Lukas Trippe (:octicons-mark-github-16: [@lkstrp](https://github.com/lkstrp)) -- Markus Groissböck (:octicons-mark-github-16: [@gincrement](https://github.com/gincrement)) -- Martha Frysztacki (:octicons-mark-github-16: [@martacki](https://github.com/martacki)) -- Martin Hjelmeland (:octicons-mark-github-16: [@martinhjel](https://github.com/martinhjel)) -- Max Parzen (:octicons-mark-github-16: [@pz-max](https://github.com/pz-max)) -- rbaard1 (:octicons-mark-github-16: [@rbaard1](https://github.com/rbaard1)) -- Thomas Kouroughli (:octicons-mark-github-16: [@Tomkourou](https://github.com/Tomkourou)) - - - -Up-to-date statistics and a complete list of contributors to `powerplantmatching` can be found under [GitHub Insights](https://github.com/PyPSA/powerplantmatching/graphs/contributors). diff --git a/docs/custom-config.md b/docs/custom-config.md deleted file mode 100644 index c6fe48ac..00000000 --- a/docs/custom-config.md +++ /dev/null @@ -1,24 +0,0 @@ - - -# Make your own configuration - -You have the option to easily manipulate the resulting data by modifying the global configuration. Just save the [config.yaml file](https://github.com/PyPSA/powerplantmatching/blob/master/powerplantmatching/package_data/config.yaml) as `~/.powerplantmatching_config.yaml` manually or for Linux users: - -```bash -wget -O ~/.powerplantmatching_config.yaml https://raw.githubusercontent.com/PyPSA/powerplantmatching/master/powerplantmatching/package_data/config.yaml -``` - -and change the `.powerplantmaching_config.yaml` file according to your wishes. Thereby you can: - -- determine the global set of **countries** and **fueltypes** -- determine which data sources to combine and which data sources should completely be contained in the final dataset -- individually filter data sources via [`pandas.DataFrame.query`](http://pandas.pydata.org/pandas-docs/stable/indexing.html#the-query-method) statements set as an argument of data source name. See the default [config.yaml file](https://github.com/PyPSA/powerplantmatching/blob/master/powerplantmatching/package_data/config.yaml) as an example - -Optionally you can: - -- add your ENTSOE security token to the `.powerplantmaching_config.yaml` file. To enable updating the ENTSOE data by yourself. The token can be obtained by following section 2 of the [RESTful API documentation](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html#_authentication_and_authorisation) of the ENTSOE-E Transparency platform. -- add your Google API key to the config.yaml file to enable geoparsing. The key can be obtained by following the [instructions](https://developers.google.com/maps/documentation/geocoding/get-api-key). diff --git a/docs/faq.md b/docs/faq.md deleted file mode 100644 index 7969aef2..00000000 --- a/docs/faq.md +++ /dev/null @@ -1,10 +0,0 @@ - - -# Frequently Asked Questions (FAQ) - -> This section is to be expanded and could host your contribution :tada: -> (TODO) diff --git a/docs/getting-started.md b/docs/getting-started.md deleted file mode 100644 index 061f6939..00000000 --- a/docs/getting-started.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# Getting Started - -This guide will help you get started with Powerplantmatching. - -## Get the Data - -To directly load the already built data into a pandas dataframe, just call: - -```python -import powerplantmatching as pm -pm.powerplants(from_url=True) -``` - -This will parse and store the [actual dataset of powerplants of this repository](https://github.com/PyPSA/powerplantmatching/blob/master/powerplants.csv). -Setting `from_url=False` (default) will load all the necessary data files and combine them. Note that this might take some minutes. - -The resulting dataset compared with the capacity statistics provided by the [ENTSOE SO&AF](https://data.open-power-system-data.org/national_generation_capacity/2019-02-22): - -![Capacity statistics comparison](assets/images/powerplants.png) - -The merged data is also available as a bigger dataset which provides the original data entries: - -```python -pm.powerplants(reduced=False) -``` - -It links the entries of the matched power plants and lists all the related properties given by the different data-sources. - -!!! note - Calling the function with `reduced=False` will trigger the matching process which requires **Java** to be installed on your system. Please ensure that you have Java installed and that it is in your system's PATH. - - You can download and install Java from the official website: https://www.java.com/en/download/ - If Java is not installed or not in your system's PATH, you may encounter runtime errors. diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 2c953141..00000000 --- a/docs/index.md +++ /dev/null @@ -1,49 +0,0 @@ - - -# Welcome to powerplantmatching's documentation! - -![PyPI](https://img.shields.io/pypi/v/powerplantmatching.svg) -[![Conda](https://img.shields.io/conda/vn/conda-forge/powerplantmatching.svg)](https://anaconda.org/conda-forge/powerplantmatching) -![Python Version](https://img.shields.io/python/required-version-toml?tomlFilePath=https%3A%2F%2Fraw.githubusercontent.com%2FPyPSA%2Fpowerplantmatching%2Fmaster%2Fpyproject.toml) -[![Tests](https://github.com/PyPSA/powerplantmatching/actions/workflows/test.yml/badge.svg)](https://github.com/PyPSA/powerplantmatching/actions/workflows/test.yml) -[![Docs](https://readthedocs.org/projects/powerplantmatching/badge/?version=latest)](https://powerplantmatching.readthedocs.io/en/latest/) -[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/PyPSA/powerplantmatching/master.svg)](https://results.pre-commit.ci/latest/github/PyPSA/powerplantmatching/master) -[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) -![LICENSE](https://img.shields.io/pypi/l/powerplantmatching.svg) -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3358985.svg)](https://zenodo.org/record/3358985#.XUReFPxS_MU) - -## Introduction - -A toolset for cleaning, standardizing and combining multiple power plant -databases. - -This package provides ready-to-use power plant data for the European -power system. Starting from openly available power plant datasets, the -package cleans, standardizes and merges the input data to create a new -combining dataset, which includes all the important information. The -package allows to easily update the combined data as soon as new input -datasets are released. - -![Map of power plants in Europe](assets/images/powerplants.png) - -`powerplantmatching` was initially developed by the [Renewable Energy Group](https://fias.uni-frankfurt.de/physics/schramm/complex-renewable-energy-networks/) at [FIAS](https://fias.uni-frankfurt.de/) to build power plant data inputs to [PyPSA](http://www.pypsa.org/)-based models for carrying out simulations for the [CoNDyNet project](http://condynet.de/), financed by the [German Federal Ministry for Education and Research (BMBF)](https://www.bmbf.de/en/) as part of the [Stromnetze Research Initiative](http://forschung-stromnetze.info/projekte/grundlagen-und-konzepte-fuer-effiziente-dezentrale-stromnetze/). - -## Main Features - -- clean and standardize power plant data sets -- aggregate power plants units which belong to the same plant -- compare and combine different data sets -- create lookups and give statistical insight to power plant goodness -- provide cleaned data from different sources -- choose between gros/net capacity -- provide an already merged data set of six different data-sources -- scale the power plant capacities in order to match country specific - statistics about total power plant capacities -- visualize the data -- export your powerplant data to a - - [PyPSA](https://github.com/PyPSA/PyPSA), or - - [TIMES](https://iea-etsap.org/index.php/etsap-tools/model-generators/times) model diff --git a/docs/installation.md b/docs/installation.md deleted file mode 100644 index 5674713e..00000000 --- a/docs/installation.md +++ /dev/null @@ -1,21 +0,0 @@ - - -# Installation - -This package is available on [PyPI](https://pypi.org/project/powerplantmatching/) and [Conda-Forge](https://anaconda.org/conda-forge/powerplantmatching). - -To install the package via `pip` (or `uv pip`): - -```bash -pip install powerplantmatching -``` - -To install it using `conda` / `mamba`: - -```bash -conda install -c conda-forge powerplantmatching -``` diff --git a/docs/license.md b/docs/license.md deleted file mode 100644 index d88059b1..00000000 --- a/docs/license.md +++ /dev/null @@ -1,19 +0,0 @@ - - -# License - -`powerplantmatching` is released as free software under the [MIT](LICENSES/MIT.txt) license. -The default output data [powerplants.csv](powerplants.csv) generated by the package is released under [CC BY 4.0](LICENSES/CC-BY-4.0.txt). -Parts of the repository may be licensed under different licenses, especially dependent package binaries for `duke` being licensed under [Apache 2.0 license](https://github.com/PyPSA/powerplantmatching/tree/master/LICENSES/Apache-2.0.txt). - -This repository uses the [REUSE](https://reuse.software/) conventions to indicate the licenses that apply to individual files and parts of the repository. -For details on the licenses that apply, see the the header information of the respective files and [REUSE.toml](REUSE.toml) for details. - -Copyright 2018-2024 Fabian Gotzens (FZ Jülich), Jonas Hörsch (KIT), Fabian Hofmann (FIAS) -Copyright 2025- Contributors to powerplantmatching - -You can find a list of contributors in the [contributors page](https://github.com/PyPSA/powerplantmatching/graphs/contributors) and in the [contributors file](../contributors). diff --git a/docs/release-notes.md b/docs/release-notes.md deleted file mode 100644 index e6a13c21..00000000 --- a/docs/release-notes.md +++ /dev/null @@ -1,75 +0,0 @@ - - -# Release Notes - -## Upcoming Version - -* Update Marktstammdatenregister data for Germany from [open-MaStR (February 25, 2025)](https://zenodo.org/records/14783581). -* Drop support for Python 3.9, add support for Python 3.13. Minimum required Python version is now 3.10. -* Restructure documentation and move to use `mkdocs` for a nicer user experience. - -## [v0.7.1](https://github.com/PyPSA/powerplantmatching/releases/tag/v0.7.1) (30th January 2024) - -### Bug fixes - -* Patch for a bug in matching caused by faulty names for BNA hydro powerplants in the OPSD_EU input dataset. ([PR #217](https://github.com/PyPSA/powerplantmatching/pull/217)) -* Minor manual correction for CCGT powerplant. ([PR #221](https://github.com/PyPSA/powerplantmatching/pull/221)) - -## [v0.7.0](https://github.com/PyPSA/powerplantmatching/releases/tag/v0.7.0) (23rd January 2025) - -### Features - -* Add "Marktstammdatenregister" (MaStR) data source for Germany ([PR #165](https://github.com/PyPSA/powerplantmatching/pull/165)). - -## [v0.6.1](https://github.com/PyPSA/powerplantmatching/releases/tag/v0.6.1) (15th January 2025) - -### Bug fixes - -* Remove duplicate conventional power plants coming from different OPSD input files ([PR #213](https://github.com/PyPSA/powerplantmatching/pull/213)). - -## [v0.6.0](https://github.com/PyPSA/powerplantmatching/releases/tag/v0.6.0) (18th September 2024) - -### Features - -* Add support for new power plant types: biomass and geothermal. ([PR #200](https://github.com/PyPSA/powerplantmatching/pull/200)) -* Include additional parameters for hydro power plants. ([PR #205](https://github.com/PyPSA/powerplantmatching/pull/205)) - -### Bug fixes - -* Fix issue with negative capacity values for some plants. ([PR #207](https://github.com/PyPSA/powerplantmatching/pull/207)) -* Corrected data for onshore and offshore wind farms. ([PR #209](https://github.com/PyPSA/powerplantmatching/pull/209)) - -## [v0.5.0](https://github.com/PyPSA/powerplantmatching/releases/tag/v0.5.0) (10th August 2024) - -### Features - -* Introduce new matching algorithm for improved accuracy. ([PR #180](https://github.com/PyPSA/powerplantmatching/pull/180)) -* Allow user-defined priority for input data sources. ([PR #185](https://github.com/PyPSA/powerplantmatching/pull/185)) - -### Bug fixes - -* Resolve issue with duplicate entries in the final output. ([PR #190](https://github.com/PyPSA/powerplantmatching/pull/190)) -* Fix incorrect mapping of some fuel types. ([PR #195](https://github.com/PyPSA/powerplantmatching/pull/195)) - -## [v0.4.0](https://github.com/PyPSA/powerplantmatching/releases/tag/v0.4.0) (5th June 2024) - -### Features - -* Enhanced support for renewable energy sources. ([PR #150](https://github.com/PyPSA/powerplantmatching/pull/150)) -* Added new geographical regions for plant matching. ([PR #155](https://github.com/PyPSA/powerplantmatching/pull/155)) - -### Bug fixes - -* Fixed critical bug causing crashes with certain input data. ([PR #160](https://github.com/PyPSA/powerplantmatching/pull/160)) -* Minor documentation fixes and improvements. ([PR #165](https://github.com/PyPSA/powerplantmatching/pull/165)) - -## [v0.3.0](https://github.com/PyPSA/powerplantmatching/releases/tag/v0.3.0) (1st March 2024) - -### Features - -* Initial release of the Powerplantmatching tool. -* Support for matching power plants based on various input data sources. diff --git a/docs/support.md b/docs/support.md deleted file mode 100644 index d23e73a3..00000000 --- a/docs/support.md +++ /dev/null @@ -1,18 +0,0 @@ - - -# Support - -Please consider the following ways to reach out to the community and the developers: - -* To **discuss** with other `powerplantmatching` users, organise projects, share news, and get in touch with the community you can use the [PyPSA-Earth Discord server](https://discord.gg/AnuJBk23FU). -* For **bugs and feature requests**, please use the [issue tracker](https://github.com/PyPSA/powerplantmatching/issues). -* We welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on [Github](https://github.com/PyPSA/powerplantmatching). - -!!! note - - All questions benefit from being accompanied by a minimal working example (MWE) that illustrates the problem or question, and by sharing which approaches you have already tried. - See the [Troubleshooting](troubleshooting.md) section for more information on how to create an MWE. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md deleted file mode 100644 index 25506ab1..00000000 --- a/docs/troubleshooting.md +++ /dev/null @@ -1,19 +0,0 @@ - - -# Troubleshooting - -> This section is to be expanded and could host your contribution :tada: -> (TODO) - -## Minimum Working Example (MWE) - -For your own debugging and when asking for help, it is very useful to create a Minimum Working Example (MWE). -An MWE is a small example that reproduces the problem you are experiencing. -It should be as small as possible, but still reproduce the problem. -Often, when creating a MWE, you will find the problem yourself, and if not it will be much easier for others to help you. - -For details on what a MWE is and how to create one, see [this blog on how Craft Minimal Bug Reports](https://matthewrocklin.com/minimal-bug-reports). diff --git a/mkdocs.yaml b/mkdocs.yaml deleted file mode 100644 index ed113cd8..00000000 --- a/mkdocs.yaml +++ /dev/null @@ -1,142 +0,0 @@ -# SPDX-FileCopyrightText: 2025 Contributors to powerplantmatching -# -# SPDX-License-Identifier: MIT - -site_name: Documentation -site_url: https://github.com/PyPSA/powerplantmatching -repo_url: https://github.com/PyPSA/powerplantmatching -repo_name: PyPSA/powerplantmatching -edit_uri: edit/master/docs/ - -nav: - - Home: - - Home: index.md - - Usage: - - Installation: installation.md - - Getting Started: getting-started.md - - Basics: basics.md - - Custom Config: custom-config.md - - About: - - Release Notes: release-notes.md - - Contributors: contributors.md - - Citing: citing.md - - License: license.md - - Support: - - FAQ: faq.md - - Support: support.md - - Troubleshooting: troubleshooting.md - - Example: - - Example: examples/example.ipynb - - API reference: - - Core modules: api-core.md - - Data processing modules: api-data.md - - Utility modules: api-utils.md - -# Theme settings -theme: - name: material - font: - text: Overpass - features: - - navigation.tabs - - navigation.tabs.sticky - - navigation.sections - - navigation.tracking # Anchor tracking - - navigation.indexes # Section indexes - - navigation.path - - navigation.top # Back to top button - # - navigation.foot - - header.autohide - - content.action.edit - - content.action.view - - toc.follow - - search - - search.suggest - - search.highlight - - search.share - - announce.dismiss - - content.code.copy - - content.code.select - -plugins: - - search - - autolinks - - table-reader - - mkdocs-video - - mkdocs-jupyter - # Docstring generation - - mkdocstrings: - enabled: true - default_handler: python - handlers: - python: - options: - docstring_style: numpy - docstring_section_style: list - filters: - - "!logger" - - "!^_[^_]" - heading_level: 2 - merge_init_into_class: true - parameter_headings: false - separate_signature: true - show_root_heading: true - show_root_full_path: false - show_signature_annotations: true - show_source: false - show_symbol_type_heading: true - show_symbol_type_toc: true - signature_crossrefs: true - summary: true - show_if_no_docstring: true - show_inheritance_diagram: true - -markdown_extensions: - # Callouts/ Admonitions - - admonition - - pymdownx.details - - pymdownx.superfences: - custom_fences: - - name: mermaid - class: mermaid - format: !!python/name:pymdownx.superfences.fence_code_format - - pymdownx.arithmatex: - generic: true - # Footnotes - - footnotes - # Tabbed content - - pymdownx.tabbed: - alternate_style: true # Abbreviations - - pymdownx.snippets: - auto_append: - - includes/abbreviations.md - - abbr - - attr_list - - md_in_html - # Emojis - - pymdownx.emoji: - emoji_generator: !!python/name:material.extensions.emoji.to_svg - emoji_index: !!python/name:material.extensions.emoji.twemoji - # Toctree with permalinks - - toc: - title: On this page - permalink: true - toc_depth: 3 - # Grids - - md_in_html - - pymdownx.caret - - pymdownx.tilde - - -extra: - social: - - icon: fontawesome/brands/discord - link: https://discord.com/invite/AnuJBk23FU - - icon: fontawesome/brands/github - link: https://github.com/PyPSA/powerplantmatching - - extra: - homepage: https://pypsa.org - -copyright: >- - © Copyright 2015-2025 Powerplantmatching Developers and Contributors diff --git a/pyproject.toml b/pyproject.toml index 3add57f5..2bc76a5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,19 +56,11 @@ Source = "https://github.com/PyPSA/powerplantmatching" [project.optional-dependencies] docs= [ - "mkdocs-material", - "mkdocs-ezlinks-plugin", - "mkdocs-autolinks-plugin", - "mkdocs-redirects", - "mkdocs-video", - "mkdocs-git-revision-date-localized-plugin", - "mkdocs-git-committers-plugin", - "mkdocs-argref-plugin", - "mkdocs-table-reader-plugin", - "mkdocstrings-python", - "mkdocs-jupyter", - "cairosvg", - "pillow", + "numpydoc", + "sphinx", + "sphinx-book-theme", + "nbsphinx", + "sphinx-automodapi", ] dev= [ "pre-commit",