Skip to content

Commit 46c01e8

Browse files
fneumfneum
committed
Revert "process eurostat energy balances via API"
This reverts commit a81e100.
1 parent a81e100 commit 46c01e8

11 files changed

+440
-179
lines changed

‎config/config.default.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1279,7 +1279,7 @@ data:
12791279
source: archive
12801280
version: latest
12811281
eurostat_balances:
1282-
source: primary
1282+
source: archive
12831283
version: latest
12841284
eurostat_household_balances:
12851285
source: archive

‎data/versions.csv‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,10 @@
1818
"eu_nuts2013","primary","2015-12-03","['latest', 'supported']","https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2013-03m.geojson.zip"," "
1919
"eu_nuts2021","archive","2021-01-01","['latest', 'supported']","https://zenodo.org/records/15846440/files/ref-nuts-2021-01m.geojson.zip",""
2020
"eu_nuts2021","primary","2021-01-01","['latest', 'supported']","https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2021-01m.geojson.zip",""
21-
"eurostat_balances","primary","2026-01","['latest', 'supported']","https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/nrg_bal_c?format=TSV&compressed=true",""
21+
"eurostat_balances","archive","2023-04","['latest', 'supported']","https://zenodo.org/records/15849144/files/balances.zip",""
22+
"eurostat_balances","primary","2023-04","['latest', 'supported', 'broken link']","https://ec.europa.eu/eurostat/documents/38154/4956218/Balances-April2023.zip","The link is broken, use the archived versions instead."
2223
"eurostat_household_balances","archive","2025-07-09","['latest', 'supported']","https://zenodo.org/records/15849673/files/nrg_d_hhq.csv",""
23-
"eurostat_household_balances","primary","unknown","['latest', 'supported']","https://ec.europa.eu/eurostat/databrowser-backend/api/extraction/1.0/LIVE/false/sdmx/csv/nrg_d_hhq__custom_11480365?startPeriod=2013&endPeriod=2024","URL limits the period to 2013-2022, but the data is updated regularly."
24+
"eurostat_household_balances","primary","unknown","['latest', 'supported']","https://ec.europa.eu/eurostat/databrowser-backend/api/extraction/1.0/LIVE/false/sdmx/csv/nrg_d_hhq__custom_11480365?startPeriod=2013&endPeriod=2022","URL limits the period to 2013-2022, but the data is updated regularly."
2425
"osm","archive","0.1","['deprecated', 'not-supported']","https://zenodo.org/records/12799202",""
2526
"osm","archive","0.2","['deprecated', 'not-supported']","https://zenodo.org/records/13342577",""
2627
"osm","archive","0.3","['deprecated', 'not-supported']","https://zenodo.org/records/13358976",""

‎doc/data_inventory.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"nitrogen_statistics","Nitrogen Statistics and Information","Statistics and information on the worldwide supply of, demand for, and flow of the mineral commodity nitrogen.","United States Geological Survey (USGS)","https://www.usgs.gov/centers/nmic/nitrogen-statistics-and-information","Public Domain"
88
"eu_nuts2013","Nomenclature of Territorial Units for Statistics (NUTS) 2013 - shapefiles","Shapefiles of EU's Nomenclature of Territorial Units for Statistics (NUTS) 2013, which is a hierarchical system for dividing up the economic territory of the European Union.","eurostat","https://ec.europa.eu/eurostat/web/nuts/overview","Reuse policy following 2011/833/EU"
99
"eu_nuts2021","Nomenclature of Territorial Units for Statistics (NUTS) 2021 - shapefiles","Shapefiles of EU's Nomenclature of Territorial Units for Statistics (NUTS) 2021, which is a hierarchical system for dividing up the economic territory of the European Union.","eurostat","https://ec.europa.eu/eurostat/web/nuts/overview","Reuse policy following 2011/833/EU"
10-
"eurostat_balances","Energy Balances","European energy balances by country and fuel, as reported by Eurostat.","eurostat","https://ec.europa.eu/eurostat/data/database","CC-BY-4.0"
10+
"eurostat_balances","Energy Balances","European energy balances by country and fuel, as reported by Eurostat.","eurostat","https://ec.europa.eu/eurostat/data/database","Reuse policy following 2011/833/EU ; newer versions of the same data are available as CC-BY-4.0 through the eurostat API"
1111
"eurostat_household_balances","Eurostat Household Energy Balances","Disaggregated final energy consumption in household - quantities (nrg_d_hhq)","eurostat","https://ec.europa.eu/eurostat/databrowser/product/page/NRG_D_HHQ","CC-BY-4.0"
1212
"luisa_land_cover","The LUISA base map 2018","The LUISA Base Map 2018 is a high-resolution land use/land cover map developed and produced by the Joint Research Centre of the European Commission","European Commission Joint Research Centre","https://data.jrc.ec.europa.eu/dataset/51858b51-8f27-4006-bf82-53eba35a142c","CC-BY-4.0"
1313
"jrc_idees","JRC-IDEES-2021","The JRC-IDEES-2021 release contains a consistent set of disaggregated energy-economy-emissions data for each Member State of the European Union, covering all sectors of the energy system for the 2000-2021 period: industry, buildings, transport, and power generation.", "European Commission Joint Research Centre", "https://data.jrc.ec.europa.eu/dataset/82322924-506a-4c9a-8532-2bdd30d69bf5", "CC-BY-4.0"

‎doc/release_notes.rst‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,6 @@ Release Notes
99
Upcoming Release
1010
================
1111

12-
* Download and process Eurostat energy balances from API. This replaces the discontinued ZIP bulk download.
13-
14-
* The function `rescale_idees_from_eurostat` was removed.
15-
1612
* Important: PyPSA-Eur now uses a validation schema for configuration files. The schema
1713
also contains the default values for all known configuration options, which means
1814
`config/config.default.yaml` still exists and can be used, but will be automatically

‎rules/build_sector.smk‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -791,7 +791,7 @@ rule build_energy_totals:
791791
swiss_transport=f"{BFS_ROAD_VEHICLE_STOCK_DATASET['folder']}/vehicle_stock.csv",
792792
idees=rules.retrieve_jrc_idees.output["directory"],
793793
district_heat_share="data/district_heat_share.csv",
794-
eurostat=rules.retrieve_eurostat_balances.output["tsv_gz"],
794+
eurostat=rules.retrieve_eurostat_balances.output["directory"],
795795
eurostat_households=rules.retrieve_eurostat_household_balances.output["csv"],
796796
output:
797797
transformation_output_coke=resources("transformation_output_coke.csv"),
@@ -859,7 +859,7 @@ rule build_biomass_potentials:
859859
biomass=config_provider("biomass"),
860860
input:
861861
enspreso_biomass=rules.retrieve_enspreso_biomass.output["xlsx"],
862-
eurostat=rules.retrieve_eurostat_balances.output["tsv_gz"],
862+
eurostat=rules.retrieve_eurostat_balances.output["directory"],
863863
nuts2=rules.retrieve_eu_nuts_2013.output["shapes_level_2"],
864864
regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"),
865865
nuts3_population=ancient(rules.retrieve_nuts3_population.output["gz"]),
@@ -1049,7 +1049,7 @@ rule build_industrial_production_per_country:
10491049
input:
10501050
ch_industrial_production="data/ch_industrial_production_per_subsector.csv",
10511051
ammonia_production=resources("ammonia_production.csv"),
1052-
eurostat=rules.retrieve_eurostat_balances.output["tsv_gz"],
1052+
eurostat=rules.retrieve_eurostat_balances.output["directory"],
10531053
jrc=rules.retrieve_jrc_idees.output["directory"],
10541054
output:
10551055
industrial_production_per_country=resources(
@@ -1595,7 +1595,7 @@ rule prepare_sector_network:
15951595
else []
15961596
),
15971597
network=resources("networks/base_s_{clusters}_elec_{opts}.nc"),
1598-
eurostat=rules.retrieve_eurostat_balances.output["tsv_gz"],
1598+
eurostat=rules.retrieve_eurostat_balances.output["directory"],
15991599
pop_weighted_energy_totals=resources(
16001600
"pop_weighted_energy_totals_s_{clusters}.csv"
16011601
),

‎rules/postprocess.smk‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -449,7 +449,7 @@ rule plot_summary:
449449
costs=RESULTS + "csvs/costs.csv",
450450
energy=RESULTS + "csvs/energy.csv",
451451
balances=RESULTS + "csvs/energy_balance.csv",
452-
eurostat=rules.retrieve_eurostat_balances.output["tsv_gz"],
452+
eurostat=rules.retrieve_eurostat_balances.output["directory"],
453453
co2=rules.retrieve_ghg_emissions.output["csv"],
454454
output:
455455
costs=RESULTS + "graphs/costs.svg",

‎rules/retrieve.smk‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,13 @@ if (EUROSTAT_BALANCES_DATASET := dataset_version("eurostat_balances"))["source"]
3131
message:
3232
"Retrieving Eurostat balances data"
3333
input:
34-
tsv_gz=storage(EUROSTAT_BALANCES_DATASET["url"]),
34+
zip_file=storage(EUROSTAT_BALANCES_DATASET["url"]),
3535
output:
36-
tsv_gz=f"{EUROSTAT_BALANCES_DATASET['folder']}/estat_nrg_bal_c.tsv.gz",
36+
zip_file=f"{EUROSTAT_BALANCES_DATASET['folder']}/balances.zip",
37+
directory=directory(EUROSTAT_BALANCES_DATASET["folder"]),
3738
run:
38-
copy2(input["tsv_gz"], output["tsv_gz"])
39+
copy2(input["zip_file"], output["zip_file"])
40+
unpack_archive(output["zip_file"], output["directory"])
3941

4042

4143
if (

‎scripts/build_biomass_potentials.py‎

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ def convert_nuts2_to_regions(bio_nuts2, regions):
246246
return bio_regions
247247

248248

249-
def add_unsustainable_potentials(df, input_eurostat):
249+
def add_unsustainable_potentials(df):
250250
"""
251251
Add unsustainable biomass potentials to the given dataframe. The difference
252252
between the data of JRC and Eurostat is assumed to be unsustainable
@@ -269,29 +269,36 @@ def add_unsustainable_potentials(df, input_eurostat):
269269
else:
270270
latest_year = 2021
271271
idees_rename = {"GR": "EL", "GB": "UK"}
272-
year = max(min(latest_year, int(snakemake.wildcards.planning_horizons)), 1990) # noqa: F841
273272
df_unsustainable = (
274-
build_eurostat(input_eurostat)
275-
.query("year == @year and nrg_bal == 'PPRD'") # Primary production
276-
.set_index(["country", "siec"])
277-
.value.unstack("siec")
273+
build_eurostat(
274+
countries=snakemake.config["countries"],
275+
input_eurostat=snakemake.input.eurostat,
276+
nprocesses=int(snakemake.threads),
277+
)
278+
.xs(
279+
max(min(latest_year, int(snakemake.wildcards.planning_horizons)), 1990),
280+
level=1,
281+
)
282+
.xs("Primary production", level=2)
283+
.droplevel([1, 2, 3])
278284
)
279285

286+
df_unsustainable.index = df_unsustainable.index.str.strip()
280287
df_unsustainable = df_unsustainable.rename(
281288
{v: k for k, v in idees_rename.items()}, axis=0
282289
)
283290

284291
bio_carriers = [
285-
"R5110-5150_W6000RI", # Primary solid biofuels
286-
"R5300", # Biogases
287-
"W6210", # Renewable municipal waste
288-
"R5210P", # Pure biogasoline
289-
"R5210B", # Blended biogasoline
290-
"R5220P", # Pure biodiesels
291-
"R5220B", # Blended biodiesels
292-
"R5230P", # Pure bio jet kerosene
293-
"R5230B", # Blended bio jet kerosene
294-
"R5290", # Other liquid biofuels
292+
"Primary solid biofuels",
293+
"Biogases",
294+
"Renewable municipal waste",
295+
"Pure biogasoline",
296+
"Blended biogasoline",
297+
"Pure biodiesels",
298+
"Blended biodiesels",
299+
"Pure bio jet kerosene",
300+
"Blended bio jet kerosene",
301+
"Other liquid biofuels",
295302
]
296303

297304
df_unsustainable = df_unsustainable[bio_carriers]
@@ -303,26 +310,20 @@ def add_unsustainable_potentials(df, input_eurostat):
303310

304311
# Calculate unsustainable solid biomass
305312
df_wo_ch["unsustainable solid biomass"] = _calc_unsustainable_potential(
306-
df_wo_ch,
307-
df_unsustainable,
308-
share_unsus,
309-
"R5110-5150_W6000RI", # Primary solid biofuels
313+
df_wo_ch, df_unsustainable, share_unsus, "Primary solid biofuels"
310314
)
311315

312316
# Calculate unsustainable biogas
313317
df_wo_ch["unsustainable biogas"] = _calc_unsustainable_potential(
314-
df_wo_ch,
315-
df_unsustainable,
316-
share_unsus,
317-
"R5300", # Biogases
318+
df_wo_ch, df_unsustainable, share_unsus, "Biogases"
318319
)
319320

320321
# Calculate unsustainable bioliquids
321322
df_wo_ch["unsustainable bioliquids"] = _calc_unsustainable_potential(
322323
df_wo_ch,
323324
df_unsustainable,
324325
share_unsus,
325-
resource_type="R5210|R5220|R5230|R5290", # gasoline, diesel, kerosene, liquids
326+
resource_type="gasoline|diesel|kerosene|liquid",
326327
)
327328

328329
share_sus = params.get("share_sustainable_potential_available").get(investment_year)
@@ -390,8 +391,7 @@ def add_unsustainable_potentials(df, input_eurostat):
390391
grouper = {v: k for k, vv in params["classes"].items() for v in vv}
391392
df = df.T.groupby(grouper).sum().T
392393

393-
input_eurostat = snakemake.input.eurostat
394-
df = add_unsustainable_potentials(df, input_eurostat)
394+
df = add_unsustainable_potentials(df)
395395

396396
df *= 1e6 # TWh/a to MWh/a
397397
df.index.name = "MWh/a"

0 commit comments

Comments
 (0)