fix: Implement fallback to Google Drive for UNSD data download (pypsa-meets-earth#1653)

* fix: Implement fallback to Google Drive for UNSD data download

* fix: Update release notes to include hot-fix for UNSD data downtime affecting CI

Author: GbotemiB

doc/release_notes.rst

@@ -12,6 +12,7 @@ Upcoming release
 This part of documentation collects descriptive release notes to capture the main improvements introduced by developing the model before the next release.
 
 **New Features and Major Changes**
+* Added a hot-fix to handle UNSD data downtime causing CI to fail `PR #1653 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1653>`__
 
 * Align PyPSA-Earth costs with the reference units used in `technology-data`, avoiding discrepancies when combining technologies with different original currency years `PR #1604 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1604>`__
 

scripts/build_base_energy_totals.py

@@ -7,6 +7,7 @@
 import glob
 import logging
 import os
+import shutil
 import sys
 from io import BytesIO
 from pathlib import Path
@@ -20,6 +21,7 @@
 import py7zr
 import requests
 from _helpers import BASE_DIR, aggregate_fuels, get_conv_factors
+from googledrivedownloader import download_file_from_google_drive as download_gdrive
 
 _logger = logging.getLogger(__name__)
 
@@ -380,14 +382,32 @@ def calc_sector(sector):
             os.remove(f)
 
         # Feed the dictionary of links to the for loop, download and unzip all files
-        for key, value in d.items():
-            zipurl = value
-
-            with urlopen(zipurl) as zipresp:
-                with ZipFile(BytesIO(zipresp.read())) as zfile:
-                    zfile.extractall(os.path.join(BASE_DIR, "data/demand/unsd/data"))
+        try:
+            for key, value in d.items():
+                zipurl = value
+
+                with urlopen(zipurl) as zipresp:
+                    with ZipFile(BytesIO(zipresp.read())) as zfile:
+                        zfile.extractall(
+                            os.path.join(BASE_DIR, "data/demand/unsd/data")
+                        )
+
+                path = os.path.join(BASE_DIR, "data/demand/unsd/data")
+        except:
+            _logger.warning(
+                f"Could not open the file from {zipurl}. "
+                "Using the data stored in google drive."
+            )
+            download_gdrive(
+                file_id="1VUV0X-tTQECi2pHdE5EWXjPI2yeCdk6F",
+                dest_path=os.path.join(BASE_DIR, "data/demand/unsd/unsd.zip"),
+                unzip=True,
+            )
 
-                    path = os.path.join(BASE_DIR, "data/demand/unsd/data")
+            # clean up __MACOSX folder if it exists
+            macosx_root_path = os.path.join(BASE_DIR, "data/demand/unsd/__MACOSX")
+            if os.path.exists(macosx_root_path):
+                shutil.rmtree(macosx_root_path)
 
     # Get the files from the path provided in the OP
     all_files = list(