Move download script to root directory (#9)

Author: stefsmeets

esmvaltool_sample_data/datasets.yml datasets.ymlesmvaltool_sample_data/datasets.yml renamed to datasets.yml

@@ -9,10 +9,8 @@
 
 2) A configuration file called datasets.yml in esmvaltool_sample_data that
    defines the datasets to download.
-
 """
 import datetime
-import os
 import warnings
 from pathlib import Path
 
@@ -34,7 +32,6 @@ def get_time(filename):
     -------
     (datetetime.datetime, datetime.datetime)
         A tuple containing the start and end date of the file.
-
     """
     times = Path(filename).stem.split('_')[-1].split('-')
     fmt = {
@@ -57,7 +54,6 @@ def select_by_time(filename, from_timestamp, to_timestamp):
         The required start date, formatted as "2000-01-01T00:00:00Z".
     to_timestamp : str or None
         The required end date, formatted as "2000-01-01T00:00:00Z".
-
     """
     if from_timestamp is None and to_timestamp is None:
         return True
@@ -99,7 +95,6 @@ def select_host(hosts, preferred_hosts, ignore_hosts):
     -----
         Not sure if this is reliable: sometimes no files are found on the
         selected host.
-
     """
     hosts = [h for h in hosts if h not in ignore_hosts]
     if not hosts:
@@ -205,7 +200,6 @@ def save_sample(in_file, out_file):
         Path to the input file.
     out_file : str
         Path to the output file.
-
     """
     print("Saving sample of", in_file, "to", out_file)
     with warnings.catch_warnings():
@@ -262,13 +256,14 @@ def sample_files(plot_type, dataset_name, files):
         Name of the dataset to sample.
     files : :obj:`list` of :obj:`str`
         A list of filenames that comprise the dataset.
-
     """
+    project_dir = Path(__file__).parent
+    data_dir = project_dir / 'esmvaltool_sample_data' / 'data' / plot_type
+    data_path = data_dir.joinpath(*dataset_name.split('.'))
+    data_path.mkdir(parents=True, exist_ok=True)
+
     for filename in files:
-        dirpath = (Path(__file__).parent / 'data' / plot_type /
-                   dataset_name.replace('.', os.sep))
-        dirpath.mkdir(parents=True, exist_ok=True)
-        out_file = dirpath / Path(filename).name
+        out_file = data_path / Path(filename).name
         if out_file.exists():
             print("File exists, skipping:", out_file)
         else:
@@ -283,13 +278,13 @@ def main():
 
     The resulting list of files is then sampled and stored locally in the
     directory 'data' using a commonly used directory structure.
-
     """
-    cfg_file = Path(__file__).parent.parent / "config.yml"
+    project_dir = Path(__file__).parent
+    cfg_file = project_dir / "config.yml"
     with cfg_file.open() as file:
         cfg = yaml.safe_load(file)
 
-    facets_file = Path(__file__).parent / "datasets.yml"
+    facets_file = project_dir / "datasets.yml"
     with facets_file.open() as file:
         cfg_data = yaml.safe_load(file)
 

esmvaltool_sample_data/sample_data.py download_sample_data.pyesmvaltool_sample_data/sample_data.py renamed to download_sample_data.py

@@ -2,16 +2,15 @@
 
 import cf_units
 import iris
-import yaml
 
 base_dir = Path(__file__).parent
 
 VERBOSE = False
 
-with open(base_dir / 'datasets.yml', 'r') as f:
-    config = yaml.safe_load(f)
-
-ignore_list = [fn.replace('.', '/') for fn in config['ignore']]
+# This ignore list is used to help with debugging
+ignore_list = [
+    # 'CMIP6/CMIP/AWI/AWI-ESM-1-1-LR/historical/r1i1p1f1/Amon/ta/gn/v20200212',
+]
 
 
 def strip_attributes(cube: 'iris.Cube') -> None: