Check for existing files in testing/setup.py

Author: jochenklar

.github/workflows/ci.yaml

@@ -98,7 +98,7 @@ jobs:
     - name: Install cdo 🌍
       run: |
         sudo apt-get update
-        sudo apt-get install -y cdo --no-install-recommends
+        sudo apt-get install -y cdo netcdf-bin --no-install-recommends
 
     - name: Set up Python 🐍
       uses: actions/setup-python@v6

testing/setup.py

@@ -45,7 +45,8 @@ def run_gridfile():
     output_path = constants.SHARE_PATH / 'gridarea.nc'
     output_path.parent.mkdir(parents=True, exist_ok=True)
 
-    helper.call(f'cdo gridarea {input_path} {output_path}')
+    if not output_path.exists():
+        helper.call(f'cdo gridarea {input_path} {output_path}')
 
 
 def run_seldate():
@@ -57,7 +58,9 @@ def run_seldate():
         ('2019-01-01', '2020-12-31', '2019_2020')
     ]:
         output_path = constants.DATASETS_PATH / constants.TAS_PATH.replace('2015_2020', specifiers)
-        helper.call(f'cdo -f nc4c -z zip_5 -L seldate,{start_date},{end_date} {input_path} {output_path}')
+
+        if not output_path.exists():
+            helper.call(f'cdo -f nc4c -z zip_5 -L seldate,{start_date},{end_date} {input_path} {output_path}')
 
 
 def run_select_time():
@@ -70,9 +73,9 @@ def run_select_time():
     output_path = constants.EXTRACTIONS_PATH / path.replace('_global_', '_select-time-cdo_') \
                                                    .replace('2015_2020', '20180101')
     output_path.parent.mkdir(parents=True, exist_ok=True)
-    output_path.unlink(missing_ok=True)
 
-    helper.call(f'cdo -f nc4c -z zip_5 -L seldate,{date} {input_path} {output_path}')
+    if not output_path.exists():
+        helper.call(f'cdo -f nc4c -z zip_5 -L seldate,{date} {input_path} {output_path}')
 
 
 def run_select_period():
@@ -85,9 +88,9 @@ def run_select_period():
     output_path = constants.EXTRACTIONS_PATH / path.replace('_global_', '_select-period-cdo_') \
                                                    .replace('2015_2020', '2015')
     output_path.parent.mkdir(parents=True, exist_ok=True)
-    output_path.unlink(missing_ok=True)
 
-    helper.call(f'cdo -f nc4c -z zip_5 -L seldate,{start_date},{end_date} {input_path} {output_path}')
+    if not output_path.exists():
+        helper.call(f'cdo -f nc4c -z zip_5 -L seldate,{start_date},{end_date} {input_path} {output_path}')
 
 
 def run_select_point():
@@ -96,50 +99,42 @@ def run_select_point():
     # add one since cdo is counting from 1!
     ix, iy = ix + 1, iy + 1
 
-    output_paths = []
     for path in [constants.TAS_PATH, *constants.TAS_SPLIT_PATHS]:
         input_path = constants.DATASETS_PATH / path
 
         output_path = constants.EXTRACTIONS_PATH / path.replace('_global_', '_select-point-cdo_')
         output_path.parent.mkdir(parents=True, exist_ok=True)
-        output_path.unlink(missing_ok=True)
-
-        output_paths.append(str(output_path))
 
-        helper.call(f'cdo -f nc4c -z zip_5 -L -selindexbox,{ix},{ix},{iy},{iy} {input_path} {output_path}')
+        if not output_path.exists():
+            helper.call(f'cdo -f nc4c -z zip_5 -L -selindexbox,{ix},{ix},{iy},{iy} {input_path} {output_path}')
 
 
 def run_select_bbox():
     west, east, south, north = constants.BBOX
 
-    output_paths = []
     for path in [constants.TAS_PATH, *constants.TAS_SPLIT_PATHS]:
         input_path = constants.DATASETS_PATH / path
 
         output_path = constants.EXTRACTIONS_PATH / path.replace('_global_', '_select-bbox-cdo_')
         output_path.parent.mkdir(parents=True, exist_ok=True)
-        output_path.unlink(missing_ok=True)
 
-        output_paths.append(str(output_path))
-
-        helper.call(f'cdo -f nc4c -z zip_5 -L -sellonlatbox,{west},{east},{south},{north} {input_path} {output_path}')
+        if not output_path.exists():
+            helper.call(f'cdo -f nc4c -z zip_5 -L ' \
+                        f'-sellonlatbox,{west},{east},{south},{north} {input_path} {output_path}')
 
 
 def run_select_bbox_mean():
     west, east, south, north = constants.BBOX
 
-    output_paths = []
     for path in [constants.TAS_PATH, *constants.TAS_SPLIT_PATHS]:
         input_path = constants.DATASETS_PATH / path
 
         output_path = constants.EXTRACTIONS_PATH / path.replace('_global_', '_select-bbox-mean-cdo_')
         output_path.parent.mkdir(parents=True, exist_ok=True)
-        output_path.unlink(missing_ok=True)
-
-        output_paths.append(str(output_path))
 
-        helper.call('cdo -f nc4c -z zip_5 -L -fldmean ' \
-                    f'-sellonlatbox,{west},{east},{south},{north} {input_path} {output_path}')
+        if not output_path.exists():
+            helper.call('cdo -f nc4c -z zip_5 -L -fldmean '
+                        f'-sellonlatbox,{west},{east},{south},{north} {input_path} {output_path}')
 
 
 def run_select_bbox_map():
@@ -150,42 +145,37 @@ def run_select_bbox_map():
 
         output_path = constants.EXTRACTIONS_PATH / path.replace('_global_', '_select-bbox-map-cdo_')
         output_path.parent.mkdir(parents=True, exist_ok=True)
-        output_path.unlink(missing_ok=True)
 
-        helper.call('cdo -f nc4c -z zip_5 -L timmean ' \
-                    f'-sellonlatbox,{west},{east},{south},{north} {input_path} {output_path}')
+        if not output_path.exists():
+            helper.call('cdo -f nc4c -z zip_5 -L timmean '
+                        f'-sellonlatbox,{west},{east},{south},{north} {input_path} {output_path}')
 
 
 def run_mask_bbox():
     west, east, south, north = constants.BBOX
 
-    output_paths = []
     for path in [constants.TAS_PATH, *constants.TAS_SPLIT_PATHS]:
         input_path = constants.DATASETS_PATH / path
 
         output_path = constants.EXTRACTIONS_PATH / path.replace('_global_', '_mask-bbox-cdo_')
         output_path.parent.mkdir(parents=True, exist_ok=True)
-        output_path.unlink(missing_ok=True)
 
-        output_paths.append(str(output_path))
-
-        helper.call(f'cdo -f nc4c -z zip_5 -L -masklonlatbox,{west},{east},{south},{north} {input_path} {output_path}')
+        if not output_path.exists():
+            helper.call(f'cdo -f nc4c -z zip_5 -L '
+                        f'-masklonlatbox,{west},{east},{south},{north} {input_path} {output_path}')
 
 
 def run_mask_mask():
     mask_path = constants.DATASETS_PATH / constants.LANDSEAMASK_PATH
 
-    output_paths = []
     for path in [constants.TAS_PATH, *constants.TAS_SPLIT_PATHS]:
         input_path = constants.DATASETS_PATH / path
 
         output_path = constants.EXTRACTIONS_PATH / path.replace('_global_', '_mask-mask-cdo_')
         output_path.parent.mkdir(parents=True, exist_ok=True)
-        output_path.unlink(missing_ok=True)
-
-        output_paths.append(str(output_path))
 
-        helper.call(f'cdo -f nc4c -z zip_5 -L -ifthen -selname,mask {mask_path} {input_path} {output_path}')
+        if not output_path.exists():
+            helper.call(f'cdo -f nc4c -z zip_5 -L -ifthen -selname,mask {mask_path} {input_path} {output_path}')
 
 
 if __name__ == "__main__":