Raise an error in case the RAM usage goes too high and retry with les…

s2p/__init__.py

@@ -644,16 +644,18 @@ def main(user_cfg, start_from=0):
         if cfg['max_processes_stereo_matching'] is not None:
             nb_workers_stereo = cfg['max_processes_stereo_matching']
         else:
-            nb_workers_stereo = nb_workers
+            nb_workers_stereo = max(1, int(nb_workers / 2.0))
         try:
 
             print(f'4) running stereo matching using {nb_workers_stereo} workers...')
             parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo,
                           timeout=timeout)
-        except subprocess.CalledProcessError as e:
-            print(f'ERROR: stereo matching failed. In case this is due too little RAM set '
-                  f'"max_processes_stereo_matching" to a lower value (currently set to: {nb_workers_stereo}).')
-            raise e
+        except ValueError as e:
+            nb_workers_stereo /= 2.0
+            nb_workers_stereo = max(1, int(nb_workers_stereo))
+            print(f"Retrying stereo matching with {nb_workers_stereo} workers...")
+            parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo,
+                          timeout=timeout)
 
     if start_from <= 5:
         if n > 2:

s2p/parallel.py

@@ -2,6 +2,8 @@
 # Copyright (C) 2017, Carlo de Franchis <[email protected]>
 
 import os
+import time
+import psutil
 import sys
 import traceback
 import multiprocessing
@@ -97,6 +99,13 @@ def launch_calls(fun, list_of_args, nb_workers, *extra_args, tilewise=True,
         else:
             results.append(pool.apply_async(fun, args=args, callback=show_progress))
 
+    while any([not r.ready() for r in results]):
+        time.sleep(1)
+        if (psutil.virtual_memory().available * 100 / psutil.virtual_memory().total) < 5.0:
+            pool.terminate()
+            raise ValueError(f'RAM usage is too high using {nb_workers} workers, killing the process')
+
+
     for r in results:
         try:
             outputs.append(r.get(timeout))

setup.py

@@ -43,6 +43,7 @@ def finalize_options(self):
 
 
 requirements = ['numpy',
+                'psutil',
                 'scipy',
                 'rasterio[s3] @ https://github.com/rasterio/rasterio/archive/refs/tags/1.3.3.tar.gz',
                 'utm',