diff --git a/s2p/__init__.py b/s2p/__init__.py index 3da0d407..bcbafd56 100644 --- a/s2p/__init__.py +++ b/s2p/__init__.py @@ -644,16 +644,18 @@ def main(user_cfg, start_from=0): if cfg['max_processes_stereo_matching'] is not None: nb_workers_stereo = cfg['max_processes_stereo_matching'] else: - nb_workers_stereo = nb_workers + nb_workers_stereo = max(1, int(nb_workers / 2.0)) try: print(f'4) running stereo matching using {nb_workers_stereo} workers...') parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo, timeout=timeout) - except subprocess.CalledProcessError as e: - print(f'ERROR: stereo matching failed. In case this is due too little RAM set ' - f'"max_processes_stereo_matching" to a lower value (currently set to: {nb_workers_stereo}).') - raise e + except ValueError as e: + nb_workers_stereo /= 2.0 + nb_workers_stereo = max(1, int(nb_workers_stereo)) + print(f"Retrying stereo matching with {nb_workers_stereo} workers...") + parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo, + timeout=timeout) if start_from <= 5: if n > 2: diff --git a/s2p/parallel.py b/s2p/parallel.py index 1ac27886..84f9be45 100644 --- a/s2p/parallel.py +++ b/s2p/parallel.py @@ -2,6 +2,8 @@ # Copyright (C) 2017, Carlo de Franchis import os +import time +import psutil import sys import traceback import multiprocessing @@ -97,6 +99,13 @@ def launch_calls(fun, list_of_args, nb_workers, *extra_args, tilewise=True, else: results.append(pool.apply_async(fun, args=args, callback=show_progress)) + while any([not r.ready() for r in results]): + time.sleep(1) + if (psutil.virtual_memory().available * 100 / psutil.virtual_memory().total) < 5.0: + pool.terminate() + raise ValueError(f'RAM usage is too high using {nb_workers} workers, killing the process') + + for r in results: try: outputs.append(r.get(timeout)) diff --git a/setup.py b/setup.py index e85173f8..a3fd482d 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ def finalize_options(self): requirements = ['numpy', + 'psutil', 'scipy', 'rasterio[s3] @ https://github.com/rasterio/rasterio/archive/refs/tags/1.3.3.tar.gz', 'utm',