From 285f29954d6360ae983fdaf82eb466b8556b9eb7 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Wed, 16 Nov 2022 16:56:29 +0100 Subject: [PATCH 1/4] Raise an error in case the RAM usage goes too high and retry with less stereo workers. --- s2p/__init__.py | 10 ++++++---- s2p/parallel.py | 9 +++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/s2p/__init__.py b/s2p/__init__.py index 3da0d407..1fe9a250 100644 --- a/s2p/__init__.py +++ b/s2p/__init__.py @@ -650,10 +650,12 @@ def main(user_cfg, start_from=0): print(f'4) running stereo matching using {nb_workers_stereo} workers...') parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo, timeout=timeout) - except subprocess.CalledProcessError as e: - print(f'ERROR: stereo matching failed. In case this is due too little RAM set ' - f'"max_processes_stereo_matching" to a lower value (currently set to: {nb_workers_stereo}).') - raise e + except ValueError as e: + nb_workers_stereo /= 2.0 + nb_workers_stereo = min(1, int(nb_workers_stereo)) + print(f"Retrying stereo matching with {nb_workers_stereo} workers...") + parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo, + timeout=timeout) if start_from <= 5: if n > 2: diff --git a/s2p/parallel.py b/s2p/parallel.py index 1ac27886..a2dee5e1 100644 --- a/s2p/parallel.py +++ b/s2p/parallel.py @@ -2,6 +2,8 @@ # Copyright (C) 2017, Carlo de Franchis import os +import time +import psutil import sys import traceback import multiprocessing @@ -97,6 +99,13 @@ def launch_calls(fun, list_of_args, nb_workers, *extra_args, tilewise=True, else: results.append(pool.apply_async(fun, args=args, callback=show_progress)) + while any([not r.ready() for r in results]): + time.sleep(1) + if (psutil.virtual_memory().available * 100 / psutil.virtual_memory().total) < 5.0: + pool.terminate() + raise ValueError('RAM usage is too high, killing the process') + + for r in results: try: outputs.append(r.get(timeout)) From aad852ed95b2e859efa2dd362597e67848212dcb Mon Sep 17 00:00:00 2001 From: David de Meij Date: Wed, 16 Nov 2022 17:19:00 +0100 Subject: [PATCH 2/4] Add number of workers to error raised --- s2p/parallel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s2p/parallel.py b/s2p/parallel.py index a2dee5e1..84f9be45 100644 --- a/s2p/parallel.py +++ b/s2p/parallel.py @@ -103,7 +103,7 @@ def launch_calls(fun, list_of_args, nb_workers, *extra_args, tilewise=True, time.sleep(1) if (psutil.virtual_memory().available * 100 / psutil.virtual_memory().total) < 5.0: pool.terminate() - raise ValueError('RAM usage is too high, killing the process') + raise ValueError(f'RAM usage is too high using {nb_workers} workers, killing the process') for r in results: From 35694987a33bfc750a25acdd6450be0b445d3d4f Mon Sep 17 00:00:00 2001 From: David de Meij Date: Wed, 16 Nov 2022 17:55:40 +0100 Subject: [PATCH 3/4] Add psutil to requirements --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e85173f8..a3fd482d 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ def finalize_options(self): requirements = ['numpy', + 'psutil', 'scipy', 'rasterio[s3] @ https://github.com/rasterio/rasterio/archive/refs/tags/1.3.3.tar.gz', 'utm', From 0509507337536e073135d1f38e3f29a9e4a3dc26 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Wed, 16 Nov 2022 18:20:24 +0100 Subject: [PATCH 4/4] Use max instead of min --- s2p/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s2p/__init__.py b/s2p/__init__.py index 1fe9a250..bcbafd56 100644 --- a/s2p/__init__.py +++ b/s2p/__init__.py @@ -644,7 +644,7 @@ def main(user_cfg, start_from=0): if cfg['max_processes_stereo_matching'] is not None: nb_workers_stereo = cfg['max_processes_stereo_matching'] else: - nb_workers_stereo = nb_workers + nb_workers_stereo = max(1, int(nb_workers / 2.0)) try: print(f'4) running stereo matching using {nb_workers_stereo} workers...') @@ -652,7 +652,7 @@ def main(user_cfg, start_from=0): timeout=timeout) except ValueError as e: nb_workers_stereo /= 2.0 - nb_workers_stereo = min(1, int(nb_workers_stereo)) + nb_workers_stereo = max(1, int(nb_workers_stereo)) print(f"Retrying stereo matching with {nb_workers_stereo} workers...") parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo, timeout=timeout)