Commit 28ce2c12 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'bugfix/fix_mp_deadlock' into 'master'

Bugfix/fix mp deadlock

See merge request !9
parents 30dbee81 02973a4b
Pipeline #3817 failed with stages
in 16 minutes and 2 seconds
......@@ -5,6 +5,7 @@ import multiprocessing
import os
from tempfile import TemporaryDirectory
from typing import Union, Tuple, List, Any # noqa: F401
import sys
# custom
try:
......@@ -818,8 +819,8 @@ def _initializer(lats, lons, data):
class SensorMapGeometryTransformer3D(object):
def __init__(self, lons, lats, resamp_alg='nearest', radius_of_influence=30, **opts):
# type: (np.ndarray, np.ndarray, str, int, Any) -> None
def __init__(self, lons, lats, resamp_alg='nearest', radius_of_influence=30, mp_alg='auto', **opts):
# type: (np.ndarray, np.ndarray, str, int, str, Any) -> None
"""Get an instance of SensorMapGeometryTransformer.
:param lons: 3D longitude array corresponding to the 3D sensor geometry array
......@@ -829,6 +830,10 @@ class SensorMapGeometryTransformer3D(object):
- resamp_alg: resampling algorithm ('nearest', 'bilinear', 'gauss', 'custom')
- radius_of_influence: <float> Cut off distance in meters (default: 30)
NOTE: keyword is named 'radius' in case of bilinear resampling
- mp_alg multiprocessing algorithm
'bands': parallelize over bands using multiprocessing lib
'tiles': parallelize over tiles using OpenMP
'auto': automatically choose the algorithm
- sigmas: <list of floats or float> [ONLY 'gauss'] List of sigmas to use for the gauss
weighting of each channel 1 to k, w_k = exp(-dist^2/sigma_k^2). If only one channel
is resampled sigmas is a single float value.
......@@ -867,7 +872,14 @@ class SensorMapGeometryTransformer3D(object):
# bands: multiprocessing uses multiprocessing.Pool, implemented in to_map_geometry / to_sensor_geometry
# tiles: multiprocessing uses OpenMP implemented in pykdtree which is used by pyresample
self.opts['nprocs'] = opts.get('nprocs', multiprocessing.cpu_count())
self.mp_alg = 'bands' if self.lons.shape[2] >= opts['nprocs'] else 'tiles'
self.mp_alg = ('bands' if self.lons.shape[2] >= opts['nprocs'] else 'tiles') if mp_alg == 'auto' else mp_alg
# override self.mp_alg if SensorMapGeometryTransformer3D is called by nosetest or unittest
is_called_by_nose_cmd = 'nosetest' in sys.argv[0]
if self.opts['nprocs'] > 1 and self.mp_alg == 'bands' and is_called_by_nose_cmd:
warnings.warn("mp_alg='bands' causes deadlocks if SensorMapGeometryTransformer3D is called within a "
"nosetest console call. Using mp_alg='tiles'.")
self.mp_alg = 'tiles'
@staticmethod
def _to_map_geometry_2D(kwargs_dict):
......
__version__ = '0.14.16'
__versionalias__ = '20190322_02'
__version__ = '0.14.17'
__versionalias__ = '20190322_03'
......@@ -120,7 +120,9 @@ class Test_SensorMapGeometryTransformer3D(TestCase):
def test_to_map_geometry_lonlat_3D_geolayer(self):
SMGT = SensorMapGeometryTransformer3D(lons=self.lons_3D,
lats=self.lats_3D,
resamp_alg='nearest')
# resamp_alg='nearest',
resamp_alg='bilinear',
)
# to Lon/Lat
data_mapgeo_3D, dem_gt, dem_prj = SMGT.to_map_geometry(self.data_sensor_geo_3D, tgt_prj=4326)
......@@ -137,7 +139,8 @@ class Test_SensorMapGeometryTransformer3D(TestCase):
def test_to_sensor_geometry(self):
SMGT = SensorMapGeometryTransformer3D(lons=self.lons_3D,
lats=self.lats_3D,
resamp_alg='nearest')
resamp_alg='nearest',
)
dem_sensors_geo = SMGT.to_sensor_geometry(self.data_map_geo_3D,
src_prj=32632, src_extent=self.dem_area_extent_coarse_subset_utm)
self.assertIsInstance(dem_sensors_geo, np.ndarray)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment