Commit 55a3b144 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Fixed sub-multiprocessing error.

parent 8b6428e7
Pipeline #3562 passed with stages
in 2 minutes and 15 seconds
......@@ -829,7 +829,21 @@ class SensorMapGeometryTransformer3D(object):
self.resamp_alg = resamp_alg
self.radius_of_influence = radius_of_influence
self.opts = opts
self.CPUs = opts['nprocs'] if 'nprocs' in opts and opts['nprocs'] else multiprocessing.cpu_count()
# define number of CPUs to use (but avoid sub-multiprocessing)
# -> parallelize either over bands or over image tiles
ncpus_avail = multiprocessing.cpu_count()
if 'nprocs' in opts and opts['nprocs']:
if self.lons.shape[2] > opts['nprocs']:
self.CPUs = opts['nprocs'] # parallelize over bands
opts['nprocs'] = 1
else:
self.CPUs = 1 # parallelize over imgage tiles
elif self.lons.shape[2] > ncpus_avail:
self.CPUs = ncpus_avail # parallelize over bands
else:
self.CPUs = 1
opts['nprocs'] = ncpus_avail # parallelize over imgage tiles
@staticmethod
def _to_map_geometry_2D(kwargs_dict):
......@@ -893,8 +907,11 @@ class SensorMapGeometryTransformer3D(object):
band_idx=band
) for band in range(data.shape[2])]
if self.CPUs > 1:
with multiprocessing.Pool(self.CPUs) as pool:
result = pool.map(self._to_map_geometry_2D, args)
else:
result = [self._to_map_geometry_2D(argsdict) for argsdict in args]
band_inds = list(np.array(result)[:, -1])
data_mapgeo = np.dstack((result[band_inds.index(i)][0] for i in range(data.shape[2])))
......@@ -940,8 +957,11 @@ class SensorMapGeometryTransformer3D(object):
band_idx=band
) for band in range(data.shape[2])]
if self.CPUs > 1:
with multiprocessing.Pool(self.CPUs) as pool:
result = pool.map(self._to_sensor_geometry_2D, args)
else:
result = [self._to_sensor_geometry_2D(argsdict) for argsdict in args]
band_inds = list(np.array(result)[:, -1])
data_sensorgeo = np.dstack((result[band_inds.index(i)][0] for i in range(data.shape[2])))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment