Commit a1b8983d authored by Niklas Bohn's avatar Niklas Bohn
Browse files

Added missing initializer to multiprocessing pool in empirical line...


Added missing initializer to multiprocessing pool in empirical line calculation and set multiprocessing start method to fork.
Signed-off-by: Niklas Bohn's avatarnbohn <nbohn@gfz-potsdam.de>
parent ad1439f3
......@@ -28,7 +28,7 @@ import logging
import numpy as np
import pandas as pd
import dill
from multiprocessing import Pool
import multiprocessing as mp
from itertools import product
from time import time
import warnings
......@@ -1125,11 +1125,7 @@ def __minimize__(fo, opt_func, unknowns=False, logger=None):
warnings.filterwarnings("always")
processes = fo.cpu
if platform.system() == "Windows" and processes > 1:
logger.warning('Multiprocessing is currently not available on Windows.')
if platform.system() == "Darwin" and processes > 1:
logger.warning('Multiprocessing is currently not available on macOS.')
if platform.system() == "Windows" or platform.system() == "Darwin" or processes == 1:
if processes == 1:
logger.info("Singleprocessing on 1 cpu")
else:
logger.info("Setting up multiprocessing...")
......@@ -1231,14 +1227,13 @@ def __minimize__(fo, opt_func, unknowns=False, logger=None):
# start optimization
logger.info("Optimization...")
t0 = time()
# check if operating system is 'Windows'; in that case, multiprocessing is currently not working
# TODO: enable Windows compatibility for multiprocessing
if platform.system() == "Windows" or platform.system() == "Darwin" or processes == 1:
if processes == 1:
initializer(globals(), globs)
[mp_fun(ii) for ii in tqdm(rng, disable=fo.disable_progressbars)]
else:
with closing(Pool(processes=processes, initializer=initializer, initargs=(globals(), globs,))) as pl:
with closing(mp.get_context("fork").Pool(processes=processes, initializer=initializer, initargs=(globals(),
globs))) as pl:
results = pl.map_async(mp_fun, rng, chunksize=1)
if not fo.disable_progressbars:
bar = ProgressBar(prefix='\tprogress:')
......
......@@ -25,7 +25,7 @@
import numpy as np
from multiprocessing import Pool
import multiprocessing as mp
from tqdm import tqdm
import platform
......@@ -145,14 +145,15 @@ def empirical_line_solution(X, rdn_subset, data_l2a_seg, rows, cols, bands, segs
global _globs
_globs = dict(tree=tree, locs=locations_subset, k=15, rdn=rdn_subset, data=data_l2a_seg, bands=bands)
if platform.system() == "Windows" or platform.system() == "Darwin" or processes == 1:
if processes == 1:
initializer(globals(), _globs)
results = []
for ii in tqdm(unique_labels, disable=disable_progressbars):
(sl, ic) = _compute_coefficients_for_label(ii)
results.append((sl, ic))
else:
with Pool(processes=processes) as pool:
with mp.get_context("fork").Pool(processes=processes, initializer=initializer, initargs=(globals(),
_globs)) as pool:
results = pool.map(_compute_coefficients_for_label, unique_labels)
slopes, intercepts = np.empty([rows * cols, bands]), np.empty([rows * cols, bands])
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment