Commit 44c93f36 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Added ECMWF download lock -> fixes feature request #71 ([ECMWF downloads] Add...

Added ECMWF download lock -> fixes feature request #71 ([ECMWF downloads] Add lock to avoid too many connections to ECMWF download API).
parent 663d4555
......@@ -23,6 +23,7 @@ from . import geoprocessing as GEOP
from .L1B_P import L1B_object
from ..model.metadata import get_LayerBandsAssignment
from ..misc.definition_dicts import get_outFillZeroSaturated, proc_chain, get_mask_classdefinition
from ..misc.locks import MultiSlotLock
from ..io.input_reader import SRF
# from .cloud_masking import Cloud_Mask_Creator # circular dependencies
......@@ -774,20 +775,20 @@ class AtmCorr(object):
@timeout_decorator.timeout(seconds=60*5, timeout_exception=TimeoutError)
def run_request():
try:
t0 = time()
# TODO Implement Lock to avoid too many parallel requests to ECMWF API. This might be an issue in
# TODO multiprocessing.
results = download_variables(date_from=self.inObjs[0].acq_datetime,
date_to=self.inObjs[0].acq_datetime,
db_path=CFG.path_ECMWF_db,
max_step=120, # default
ecmwf_variables=default_products,
processes=0, # singleprocessing
force=False) # dont force download if files already exist
t1 = time()
self.logger.info("Runtime: %.2f" % (t1 - t0))
for result in results:
self.logger.info(result)
with MultiSlotLock('ECMWF download lock', allowed_slots=1, logger=self.logger):
t0 = time()
# NOTE: download_variables does not accept a logger -> so the output may be invisible in WebApp
results = download_variables(date_from=self.inObjs[0].acq_datetime,
date_to=self.inObjs[0].acq_datetime,
db_path=CFG.path_ECMWF_db,
max_step=120, # default
ecmwf_variables=default_products,
processes=0, # singleprocessing
force=False) # dont force download if files already exist
t1 = time()
self.logger.info("Runtime: %.2f" % (t1 - t0))
for result in results:
self.logger.info(result)
except APIKeyFetchError:
self.logger.error("ECMWF data download failed due to missing API credentials.")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment