Commit be56836c authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Added IO locks for array reader and writer.

parent 00c74adf
...@@ -24,6 +24,7 @@ from . import geoprocessing as GEOP ...@@ -24,6 +24,7 @@ from . import geoprocessing as GEOP
from ..io import output_writer as OUT_W from ..io import output_writer as OUT_W
from ..misc import helper_functions as HLP_F from ..misc import helper_functions as HLP_F
from ..misc.definition_dicts import get_outFillZeroSaturated, is_dataset_provided_as_fullScene from ..misc.definition_dicts import get_outFillZeroSaturated, is_dataset_provided_as_fullScene
from ..misc.locks import Lock
from ..model.gms_object import GMS_object from ..model.gms_object import GMS_object
from ..model import metadata as META from ..model import metadata as META
...@@ -175,12 +176,13 @@ class L1A_object(GMS_object): ...@@ -175,12 +176,13 @@ class L1A_object(GMS_object):
rasObj = GEOP.GEOPROCESSING(paths_files2stack[0], self.logger, subset=subset) rasObj = GEOP.GEOPROCESSING(paths_files2stack[0], self.logger, subset=subset)
# perform layer stack # perform layer stack
if CFG.inmem_serialization and path_output is None: # numpy array output with Lock('IO', allowed_threads=1, logger=self.logger): # FIXME hardcoded
self.arr = rasObj.Layerstacking(paths_files2stack) if CFG.inmem_serialization and path_output is None: # numpy array output
self.path_InFilePreprocessor = paths_files2stack[0] self.arr = rasObj.Layerstacking(paths_files2stack)
else: # 'MEMORY' or physical output self.path_InFilePreprocessor = paths_files2stack[0]
rasObj.Layerstacking(paths_files2stack, path_output=path_output) # this writes an output (gdal_merge) else: # 'MEMORY' or physical output
self.arr = path_output rasObj.Layerstacking(paths_files2stack, path_output=path_output) # writes an output (gdal_merge)
self.arr = path_output
else: else:
assert image_files != [], 'No image files found within the archive matching the expected naming scheme.' assert image_files != [], 'No image files found within the archive matching the expected naming scheme.'
...@@ -194,14 +196,16 @@ class L1A_object(GMS_object): ...@@ -194,14 +196,16 @@ class L1A_object(GMS_object):
subset = ['block', [[sub_dim[0], sub_dim[1] + 1], [sub_dim[2], sub_dim[3] + 1]]] subset = ['block', [[sub_dim[0], sub_dim[1] + 1], [sub_dim[2], sub_dim[3] + 1]]]
rasObj = GEOP.GEOPROCESSING(path_file2load, self.logger, subset=subset) rasObj = GEOP.GEOPROCESSING(path_file2load, self.logger, subset=subset)
if CFG.inmem_serialization and path_output is None: # numpy array output # read a single file
self.arr = gdalnumeric.LoadFile(path_file2load) if subset is None else \ with Lock('IO', allowed_threads=1, logger=self.logger): # FIXME hardcoded
gdalnumeric.LoadFile(path_file2load, rasObj.colStart, rasObj.rowStart, rasObj.cols, rasObj.rows) if CFG.inmem_serialization and path_output is None: # numpy array output
self.path_InFilePreprocessor = path_file2load self.arr = gdalnumeric.LoadFile(path_file2load) if subset is None else \
else: # 'MEMORY' or physical output gdalnumeric.LoadFile(path_file2load, rasObj.colStart, rasObj.rowStart, rasObj.cols, rasObj.rows)
GEOP.ndarray2gdal(rasObj.tondarray(), path_output, self.path_InFilePreprocessor = path_file2load
geotransform=rasObj.geotransform, projection=rasObj.projection) else: # 'MEMORY' or physical output
self.arr = path_output GEOP.ndarray2gdal(rasObj.tondarray(), path_output,
geotransform=rasObj.geotransform, projection=rasObj.projection)
self.arr = path_output
os.chdir(project_dir) os.chdir(project_dir)
......
...@@ -15,6 +15,7 @@ from ecmwfapi.api import APIKeyFetchError, get_apikey_values ...@@ -15,6 +15,7 @@ from ecmwfapi.api import APIKeyFetchError, get_apikey_values
from ..options.config import GMS_config as CFG from ..options.config import GMS_config as CFG
from .spatial_index_mediator import SpatialIndexMediatorServer, Connection from .spatial_index_mediator import SpatialIndexMediatorServer, Connection
from .exceptions import GMSEnvironmentError, MissingNonPipLibraryWarning from .exceptions import GMSEnvironmentError, MissingNonPipLibraryWarning
from ..misc.locks import redis_conn
__author__ = 'Daniel Scheffler' __author__ = 'Daniel Scheffler'
...@@ -47,6 +48,11 @@ class GMSEnvironment(object): ...@@ -47,6 +48,11 @@ class GMSEnvironment(object):
self.logger.warning('Coregistration will be skipped!') self.logger.warning('Coregistration will be skipped!')
os.environ['GMS_SPAT_IDX_SRV_STATUS'] = 'unavailable' os.environ['GMS_SPAT_IDX_SRV_STATUS'] = 'unavailable'
def _check_redis_server(self):
if not redis_conn:
self.logger.warning("Unable to connect to redis server. Is the server installed and running? For "
"installation on Ubuntu, use 'sudo apt install redis-server'.")
def _check_nonpip_packages(self): def _check_nonpip_packages(self):
"""Check for not pip-installable packages.""" """Check for not pip-installable packages."""
...@@ -84,6 +90,7 @@ class GMSEnvironment(object): ...@@ -84,6 +90,7 @@ class GMSEnvironment(object):
# javaLibs = [] # javaLibs = []
self._check_spatial_index_mediator_server() self._check_spatial_index_mediator_server()
self._check_redis_server()
self._check_nonpip_packages() self._check_nonpip_packages()
def check_ports(self): def check_ports(self):
......
# -*- coding: utf-8 -*-
__author__ = 'Daniel Scheffler'
import time
import redis_lock
import logging
try:
redis_conn = redis_lock.StrictRedis(host='localhost')
redis_conn.keys() # may raise ConnectionError
except ConnectionError:
redis_conn = None
class Lock(redis_lock.Lock):
def __init__(self, name, allowed_threads=1, logger=None, **kwargs):
self.conn = redis_conn
self.allowed_threads = allowed_threads
self.allowed_slot_names = ['%s, slot #%s' % (name, i) for i in range(1, allowed_threads + 1)]
if redis_conn:
if allowed_threads > 1:
while True:
name_free_slot = self.get_free_slot_name()
if not name_free_slot:
time.sleep(0.2)
else:
break
name = name_free_slot
super().__init__(self.conn, name, **kwargs)
else:
pass
self.name = name
self.logger = logger or logging.getLogger("RedisLock: '%s'" % name)
def get_existing_locks(self):
return [i.decode('utf8').split('lock:')[1] for i in self.conn.keys()]
def get_free_slot_name(self):
free_slots = [sn for sn in self.allowed_slot_names if sn not in self.get_existing_locks()]
if free_slots:
return free_slots[0]
def __enter__(self):
if self.conn:
super().__enter__()
self.logger.info("Acquired lock '%s'." % self.name)
else:
pass
def __exit__(self, exc_type=None, exc_value=None, traceback=None):
if self.conn:
super().__exit__(exc_type=exc_type, exc_value=exc_value, traceback=traceback)
self.logger.info("Released lock '%s'." % self.name)
else:
pass
This diff is collapsed.
...@@ -17,3 +17,4 @@ cerberus ...@@ -17,3 +17,4 @@ cerberus
nested_dict nested_dict
openpyxl openpyxl
timeout_decorator timeout_decorator
redis_lock
...@@ -15,7 +15,7 @@ requirements = [ ...@@ -15,7 +15,7 @@ requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz', 'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz',
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy', 'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.7.1', 'arosics>=0.6.6', 'six', 'tqdm', 'jsmin', 'cerberus', 'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.7.1', 'arosics>=0.6.6', 'six', 'tqdm', 'jsmin', 'cerberus',
'nested_dict', 'openpyxl', 'timeout_decorator' 'nested_dict', 'openpyxl', 'timeout_decorator', 'redis_lock'
# spectral<0.16 has some problems with writing signed integer 8bit data # spectral<0.16 has some problems with writing signed integer 8bit data
# fmask # conda install -c conda-forge python-fmask # fmask # conda install -c conda-forge python-fmask
# 'pyhdf', # conda install --yes -c conda-forge pyhdf # 'pyhdf', # conda install --yes -c conda-forge pyhdf
......
...@@ -74,6 +74,7 @@ dependencies: ...@@ -74,6 +74,7 @@ dependencies:
- nested_dict - nested_dict
- openpyxl - openpyxl
- timeout_decorator - timeout_decorator
- redis_lock
- py_tools_ds>=0.12.4 - py_tools_ds>=0.12.4
- geoarray>=0.7.0 - geoarray>=0.7.0
- arosics>=0.6.6 - arosics>=0.6.6
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment