Commit e502df9a authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Replaced dependency redis_lock with redis.

parent be56836c
...@@ -24,7 +24,7 @@ from . import geoprocessing as GEOP ...@@ -24,7 +24,7 @@ from . import geoprocessing as GEOP
from ..io import output_writer as OUT_W from ..io import output_writer as OUT_W
from ..misc import helper_functions as HLP_F from ..misc import helper_functions as HLP_F
from ..misc.definition_dicts import get_outFillZeroSaturated, is_dataset_provided_as_fullScene from ..misc.definition_dicts import get_outFillZeroSaturated, is_dataset_provided_as_fullScene
from ..misc.locks import Lock from ..misc.locks import MultiSlotLock
from ..model.gms_object import GMS_object from ..model.gms_object import GMS_object
from ..model import metadata as META from ..model import metadata as META
...@@ -176,7 +176,7 @@ class L1A_object(GMS_object): ...@@ -176,7 +176,7 @@ class L1A_object(GMS_object):
rasObj = GEOP.GEOPROCESSING(paths_files2stack[0], self.logger, subset=subset) rasObj = GEOP.GEOPROCESSING(paths_files2stack[0], self.logger, subset=subset)
# perform layer stack # perform layer stack
with Lock('IO', allowed_threads=1, logger=self.logger): # FIXME hardcoded with MultiSlotLock('IO', allowed_threads=1, logger=self.logger): # FIXME hardcoded
if CFG.inmem_serialization and path_output is None: # numpy array output if CFG.inmem_serialization and path_output is None: # numpy array output
self.arr = rasObj.Layerstacking(paths_files2stack) self.arr = rasObj.Layerstacking(paths_files2stack)
self.path_InFilePreprocessor = paths_files2stack[0] self.path_InFilePreprocessor = paths_files2stack[0]
...@@ -197,7 +197,7 @@ class L1A_object(GMS_object): ...@@ -197,7 +197,7 @@ class L1A_object(GMS_object):
rasObj = GEOP.GEOPROCESSING(path_file2load, self.logger, subset=subset) rasObj = GEOP.GEOPROCESSING(path_file2load, self.logger, subset=subset)
# read a single file # read a single file
with Lock('IO', allowed_threads=1, logger=self.logger): # FIXME hardcoded with MultiSlotLock('IO', allowed_threads=1, logger=self.logger): # FIXME hardcoded
if CFG.inmem_serialization and path_output is None: # numpy array output if CFG.inmem_serialization and path_output is None: # numpy array output
self.arr = gdalnumeric.LoadFile(path_file2load) if subset is None else \ self.arr = gdalnumeric.LoadFile(path_file2load) if subset is None else \
gdalnumeric.LoadFile(path_file2load, rasObj.colStart, rasObj.rowStart, rasObj.cols, rasObj.rows) gdalnumeric.LoadFile(path_file2load, rasObj.colStart, rasObj.rowStart, rasObj.cols, rasObj.rows)
......
...@@ -2,17 +2,18 @@ ...@@ -2,17 +2,18 @@
__author__ = 'Daniel Scheffler' __author__ = 'Daniel Scheffler'
import time import time
import redis_lock from redis import StrictRedis
from redis.lock import Lock
import logging import logging
try: try:
redis_conn = redis_lock.StrictRedis(host='localhost') redis_conn = StrictRedis(host='localhost')
redis_conn.keys() # may raise ConnectionError redis_conn.keys() # may raise ConnectionError
except ConnectionError: except ConnectionError:
redis_conn = None redis_conn = None
class Lock(redis_lock.Lock): class MultiSlotLock(Lock):
def __init__(self, name, allowed_threads=1, logger=None, **kwargs): def __init__(self, name, allowed_threads=1, logger=None, **kwargs):
self.conn = redis_conn self.conn = redis_conn
self.allowed_threads = allowed_threads self.allowed_threads = allowed_threads
......
...@@ -48,7 +48,7 @@ from ..io import input_reader as INP_R ...@@ -48,7 +48,7 @@ from ..io import input_reader as INP_R
from ..io import output_writer as OUT_W from ..io import output_writer as OUT_W
from ..misc import helper_functions as HLP_F from ..misc import helper_functions as HLP_F
from ..misc import definition_dicts as DEF_D from ..misc import definition_dicts as DEF_D
from ..misc.locks import Lock from ..misc.locks import MultiSlotLock
if TYPE_CHECKING: if TYPE_CHECKING:
from ..algorithms.L1C_P import L1C_object # noqa F401 # flake8 issue from ..algorithms.L1C_P import L1C_object # noqa F401 # flake8 issue
...@@ -1308,7 +1308,7 @@ class GMS_object(Dataset): ...@@ -1308,7 +1308,7 @@ class GMS_object(Dataset):
# loop through all attributes to write and execute writer # # loop through all attributes to write and execute writer #
########################################################### ###########################################################
with Lock('IO', allowed_threads=1, logger=self.logger): with MultiSlotLock('IO', allowed_threads=1, logger=self.logger):
for arrayname in attributes2write: for arrayname in attributes2write:
descriptor = '%s_%s' % (image_type_dict[arrayname], self.proc_level) descriptor = '%s_%s' % (image_type_dict[arrayname], self.proc_level)
......
...@@ -17,4 +17,4 @@ cerberus ...@@ -17,4 +17,4 @@ cerberus
nested_dict nested_dict
openpyxl openpyxl
timeout_decorator timeout_decorator
redis_lock redis
...@@ -15,7 +15,7 @@ requirements = [ ...@@ -15,7 +15,7 @@ requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz', 'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz',
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy', 'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.7.1', 'arosics>=0.6.6', 'six', 'tqdm', 'jsmin', 'cerberus', 'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.7.1', 'arosics>=0.6.6', 'six', 'tqdm', 'jsmin', 'cerberus',
'nested_dict', 'openpyxl', 'timeout_decorator', 'redis_lock' 'nested_dict', 'openpyxl', 'timeout_decorator', 'redis'
# spectral<0.16 has some problems with writing signed integer 8bit data # spectral<0.16 has some problems with writing signed integer 8bit data
# fmask # conda install -c conda-forge python-fmask # fmask # conda install -c conda-forge python-fmask
# 'pyhdf', # conda install --yes -c conda-forge pyhdf # 'pyhdf', # conda install --yes -c conda-forge pyhdf
......
...@@ -74,7 +74,7 @@ dependencies: ...@@ -74,7 +74,7 @@ dependencies:
- nested_dict - nested_dict
- openpyxl - openpyxl
- timeout_decorator - timeout_decorator
- redis_lock - redis
- py_tools_ds>=0.12.4 - py_tools_ds>=0.12.4
- geoarray>=0.7.0 - geoarray>=0.7.0
- arosics>=0.6.6 - arosics>=0.6.6
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment