Commit b9d8c7bb authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Revised environment and spatial_index_mediator modules. Added config parameter...

Revised environment and spatial_index_mediator modules. Added config parameter to skip coregistration.
Former-commit-id: 9d2cbe5c
Former-commit-id: abf5c744
parent 6203b0f8
...@@ -490,7 +490,10 @@ class L1B_object(L1A_object): ...@@ -490,7 +490,10 @@ class L1B_object(L1A_object):
return ref_band4match, shift_band4match return ref_band4match, shift_band4match
def coregister_spatially(self): def coregister_spatially(self):
if self.coreg_needed and self.spatRef_available: if not CFG.job.skip_coreg:
self.logger.warning('Coregistration skipped according to user configuration.')
elif self.coreg_needed and self.spatRef_available:
geoArr_ref = GeoArray(self.spatRef_scene.filePath) geoArr_ref = GeoArray(self.spatRef_scene.filePath)
geoArr_shift = GeoArray(self.arr) geoArr_shift = GeoArray(self.arr)
r_b4match, s_b4match = self.get_opt_bands4matching(target_cwlPos_nm=550, v=False) r_b4match, s_b4match = self.get_opt_bands4matching(target_cwlPos_nm=550, v=False)
......
...@@ -131,6 +131,7 @@ class Job(object): ...@@ -131,6 +131,7 @@ class Job(object):
self.exec_L2AP = exec_L2AP or [1, 1, 1] self.exec_L2AP = exec_L2AP or [1, 1, 1]
self.exec_L2BP = exec_L2BP or [1, 1, 0] self.exec_L2BP = exec_L2BP or [1, 1, 0]
self.exec_L2CP = exec_L2CP or [1, 1, 0] self.exec_L2CP = exec_L2CP or [1, 1, 0]
self.skip_coreg = False
self.validate_exec_configs() self.validate_exec_configs()
self.CPUs = CPUs if CPUs is not None else multiprocessing.cpu_count() self.CPUs = CPUs if CPUs is not None else multiprocessing.cpu_count()
......
...@@ -6,6 +6,7 @@ try: ...@@ -6,6 +6,7 @@ try:
from osgeo import gdal from osgeo import gdal
except ImportError: except ImportError:
import gdal import gdal
from logging import Logger
from ..config import GMS_config as CFG from ..config import GMS_config as CFG
from .spatial_index_mediator import SpatialIndexMediatorServer from .spatial_index_mediator import SpatialIndexMediatorServer
...@@ -14,77 +15,82 @@ from .exceptions import GMSEnvironmentError, MissingNonPipLibraryWarning ...@@ -14,77 +15,82 @@ from .exceptions import GMSEnvironmentError, MissingNonPipLibraryWarning
__author__ = 'Daniel Scheffler' __author__ = 'Daniel Scheffler'
def check_ports(): class GMSEnvironment(object):
# portsDict = { """GeoMultiSens Environment class."""
# 8654: 'SpatialIndexMediator'
# } def __init__(self, logger=Logger(__name__)):
self.logger = logger
pass self.logger.info('Checking system environment...')
def _check_spatial_index_mediator_server(self):
def check_read_write_permissions(): try:
# folders = [] # get instance of SpatialIndexMediatorServer (raises GMSEnvironmentError if bindings are missing)
SpatIdxSrv = SpatialIndexMediatorServer(CFG.job.path_spatIdxSrv, logger=self.logger)
pass
# check if spatial index mediator is running and start if needed
if not SpatIdxSrv.is_running:
def check_dependencies(logger=None): SpatIdxSrv.start()
# pyLibs = []
# javaLibs = [] except GMSEnvironmentError as e:
self.logger.error(e, exc_info=False)
# check if spatial index mediator is running self.logger.warning('Coregistration will be skipped!')
if not CFG.job.is_test: CFG.job.skip_coreg = True
SpatIdxSrv = SpatialIndexMediatorServer(CFG.job.path_spatIdxSrv)
if not SpatIdxSrv.is_running: def _check_nonpip_packages(self):
SpatIdxSrv.start() """Check for not pip-installable packages."""
_log_or_print('Spatial Index Mediator Server started successfully.', logger.info)
# fmask # conda install -c conda-forge python-fmask
if not SpatIdxSrv.is_running: try:
msg = 'Attempt to start Spatial Index Mediator Server failed.' # noinspection PyUnresolvedReferences
_log_or_print(repr(GMSEnvironmentError(msg)), logger.error) import fmask # noqa F401 unused
except ImportError:
# check for not pip-installable packages if 'FMASK' in list(CFG.job.cloud_masking_algorithm.values()):
msg = "FMASK library is not installed because it is not pip-installable and must be installed " \
# fmask # conda install -c conda-forge python-fmask "manually, e.g. for Anaconda by running 'conda install -c conda-forge python-fmask'!"
try: self.logger.warning(MissingNonPipLibraryWarning(msg))
# noinspection PyUnresolvedReferences
import fmask # noqa F401 unused # 'pyhdf', # conda install --yes -c conda-forge pyhdf
except ImportError: try:
if 'FMASK' in list(CFG.job.cloud_masking_algorithm.values()): # noinspection PyUnresolvedReferences
msg = "FMASK library is not installed because it is not pip-installable and must be installed " \ import pyhdf # noqa F401 unused
"manually, e.g. for Anaconda by running 'conda install -c conda-forge python-fmask'!" except ImportError:
_log_or_print(repr(MissingNonPipLibraryWarning(msg)), logger.warning) if gdal.GetDriverByName('HDF4') is None:
msg = "The library 'pyhdf' is missing and the HDF4 driver of GDAL is not available. ASTER data cannot "\
# 'pyhdf', # conda install --yes -c conda-forge pyhdf "be proceessed! For Anaconda, run 'conda install --yes -c conda-forge pyhdf' to fix that!"
try: self.logger.warning(MissingNonPipLibraryWarning(msg))
# noinspection PyUnresolvedReferences
import pyhdf # noqa F401 unused # 'sicor', # pip install git+https://gitext.gfz-potsdam.de/hollstei/sicor.git
except ImportError: try:
if gdal.GetDriverByName('HDF4') is None: # noinspection PyUnresolvedReferences
msg = "The library 'pyhdf' is missing and the HDF4 driver of GDAL is not available. ASTER data cannot be " \ import sicor # noqa F401 unused
"proceessed! For Anaconda, run 'conda install --yes -c conda-forge pyhdf' to fix that!" except ImportError:
_log_or_print(repr(MissingNonPipLibraryWarning(msg)), logger.warning) msg = "The library 'sicor' has not been installed automatically because installation requires login " \
"credentials. See installation instrucions here: https://gitext.gfz-potsdam.de/hollstei/sicor"
# 'sicor', # pip install git+https://gitext.gfz-potsdam.de/hollstei/sicor.git self.logger.warning(MissingNonPipLibraryWarning(msg))
try:
# noinspection PyUnresolvedReferences def check_dependencies(self):
import sicor # noqa F401 unused # pyLibs = []
except ImportError: # javaLibs = []
msg = "The library 'sicor' has not been installed automatically because installation requires login " \
"credentials. See installation instrucions here: https://gitext.gfz-potsdam.de/hollstei/sicor" self._check_spatial_index_mediator_server()
_log_or_print(repr(MissingNonPipLibraryWarning(msg)), logger.warning) self._check_nonpip_packages()
def check_ports(self):
def check_paths(): # TODO to be completed # portsDict = {
# check existance of database paths, etc. # 8654: 'SpatialIndexMediator'
# check existance of cloud classifier dill objects from PG.get_path_cloud_class_obj() # }
if not os.path.exists(CFG.job.path_archive): pass
raise GMSEnvironmentError("The given provider archive path '%s' does not exist!" % CFG.job.path_archive)
def check_read_write_permissions(self):
# folders = []
def _log_or_print(msg, loggerLvl=None):
if loggerLvl: pass
loggerLvl(msg)
else: @staticmethod
print(msg) def check_paths(): # TODO to be completed
# check existance of database paths, etc.
# check existance of cloud classifier dill objects from PG.get_path_cloud_class_obj()
if not os.path.exists(CFG.job.path_archive):
raise GMSEnvironmentError("The given provider archive path '%s' does not exist!" % CFG.job.path_archive)
...@@ -7,13 +7,22 @@ import re ...@@ -7,13 +7,22 @@ import re
import warnings import warnings
from datetime import datetime, timedelta from datetime import datetime, timedelta
from shapely.geometry import Polygon from shapely.geometry import Polygon
from logging import Logger
from ..misc.exceptions import GMSEnvironmentError
class SpatialIndexMediatorServer: class SpatialIndexMediatorServer:
controller = 'index-mediator-server.sh' controller = 'index-mediator-server.sh'
def __init__(self, rootDir): def __init__(self, rootDir, logger=Logger(__name__)):
self.rootDir = rootDir self.rootDir = rootDir
self.path_idxMedSrv = os.path.join(rootDir, self.controller)
self.logger = logger
# validate
if not os.path.isfile(self.path_idxMedSrv):
raise GMSEnvironmentError('File path of index mediator server does not exist at %s.' % self.path_idxMedSrv)
@property @property
def is_running(self): def is_running(self):
...@@ -42,14 +51,16 @@ class SpatialIndexMediatorServer: ...@@ -42,14 +51,16 @@ class SpatialIndexMediatorServer:
def start(self): # FIXME can be executed twice without a message that server is already running def start(self): # FIXME can be executed twice without a message that server is already running
outputStr = self._communicate('start') outputStr = self._communicate('start')
if outputStr == 'success' and self.is_running: if outputStr == 'success' and self.is_running:
self.logger.info('Spatial Index Mediator Server started successfully.')
return 'started' return 'started'
else: else:
if outputStr != 'success': if outputStr != 'success':
warnings.warn("\nStarting Spatial Index Mediator Server failed with message '%s'!" self.logger.warning("\nAttempt to start Spatial Index Mediator Server failed with message '%s'!"
% outputStr.replace('\n', '')) % outputStr.replace('\n', ''))
else: else:
warnings.warn("\nCommunication to Spatial Index Mediator Server was successful but " self.logger.warning("\nCommunication to Spatial Index Mediator Server was successful but "
"the server is still not running. Returned message was: %s" % outputStr.replace('\n', '')) "the server is still not running. Returned message was: %s"
% outputStr.replace('\n', ''))
def stop(self): def stop(self):
outputStr = self._communicate('stop') outputStr = self._communicate('stop')
...@@ -75,7 +86,7 @@ class SpatialIndexMediatorServer: ...@@ -75,7 +86,7 @@ class SpatialIndexMediatorServer:
no_stdout = no_stderr = controller_cmd in ['start', 'restart'] no_stdout = no_stderr = controller_cmd in ['start', 'restart']
# no_stdout = no_stderr = None, None # no_stdout = no_stderr = None, None
from ..misc.helper_functions import subcall_with_output from ..misc.helper_functions import subcall_with_output
output, exitcode, err = subcall_with_output('bash %s %s' % (os.path.join(self.rootDir, self.controller), output, exitcode, err = subcall_with_output('bash %s %s' % (self.path_idxMedSrv,
controller_cmd), no_stdout, no_stderr) controller_cmd), no_stdout, no_stderr)
os.chdir(curdir) os.chdir(curdir)
......
...@@ -87,8 +87,9 @@ class process_controller(object): ...@@ -87,8 +87,9 @@ class process_controller(object):
self.usecase = GMS_config.usecase self.usecase = GMS_config.usecase
# check environment # check environment
self.logger.info('Checking system environment...') GMSEnv = ENV.GMSEnvironment(self.logger)
ENV.check_dependencies(self.logger) GMSEnv.check_dependencies()
# check if process_controller is executed by debugger # check if process_controller is executed by debugger
# isdebugging = 1 if True in [frame[1].endswith("pydevd.py") for frame in inspect.stack()] else False # isdebugging = 1 if True in [frame[1].endswith("pydevd.py") for frame in inspect.stack()] else False
# if isdebugging: # override the existing settings in order to get write access everywhere # if isdebugging: # override the existing settings in order to get write access everywhere
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment