Commit abf5c744 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Revised environment and spatial_index_mediator modules. Added config parameter...

Revised environment and spatial_index_mediator modules. Added config parameter to skip coregistration.


Former-commit-id: 9d2cbe5c
parent 07eca007
......@@ -490,7 +490,10 @@ class L1B_object(L1A_object):
return ref_band4match, shift_band4match
def coregister_spatially(self):
if self.coreg_needed and self.spatRef_available:
if not CFG.job.skip_coreg:
self.logger.warning('Coregistration skipped according to user configuration.')
elif self.coreg_needed and self.spatRef_available:
geoArr_ref = GeoArray(self.spatRef_scene.filePath)
geoArr_shift = GeoArray(self.arr)
r_b4match, s_b4match = self.get_opt_bands4matching(target_cwlPos_nm=550, v=False)
......
......@@ -131,6 +131,7 @@ class Job(object):
self.exec_L2AP = exec_L2AP or [1, 1, 1]
self.exec_L2BP = exec_L2BP or [1, 1, 0]
self.exec_L2CP = exec_L2CP or [1, 1, 0]
self.skip_coreg = False
self.validate_exec_configs()
self.CPUs = CPUs if CPUs is not None else multiprocessing.cpu_count()
......
......@@ -6,6 +6,7 @@ try:
from osgeo import gdal
except ImportError:
import gdal
from logging import Logger
from ..config import GMS_config as CFG
from .spatial_index_mediator import SpatialIndexMediatorServer
......@@ -14,77 +15,82 @@ from .exceptions import GMSEnvironmentError, MissingNonPipLibraryWarning
__author__ = 'Daniel Scheffler'
def check_ports():
# portsDict = {
# 8654: 'SpatialIndexMediator'
# }
pass
def check_read_write_permissions():
# folders = []
pass
def check_dependencies(logger=None):
# pyLibs = []
# javaLibs = []
# check if spatial index mediator is running
if not CFG.job.is_test:
SpatIdxSrv = SpatialIndexMediatorServer(CFG.job.path_spatIdxSrv)
if not SpatIdxSrv.is_running:
SpatIdxSrv.start()
_log_or_print('Spatial Index Mediator Server started successfully.', logger.info)
if not SpatIdxSrv.is_running:
msg = 'Attempt to start Spatial Index Mediator Server failed.'
_log_or_print(repr(GMSEnvironmentError(msg)), logger.error)
# check for not pip-installable packages
# fmask # conda install -c conda-forge python-fmask
try:
# noinspection PyUnresolvedReferences
import fmask # noqa F401 unused
except ImportError:
if 'FMASK' in list(CFG.job.cloud_masking_algorithm.values()):
msg = "FMASK library is not installed because it is not pip-installable and must be installed " \
"manually, e.g. for Anaconda by running 'conda install -c conda-forge python-fmask'!"
_log_or_print(repr(MissingNonPipLibraryWarning(msg)), logger.warning)
# 'pyhdf', # conda install --yes -c conda-forge pyhdf
try:
# noinspection PyUnresolvedReferences
import pyhdf # noqa F401 unused
except ImportError:
if gdal.GetDriverByName('HDF4') is None:
msg = "The library 'pyhdf' is missing and the HDF4 driver of GDAL is not available. ASTER data cannot be " \
"proceessed! For Anaconda, run 'conda install --yes -c conda-forge pyhdf' to fix that!"
_log_or_print(repr(MissingNonPipLibraryWarning(msg)), logger.warning)
# 'sicor', # pip install git+https://gitext.gfz-potsdam.de/hollstei/sicor.git
try:
# noinspection PyUnresolvedReferences
import sicor # noqa F401 unused
except ImportError:
msg = "The library 'sicor' has not been installed automatically because installation requires login " \
"credentials. See installation instrucions here: https://gitext.gfz-potsdam.de/hollstei/sicor"
_log_or_print(repr(MissingNonPipLibraryWarning(msg)), logger.warning)
def check_paths(): # TODO to be completed
# check existance of database paths, etc.
# check existance of cloud classifier dill objects from PG.get_path_cloud_class_obj()
if not os.path.exists(CFG.job.path_archive):
raise GMSEnvironmentError("The given provider archive path '%s' does not exist!" % CFG.job.path_archive)
def _log_or_print(msg, loggerLvl=None):
if loggerLvl:
loggerLvl(msg)
else:
print(msg)
class GMSEnvironment(object):
"""GeoMultiSens Environment class."""
def __init__(self, logger=Logger(__name__)):
self.logger = logger
self.logger.info('Checking system environment...')
def _check_spatial_index_mediator_server(self):
try:
# get instance of SpatialIndexMediatorServer (raises GMSEnvironmentError if bindings are missing)
SpatIdxSrv = SpatialIndexMediatorServer(CFG.job.path_spatIdxSrv, logger=self.logger)
# check if spatial index mediator is running and start if needed
if not SpatIdxSrv.is_running:
SpatIdxSrv.start()
except GMSEnvironmentError as e:
self.logger.error(e, exc_info=False)
self.logger.warning('Coregistration will be skipped!')
CFG.job.skip_coreg = True
def _check_nonpip_packages(self):
"""Check for not pip-installable packages."""
# fmask # conda install -c conda-forge python-fmask
try:
# noinspection PyUnresolvedReferences
import fmask # noqa F401 unused
except ImportError:
if 'FMASK' in list(CFG.job.cloud_masking_algorithm.values()):
msg = "FMASK library is not installed because it is not pip-installable and must be installed " \
"manually, e.g. for Anaconda by running 'conda install -c conda-forge python-fmask'!"
self.logger.warning(MissingNonPipLibraryWarning(msg))
# 'pyhdf', # conda install --yes -c conda-forge pyhdf
try:
# noinspection PyUnresolvedReferences
import pyhdf # noqa F401 unused
except ImportError:
if gdal.GetDriverByName('HDF4') is None:
msg = "The library 'pyhdf' is missing and the HDF4 driver of GDAL is not available. ASTER data cannot "\
"be proceessed! For Anaconda, run 'conda install --yes -c conda-forge pyhdf' to fix that!"
self.logger.warning(MissingNonPipLibraryWarning(msg))
# 'sicor', # pip install git+https://gitext.gfz-potsdam.de/hollstei/sicor.git
try:
# noinspection PyUnresolvedReferences
import sicor # noqa F401 unused
except ImportError:
msg = "The library 'sicor' has not been installed automatically because installation requires login " \
"credentials. See installation instrucions here: https://gitext.gfz-potsdam.de/hollstei/sicor"
self.logger.warning(MissingNonPipLibraryWarning(msg))
def check_dependencies(self):
# pyLibs = []
# javaLibs = []
self._check_spatial_index_mediator_server()
self._check_nonpip_packages()
def check_ports(self):
# portsDict = {
# 8654: 'SpatialIndexMediator'
# }
pass
def check_read_write_permissions(self):
# folders = []
pass
@staticmethod
def check_paths(): # TODO to be completed
# check existance of database paths, etc.
# check existance of cloud classifier dill objects from PG.get_path_cloud_class_obj()
if not os.path.exists(CFG.job.path_archive):
raise GMSEnvironmentError("The given provider archive path '%s' does not exist!" % CFG.job.path_archive)
......@@ -7,13 +7,22 @@ import re
import warnings
from datetime import datetime, timedelta
from shapely.geometry import Polygon
from logging import Logger
from ..misc.exceptions import GMSEnvironmentError
class SpatialIndexMediatorServer:
controller = 'index-mediator-server.sh'
def __init__(self, rootDir):
def __init__(self, rootDir, logger=Logger(__name__)):
self.rootDir = rootDir
self.path_idxMedSrv = os.path.join(rootDir, self.controller)
self.logger = logger
# validate
if not os.path.isfile(self.path_idxMedSrv):
raise GMSEnvironmentError('File path of index mediator server does not exist at %s.' % self.path_idxMedSrv)
@property
def is_running(self):
......@@ -42,14 +51,16 @@ class SpatialIndexMediatorServer:
def start(self): # FIXME can be executed twice without a message that server is already running
outputStr = self._communicate('start')
if outputStr == 'success' and self.is_running:
self.logger.info('Spatial Index Mediator Server started successfully.')
return 'started'
else:
if outputStr != 'success':
warnings.warn("\nStarting Spatial Index Mediator Server failed with message '%s'!"
% outputStr.replace('\n', ''))
self.logger.warning("\nAttempt to start Spatial Index Mediator Server failed with message '%s'!"
% outputStr.replace('\n', ''))
else:
warnings.warn("\nCommunication to Spatial Index Mediator Server was successful but "
"the server is still not running. Returned message was: %s" % outputStr.replace('\n', ''))
self.logger.warning("\nCommunication to Spatial Index Mediator Server was successful but "
"the server is still not running. Returned message was: %s"
% outputStr.replace('\n', ''))
def stop(self):
outputStr = self._communicate('stop')
......@@ -75,7 +86,7 @@ class SpatialIndexMediatorServer:
no_stdout = no_stderr = controller_cmd in ['start', 'restart']
# no_stdout = no_stderr = None, None
from ..misc.helper_functions import subcall_with_output
output, exitcode, err = subcall_with_output('bash %s %s' % (os.path.join(self.rootDir, self.controller),
output, exitcode, err = subcall_with_output('bash %s %s' % (self.path_idxMedSrv,
controller_cmd), no_stdout, no_stderr)
os.chdir(curdir)
......
......@@ -87,8 +87,9 @@ class process_controller(object):
self.usecase = GMS_config.usecase
# check environment
self.logger.info('Checking system environment...')
ENV.check_dependencies(self.logger)
GMSEnv = ENV.GMSEnvironment(self.logger)
GMSEnv.check_dependencies()
# check if process_controller is executed by debugger
# isdebugging = 1 if True in [frame[1].endswith("pydevd.py") for frame in inspect.stack()] else False
# if isdebugging: # override the existing settings in order to get write access everywhere
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment