Commit 587ca1d6 authored by Daniel Scheffler's avatar Daniel Scheffler Committed by Mathias Peters
Browse files

Moved environment checks to options.config.set_config(). Revised paths configs...

Moved environment checks to options.config.set_config(). Revised paths configs and removed deprecated paths settings. Added pyopenxl to dependencies.
parent b69b81e3
Pipeline #1805 failed with stage
in 11 minutes and 39 seconds
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
......@@ -309,7 +309,7 @@ if __name__ == '__main__':
#
# else:
# ID = int(sys.argv[1])
#ID = 26187005
# RUN!
parsed_args = get_gms_argparser().parse_args()
parsed_args.func(parsed_args)
......
File mode changed from 100644 to 100755
......@@ -9,7 +9,7 @@ try:
from osgeo import gdal
except ImportError:
import gdal
from logging import Logger
from logging import getLogger
from ..options.config import GMS_config as CFG
from .spatial_index_mediator import SpatialIndexMediatorServer
......@@ -21,8 +21,8 @@ __author__ = 'Daniel Scheffler'
class GMSEnvironment(object):
"""GeoMultiSens Environment class."""
def __init__(self, logger=Logger(__name__)):
self.logger = logger
def __init__(self, logger=None):
self.logger = logger or getLogger('GMSEnvironment')
self.logger.info('Checking system environment...')
self.spatIdxSrvRunning = None
......
......@@ -105,6 +105,17 @@ def set_config(job_ID, json_config='', exec_mode='Python', parallelization_level
kwargs = dict([x for x in locals().items() if x[0] != "self" and not x[0].startswith('__')])
builtins.GMS_JobConfig = JobConfig(job_ID, **kwargs)
#####################
# check environment #
#####################
if not hasattr(builtins, 'GMS_EnvOK') or not getattr(builtins, 'GMS_EnvOK'):
# check environment
from ..misc import environment as ENV
GMSEnv = ENV.GMSEnvironment()
GMSEnv.check_dependencies()
GMSEnv.check_read_write_permissions()
GMSEnv.ensure_properly_activated_GDAL()
return getattr(builtins, 'GMS_JobConfig')
......@@ -207,13 +218,21 @@ class JobConfig(object):
#########
json_paths = json_opts['paths'] # type: dict
# external
self.path_spatIdxSrv = self.DB_config_table['path_spatial_index_mediator_server']
self.path_tempdir = self.absP(self.DB_config_table['path_tempdir'])
self.path_ac_tables = self.absP(self.DB_config_table['path_ac_tables'])
self.path_SNR_models = self.absP(self.DB_config_table['path_SNR_models'])
self.path_custom_sicor_options = gp('path_custom_sicor_options', json_paths['path_custom_sicor_options'])
self.path_dem_proc_srtm_90m = self.absP(self.DB_config_table['path_dem_proc_srtm_90m'])
# internal (included in gms_preprocessing repository)
self.path_spechomo_classif = self.joinP(path_gmslib, 'database', 'spechomo_classifier')
self.path_earthSunDist = self.joinP(path_gmslib, 'database', 'earth_sun_distance',
'Earth_Sun_distances_per_day_edited.csv')
self.path_SNR_models = self.joinP(path_gmslib, 'database', 'snr')
self.path_SRFs = self.joinP(path_gmslib, 'database', 'srf')
self.path_cloud_classif = self.joinP(path_gmslib, 'database', 'cloud_classifier')
self.path_solar_irr = self.joinP(path_gmslib, 'database', 'solar_irradiance',
'SUNp1fontenla__350-2500nm_@0.1nm_converted.txt')
if not self.is_test:
# normal mode
......@@ -231,10 +250,6 @@ class JobConfig(object):
gp('path_procdata_MGRS', json_paths['path_procdata_MGRS'],
fallback=self.joinP(self.path_fileserver, self.DB_config_table['foldername_procdata_MGRS']))
self.path_earthSunDist = self.absP(self.DB_config_table['path_earthSunDist'])
self.path_SRFs = self.absP(self.DB_config_table['path_SRFs'])
self.path_cloud_classif = self.absP(self.DB_config_table['path_cloud_classif'])
self.path_solar_irr = self.absP(self.DB_config_table['path_solar_irr'])
self.path_ECMWF_db = self.absP(self.DB_config_table['path_ECMWF_db'])
self.path_benchmarks = \
......@@ -251,12 +266,6 @@ class JobConfig(object):
self.path_archive = self.joinP(path_gmslib, '..', 'tests', 'data', 'archive_data')
self.path_procdata_scenes = self.joinP(path_gmslib, '..', 'tests', 'data', 'output_scenes')
self.path_procdata_MGRS = self.joinP(path_gmslib, '..', 'tests', 'data', 'output_mgrs_tiles')
self.path_earthSunDist = self.joinP(path_gmslib, 'database', 'earth_sun_distance',
'Earth_Sun_distances_per_day_edited.csv')
self.path_SRFs = self.joinP(path_gmslib, 'database', 'srf')
self.path_cloud_classif = self.joinP(path_gmslib, 'database', 'cloud_classifier')
self.path_solar_irr = self.joinP(path_gmslib, 'database', 'solar_irradiance',
'SUNp1fontenla__350-2500nm_@0.1nm_converted.txt')
self.path_ECMWF_db = self.joinP(path_gmslib, '..', 'tests', 'data', 'processed_ECMWF')
self.path_benchmarks = self.joinP(path_gmslib, '..', 'tests', 'data', 'benchmarks')
self.path_job_logs = self.joinP(path_gmslib, 'logs', 'job_logs')
......
......@@ -26,7 +26,6 @@
"path_benchmarks": "",
"path_job_logs": "", /*output path to store job log files*/
"path_spatIdxSrv": "",
"path_ac_tables": "",
"path_SNR_models": "",
"path_SRFs": "",
"path_dem_proc_srtm_90m": "",
......
......@@ -28,7 +28,6 @@ gms_schema_input = dict(
path_benchmarks=dict(type='string', required=False),
path_job_logs=dict(type='string', required=False),
path_spatIdxSrv=dict(type='string', required=False),
path_ac_tables=dict(type='string', required=False),
path_SNR_models=dict(type='string', required=False),
path_SRFs=dict(type='string', required=False),
path_dem_proc_srtm_90m=dict(type='string', required=False),
......
......@@ -16,7 +16,6 @@ from ..io import output_writer as OUT_W
from ..io import input_reader as INP_R
from ..misc import database_tools as DB_T
from ..misc import helper_functions as HLP_F
from ..misc import environment as ENV
from ..misc.path_generator import path_generator
from ..misc.logging import GMS_logger, shutdown_loggers
from ..algorithms import L1A_P, L1B_P, L1C_P, L2A_P, L2B_P, L2C_P
......@@ -55,7 +54,6 @@ class process_controller(object):
self.config = GMS_config # type: GMS_config
# defaults
self._logger = None
self._DB_job_record = None
self.profiler = None
......@@ -71,12 +69,6 @@ class process_controller(object):
self.summary_detailed = None
self.summary_quick = None
# check environment
self.GMSEnv = ENV.GMSEnvironment(self.logger)
self.GMSEnv.check_dependencies()
self.GMSEnv.check_read_write_permissions()
self.GMSEnv.ensure_properly_activated_GDAL()
# check if process_controller is executed by debugger
# isdebugging = 1 if True in [frame[1].endswith("pydevd.py") for frame in inspect.stack()] else False
# if isdebugging: # override the existing settings in order to get write access everywhere
......
......@@ -15,3 +15,4 @@ tqdm
jsmin
cerberus
nested_dict
openpyxl
......@@ -15,7 +15,7 @@ requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz',
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.7.1', 'arosics>=0.6.6', 'six', 'tqdm', 'jsmin', 'cerberus',
'nested_dict'
'nested_dict', 'openpyxl'
# spectral<0.16 has some problems with writing signed integer 8bit data
# fmask # conda install -c conda-forge python-fmask
# 'pyhdf', # conda install --yes -c conda-forge pyhdf
......
......@@ -72,6 +72,7 @@ dependencies:
- jsmin
- cerberus
- nested_dict
- openpyxl
- py_tools_ds>=0.12.4
- geoarray>=0.7.0
- arosics>=0.6.6
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment