Commit 6bb0a829 authored by Daniel Scheffler's avatar Daniel Scheffler Committed by Mathias Peters
Browse files

Moved environment checks to options.config.set_config(). Revised paths configs...

Moved environment checks to options.config.set_config(). Revised paths configs and removed deprecated paths settings. Added pyopenxl to dependencies.


Former-commit-id: 587ca1d6
parent 26f23fd8
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
......@@ -309,7 +309,7 @@ if __name__ == '__main__':
#
# else:
# ID = int(sys.argv[1])
#ID = 26187005
# RUN!
parsed_args = get_gms_argparser().parse_args()
parsed_args.func(parsed_args)
......
File mode changed from 100644 to 100755
......@@ -9,7 +9,7 @@ try:
from osgeo import gdal
except ImportError:
import gdal
from logging import Logger
from logging import getLogger
from ..options.config import GMS_config as CFG
from .spatial_index_mediator import SpatialIndexMediatorServer
......@@ -21,8 +21,8 @@ __author__ = 'Daniel Scheffler'
class GMSEnvironment(object):
"""GeoMultiSens Environment class."""
def __init__(self, logger=Logger(__name__)):
self.logger = logger
def __init__(self, logger=None):
self.logger = logger or getLogger('GMSEnvironment')
self.logger.info('Checking system environment...')
self.spatIdxSrvRunning = None
......
......@@ -105,6 +105,17 @@ def set_config(job_ID, json_config='', exec_mode='Python', parallelization_level
kwargs = dict([x for x in locals().items() if x[0] != "self" and not x[0].startswith('__')])
builtins.GMS_JobConfig = JobConfig(job_ID, **kwargs)
#####################
# check environment #
#####################
if not hasattr(builtins, 'GMS_EnvOK') or not getattr(builtins, 'GMS_EnvOK'):
# check environment
from ..misc import environment as ENV
GMSEnv = ENV.GMSEnvironment()
GMSEnv.check_dependencies()
GMSEnv.check_read_write_permissions()
GMSEnv.ensure_properly_activated_GDAL()
return getattr(builtins, 'GMS_JobConfig')
......@@ -207,13 +218,21 @@ class JobConfig(object):
#########
json_paths = json_opts['paths'] # type: dict
# external
self.path_spatIdxSrv = self.DB_config_table['path_spatial_index_mediator_server']
self.path_tempdir = self.absP(self.DB_config_table['path_tempdir'])
self.path_ac_tables = self.absP(self.DB_config_table['path_ac_tables'])
self.path_SNR_models = self.absP(self.DB_config_table['path_SNR_models'])
self.path_custom_sicor_options = gp('path_custom_sicor_options', json_paths['path_custom_sicor_options'])
self.path_dem_proc_srtm_90m = self.absP(self.DB_config_table['path_dem_proc_srtm_90m'])
# internal (included in gms_preprocessing repository)
self.path_spechomo_classif = self.joinP(path_gmslib, 'database', 'spechomo_classifier')
self.path_earthSunDist = self.joinP(path_gmslib, 'database', 'earth_sun_distance',
'Earth_Sun_distances_per_day_edited.csv')
self.path_SNR_models = self.joinP(path_gmslib, 'database', 'snr')
self.path_SRFs = self.joinP(path_gmslib, 'database', 'srf')
self.path_cloud_classif = self.joinP(path_gmslib, 'database', 'cloud_classifier')
self.path_solar_irr = self.joinP(path_gmslib, 'database', 'solar_irradiance',
'SUNp1fontenla__350-2500nm_@0.1nm_converted.txt')
if not self.is_test:
# normal mode
......@@ -231,10 +250,6 @@ class JobConfig(object):
gp('path_procdata_MGRS', json_paths['path_procdata_MGRS'],
fallback=self.joinP(self.path_fileserver, self.DB_config_table['foldername_procdata_MGRS']))
self.path_earthSunDist = self.absP(self.DB_config_table['path_earthSunDist'])
self.path_SRFs = self.absP(self.DB_config_table['path_SRFs'])
self.path_cloud_classif = self.absP(self.DB_config_table['path_cloud_classif'])
self.path_solar_irr = self.absP(self.DB_config_table['path_solar_irr'])
self.path_ECMWF_db = self.absP(self.DB_config_table['path_ECMWF_db'])
self.path_benchmarks = \
......@@ -251,12 +266,6 @@ class JobConfig(object):
self.path_archive = self.joinP(path_gmslib, '..', 'tests', 'data', 'archive_data')
self.path_procdata_scenes = self.joinP(path_gmslib, '..', 'tests', 'data', 'output_scenes')
self.path_procdata_MGRS = self.joinP(path_gmslib, '..', 'tests', 'data', 'output_mgrs_tiles')
self.path_earthSunDist = self.joinP(path_gmslib, 'database', 'earth_sun_distance',
'Earth_Sun_distances_per_day_edited.csv')
self.path_SRFs = self.joinP(path_gmslib, 'database', 'srf')
self.path_cloud_classif = self.joinP(path_gmslib, 'database', 'cloud_classifier')
self.path_solar_irr = self.joinP(path_gmslib, 'database', 'solar_irradiance',
'SUNp1fontenla__350-2500nm_@0.1nm_converted.txt')
self.path_ECMWF_db = self.joinP(path_gmslib, '..', 'tests', 'data', 'processed_ECMWF')
self.path_benchmarks = self.joinP(path_gmslib, '..', 'tests', 'data', 'benchmarks')
self.path_job_logs = self.joinP(path_gmslib, 'logs', 'job_logs')
......
......@@ -26,7 +26,6 @@
"path_benchmarks": "",
"path_job_logs": "", /*output path to store job log files*/
"path_spatIdxSrv": "",
"path_ac_tables": "",
"path_SNR_models": "",
"path_SRFs": "",
"path_dem_proc_srtm_90m": "",
......
......@@ -28,7 +28,6 @@ gms_schema_input = dict(
path_benchmarks=dict(type='string', required=False),
path_job_logs=dict(type='string', required=False),
path_spatIdxSrv=dict(type='string', required=False),
path_ac_tables=dict(type='string', required=False),
path_SNR_models=dict(type='string', required=False),
path_SRFs=dict(type='string', required=False),
path_dem_proc_srtm_90m=dict(type='string', required=False),
......
......@@ -16,7 +16,6 @@ from ..io import output_writer as OUT_W
from ..io import input_reader as INP_R
from ..misc import database_tools as DB_T
from ..misc import helper_functions as HLP_F
from ..misc import environment as ENV
from ..misc.path_generator import path_generator
from ..misc.logging import GMS_logger, shutdown_loggers
from ..algorithms import L1A_P, L1B_P, L1C_P, L2A_P, L2B_P, L2C_P
......@@ -55,7 +54,6 @@ class process_controller(object):
self.config = GMS_config # type: GMS_config
# defaults
self._logger = None
self._DB_job_record = None
self.profiler = None
......@@ -71,12 +69,6 @@ class process_controller(object):
self.summary_detailed = None
self.summary_quick = None
# check environment
self.GMSEnv = ENV.GMSEnvironment(self.logger)
self.GMSEnv.check_dependencies()
self.GMSEnv.check_read_write_permissions()
self.GMSEnv.ensure_properly_activated_GDAL()
# check if process_controller is executed by debugger
# isdebugging = 1 if True in [frame[1].endswith("pydevd.py") for frame in inspect.stack()] else False
# if isdebugging: # override the existing settings in order to get write access everywhere
......
......@@ -15,3 +15,4 @@ tqdm
jsmin
cerberus
nested_dict
openpyxl
......@@ -15,7 +15,7 @@ requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz',
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.7.1', 'arosics>=0.6.6', 'six', 'tqdm', 'jsmin', 'cerberus',
'nested_dict'
'nested_dict', 'openpyxl'
# spectral<0.16 has some problems with writing signed integer 8bit data
# fmask # conda install -c conda-forge python-fmask
# 'pyhdf', # conda install --yes -c conda-forge pyhdf
......
......@@ -72,6 +72,7 @@ dependencies:
- jsmin
- cerberus
- nested_dict
- openpyxl
- py_tools_ds>=0.12.4
- geoarray>=0.7.0
- arosics>=0.6.6
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment