Commit 4a55a1b8 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Fixed file path issues. Bugfixes.


Former-commit-id: 81889e71
parent a2792c94
...@@ -9,6 +9,7 @@ from . import io # noqa: E402 ...@@ -9,6 +9,7 @@ from . import io # noqa: E402
from . import misc # noqa: E402 from . import misc # noqa: E402
from . import processing # noqa: E402 from . import processing # noqa: E402
from . import options # noqa: E402 from . import options # noqa: E402
from .options import config # noqa: E402
from .options.config import set_config # noqa: E402 from .options.config import set_config # noqa: E402
from .processing.process_controller import process_controller # noqa: E402 from .processing.process_controller import process_controller # noqa: E402
...@@ -20,6 +21,7 @@ __all__ = ['algorithms', ...@@ -20,6 +21,7 @@ __all__ = ['algorithms',
'io', 'io',
'misc', 'misc',
'processing', 'processing',
'config' # only to keep compatibility with HU-INF codes
'options', 'options',
'set_config', 'set_config',
'process_controller', 'process_controller',
......
...@@ -43,8 +43,8 @@ class GMS_configuration(object): ...@@ -43,8 +43,8 @@ class GMS_configuration(object):
GMS_config = GMS_configuration() GMS_config = GMS_configuration()
path_options_default = os.path.join(os.path.dirname(pkgutil.get_loader("gms_preprocessing").path), 'options', path_gmslib = os.path.dirname(pkgutil.get_loader("gms_preprocessing").path)
'options_default.json') path_options_default = os.path.join(path_gmslib, 'options', 'options_default.json')
def set_config(job_ID, exec_mode='Python', db_host='localhost', reset=False, json_config='', exec_L1AP=None, def set_config(job_ID, exec_mode='Python', db_host='localhost', reset=False, json_config='', exec_L1AP=None,
...@@ -170,10 +170,10 @@ class JobConfig(object): ...@@ -170,10 +170,10 @@ class JobConfig(object):
json_paths = json_opts['paths'] # type: dict json_paths = json_opts['paths'] # type: dict
self.path_spatIdxSrv = self.DB_config_table['path_spatial_index_mediator_server'] self.path_spatIdxSrv = self.DB_config_table['path_spatial_index_mediator_server']
self.path_tempdir = self.DB_config_table['path_tempdir'] self.path_tempdir = self.absP(self.DB_config_table['path_tempdir'])
self.path_ac_tables = self.DB_config_table['path_ac_tables'] self.path_ac_tables = self.absP(self.DB_config_table['path_ac_tables'])
self.path_SNR_models = self.DB_config_table['path_SNR_models'] self.path_SNR_models = self.absP(self.DB_config_table['path_SNR_models'])
self.path_dem_proc_srtm_90m = self.DB_config_table['path_dem_proc_srtm_90m'] self.path_dem_proc_srtm_90m = self.absP(self.DB_config_table['path_dem_proc_srtm_90m'])
if not self.is_test: if not self.is_test:
# normal mode # normal mode
...@@ -191,33 +191,35 @@ class JobConfig(object): ...@@ -191,33 +191,35 @@ class JobConfig(object):
gp('path_procdata_MGRS', json_paths['path_procdata_MGRS'], gp('path_procdata_MGRS', json_paths['path_procdata_MGRS'],
fallback=self.joinP(self.path_fileserver, self.DB_config_table['foldername_procdata_MGRS'])) fallback=self.joinP(self.path_fileserver, self.DB_config_table['foldername_procdata_MGRS']))
self.path_earthSunDist = self.DB_config_table['path_earthSunDist'] self.path_earthSunDist = self.absP(self.DB_config_table['path_earthSunDist'])
self.path_SRFs = self.DB_config_table['path_SRFs'] self.path_SRFs = self.absP(self.DB_config_table['path_SRFs'])
self.path_cloud_classif = self.DB_config_table['path_cloud_classif'] self.path_cloud_classif = self.absP(self.DB_config_table['path_cloud_classif'])
self.path_solar_irr = self.DB_config_table['path_solar_irr'] self.path_solar_irr = self.absP(self.DB_config_table['path_solar_irr'])
self.path_ECMWF_db = self.DB_config_table['path_ECMWF_db'] self.path_ECMWF_db = self.absP(self.DB_config_table['path_ECMWF_db'])
self.path_benchmarks = \ self.path_benchmarks = \
gp('path_benchmarks', json_paths['path_benchmarks'], gp('path_benchmarks', json_paths['path_benchmarks'],
fallback=self.DB_config_table['path_benchmarks']) fallback=self.absP(self.DB_config_table['path_benchmarks']))
self.path_job_logs = \ self.path_job_logs = \
gp('path_job_logs', json_paths['path_job_logs'], fallback=self.DB_config_table['path_job_logs']) gp('path_job_logs', json_paths['path_job_logs'],
fallback=self.absP(self.DB_config_table['path_job_logs']))
else: else:
# software test mode, the repository should be self-contained -> use only relative paths # software test mode, the repository should be self-contained -> use only relative paths
self.path_fileserver = self.absP('../../tests/data/') self.path_fileserver = self.joinP(path_gmslib, '..', 'tests', 'data')
self.path_archive = self.absP('../../tests/data/archive_data/') self.path_archive = self.joinP(path_gmslib, '..', 'tests', 'data', 'archive_data')
self.path_procdata_scenes = self.absP('../../tests/data/output_scenes/') self.path_procdata_scenes = self.joinP(path_gmslib, '..', 'tests', 'data', 'output_scenes')
self.path_procdata_MGRS = self.absP('../../tests/data/output_mgrs_tiles/') self.path_procdata_MGRS = self.joinP(path_gmslib, '..', 'tests', 'data', 'output_mgrs_tiles')
self.path_earthSunDist = self.absP('../database/earth_sun_distance/Earth_Sun_distances_per_day_edited.csv') self.path_earthSunDist = self.joinP(path_gmslib, 'database', 'earth_sun_distance',
self.path_SRFs = self.absP('../database/srf/') 'Earth_Sun_distances_per_day_edited.csv')
self.path_cloud_classif = self.absP('../database/cloud_classifier/') self.path_SRFs = self.joinP(path_gmslib, 'database', 'srf')
self.path_solar_irr = self.absP( self.path_cloud_classif = self.joinP(path_gmslib, 'database', 'cloud_classifier')
'../database/solar_irradiance/SUNp1fontenla__350-2500nm_@0.1nm_converted.txt') self.path_solar_irr = self.joinP(path_gmslib, 'database', 'solar_irradiance',
self.path_ECMWF_db = self.absP('../../tests/data/processed_ECMWF/') 'SUNp1fontenla__350-2500nm_@0.1nm_converted.txt')
self.path_benchmarks = self.absP('../benchmarks/') self.path_ECMWF_db = self.joinP(path_gmslib, '..', 'tests', 'data', 'processed_ECMWF')
self.path_job_logs = self.absP('../logs/job_logs/') self.path_benchmarks = self.joinP(path_gmslib, '..', 'tests', 'data', 'benchmarks')
self.path_job_logs = self.joinP(path_gmslib, 'logs', 'job_logs')
########################### ###########################
# processor configuration # # processor configuration #
...@@ -251,7 +253,7 @@ class JobConfig(object): ...@@ -251,7 +253,7 @@ class JobConfig(object):
self.SZA_SAA_calculation_accurracy = \ self.SZA_SAA_calculation_accurracy = \
gp('SZA_SAA_calculation_accurracy', json_processors['L1A']['SZA_SAA_calculation_accurracy']) gp('SZA_SAA_calculation_accurracy', json_processors['L1A']['SZA_SAA_calculation_accurracy'])
self.export_VZA_SZA_SAA_RAA_stats = \ self.export_VZA_SZA_SAA_RAA_stats = \
gp('export_VZA_SZA_SAA_RAA_stats', json_processors['L1A']['SZA_SAA_calculation_accurracy']) gp('export_VZA_SZA_SAA_RAA_stats', json_processors['L1A']['export_VZA_SZA_SAA_RAA_stats'])
# L1B # L1B
self.exec_L1BP = gp('exec_L1BP', [ self.exec_L1BP = gp('exec_L1BP', [
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
"exec_mode": "Python", /*"Python" or "Flink"*/ "exec_mode": "Python", /*"Python" or "Flink"*/
"db_host": "localhost", "db_host": "localhost",
"CPUs": "None", /*number of CPU cores to be used for processing (default: "None" -> use all available)*/ "CPUs": "None", /*number of CPU cores to be used for processing (default: "None" -> use all available)*/
"allow_subMultiprocessing": true, /*allow multiprocessing within workers*/ "allow_subMultiprocessing": true, /*allow multiprocessing within multiprocessing workers*/
"disable_exception_handler": false, /*enable/disable automatic handling of unexpected exceptions*/ "disable_exception_handler": false, /*enable/disable automatic handling of unexpected exceptions*/
"log_level": "INFO", /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';*/ "log_level": "INFO", /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';*/
"tiling_block_size_XY": [ "tiling_block_size_XY": [
...@@ -100,7 +100,9 @@ ...@@ -100,7 +100,9 @@
"delete_output": false "delete_output": false
} }
}, },
"usecase": {
"usecase": { /*NOTE: These options will not respected in the WebApp! Use the WebApp GUI instead.*/
"virtual_sensor_id": 10, /*"None": use WebApp input; 1: Landsat-8, 10: Sentinel-2A 10m*/ "virtual_sensor_id": 10, /*"None": use WebApp input; 1: Landsat-8, 10: Sentinel-2A 10m*/
"datasetid_spatial_ref": "None", /*"None": use WebApp input*/ "datasetid_spatial_ref": "None", /*"None": use WebApp input*/
"datasetid_spectral_ref": 249, /*249=Sentinel-2A*/ "datasetid_spectral_ref": 249, /*249=Sentinel-2A*/
......
...@@ -79,7 +79,8 @@ class process_controller(object): ...@@ -79,7 +79,8 @@ class process_controller(object):
self.summary_quick = None self.summary_quick = None
# set GMS configuration # set GMS configuration
set_config(job_ID=job_ID, exec_mode=exec_mode, db_host=db_host, reset=True, **job_config_kwargs) set_config(job_ID=job_ID, exec_mode=exec_mode, db_host=db_host, reset=True,
**job_config_kwargs if job_config_kwargs else {})
self.config = GMS_config self.config = GMS_config
# check environment # check environment
......
...@@ -12,13 +12,12 @@ import unittest ...@@ -12,13 +12,12 @@ import unittest
import os import os
import numpy as np import numpy as np
from gms_preprocessing import __file__ # noqa E402 module level import not at top of file from gms_preprocessing import __path__ # noqa E402 module level import not at top of file
from gms_preprocessing.options.config import set_config # noqa E402 module level import not at top of file from gms_preprocessing import set_config # noqa E402 module level import not at top of file
from gms_preprocessing.algorithms.L2B_P import SpecHomo_Classifier # noqa E402 module level import not at top of file from gms_preprocessing.algorithms.L2B_P import SpecHomo_Classifier # noqa E402 module level import not at top of file
testdata = os.path.join(os.path.dirname(__file__), testdata = os.path.join(__path__[0], '../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq')
'../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq')
class Test_SpecHomo_Classifier(unittest.TestCase): class Test_SpecHomo_Classifier(unittest.TestCase):
...@@ -27,7 +26,7 @@ class Test_SpecHomo_Classifier(unittest.TestCase): ...@@ -27,7 +26,7 @@ class Test_SpecHomo_Classifier(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
# Testjob Landsat-8 # Testjob Landsat-8
set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset=True) set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset=True, is_test=True)
cls.SHC = SpecHomo_Classifier([testdata, testdata, ], v=False) cls.SHC = SpecHomo_Classifier([testdata, testdata, ], v=False)
def test_generate_reference_cube_L8(self): def test_generate_reference_cube_L8(self):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment