Commit a16e19c1 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Config revision - intermediate state: JSON file completely parsed.


Former-commit-id: cc4f6a67
parent 068e26cb
...@@ -207,7 +207,6 @@ def get_gms_argparser(): ...@@ -207,7 +207,6 @@ def get_gms_argparser():
'-blocksize': dict(), '-blocksize': dict(),
'-profiling': dict(), '-profiling': dict(),
'-bench_all': dict(), '-bench_all': dict(),
'-bench_cloudMask': dict(),
} }
......
...@@ -148,8 +148,8 @@ class L1C_object(L1B_object): ...@@ -148,8 +148,8 @@ class L1C_object(L1B_object):
meshwidth=10, meshwidth=10,
nodata_mask=None, # dont overwrite areas outside the image with nodata nodata_mask=None, # dont overwrite areas outside the image with nodata
outFill=get_outFillZeroSaturated(np.float32)[0], outFill=get_outFillZeroSaturated(np.float32)[0],
accurracy=CFG.job.SZA_SAA_calculation_accurracy, accurracy=CFG.usecase.SZA_SAA_calculation_accurracy,
lonlat_arr=self.lonlat_arr if CFG.job.SZA_SAA_calculation_accurracy == 'fine' else None) lonlat_arr=self.lonlat_arr if CFG.usecase.SZA_SAA_calculation_accurracy == 'fine' else None)
return self._SZA_arr return self._SZA_arr
@SZA_arr.setter @SZA_arr.setter
...@@ -662,7 +662,7 @@ class AtmCorr(object): ...@@ -662,7 +662,7 @@ class AtmCorr(object):
# compute cloud mask if not already provided # compute cloud mask if not already provided
if no_avail_CMs: if no_avail_CMs:
algorithm = CFG.job.cloud_masking_algorithm[self.inObjs[0].satellite] algorithm = CFG.usecase.cloud_masking_algorithm[self.inObjs[0].satellite]
if algorithm == 'SICOR': if algorithm == 'SICOR':
return None return None
...@@ -792,7 +792,7 @@ class AtmCorr(object): ...@@ -792,7 +792,7 @@ class AtmCorr(object):
script = False script = False
# check if ECMWF data are available - if not, start the download # check if ECMWF data are available - if not, start the download
if CFG.job.auto_download_ecmwf: if CFG.usecase.auto_download_ecmwf:
self._check_or_download_ECMWF_data() self._check_or_download_ECMWF_data()
# validate SNR # validate SNR
......
This diff is collapsed.
...@@ -337,10 +337,10 @@ class SRF(object): ...@@ -337,10 +337,10 @@ class SRF(object):
plt.legend(loc='upper right') plt.legend(loc='upper right')
def pickle_SRF_DB(L1A_Instances): def pickle_SRF_DB(L1A_Instances, dir_out):
list_GMS_identifiers = [i.GMS_identifier for i in L1A_Instances] list_GMS_identifiers = [i.GMS_identifier for i in L1A_Instances]
out_dict = collections.OrderedDict() out_dict = collections.OrderedDict()
logger = GMS_logger('log__SRF2PKL', path_logfile=os.path.join(CFG.job.path_testing, 'out/log__SRF2PKL.log'), logger = GMS_logger('log__SRF2PKL', path_logfile=os.path.join(dir_out, 'log__SRF2PKL.log'),
log_level=CFG.job.log_level, append=False) log_level=CFG.job.log_level, append=False)
for Id, Inst in zip(list_GMS_identifiers, L1A_Instances): for Id, Inst in zip(list_GMS_identifiers, L1A_Instances):
Id['logger'] = logger Id['logger'] = logger
...@@ -348,7 +348,7 @@ def pickle_SRF_DB(L1A_Instances): ...@@ -348,7 +348,7 @@ def pickle_SRF_DB(L1A_Instances):
Inst.satellite + '_' + Inst.sensor + (('_' + Inst.subsystem) if Inst.subsystem not in ['', None] else '')] \ Inst.satellite + '_' + Inst.sensor + (('_' + Inst.subsystem) if Inst.subsystem not in ['', None] else '')] \
= SRF_reader(Id) = SRF_reader(Id)
print(list(out_dict.keys())) print(list(out_dict.keys()))
outFilename = os.path.join(CFG.job.path_testing, 'out/SRF_DB.pkl') outFilename = os.path.join(dir_out, 'SRF_DB.pkl')
with open(outFilename, 'wb') as outFile: with open(outFilename, 'wb') as outFile:
dill.dump(out_dict, outFile) dill.dump(out_dict, outFile)
print('Saved SRF dictionary to %s' % outFilename) print('Saved SRF dictionary to %s' % outFilename)
......
...@@ -610,19 +610,22 @@ class GMS_JOB(object): ...@@ -610,19 +610,22 @@ class GMS_JOB(object):
""" """
:param conn_db: <str> the database connection parameters as given by CFG.job.conn_params :param conn_db: <str> the database connection parameters as given by CFG.job.conn_params
""" """
# privates
self._virtualsensorid = None
# defaults
self.conn = conn_db self.conn = conn_db
self.dataframe = GeoDataFrame() self.dataframe = GeoDataFrame()
self.scene_counts = {} # set by self.create() self.scene_counts = {} # set by self.create()
self.exists_in_db = False self.exists_in_db = False
self.id = None #: int self.id = None #: int
self.creationtime = datetime.now() self.creationtime = datetime.now() # default, needed to create new job
self.finishtime = None self.finishtime = None
self.sceneids = [] self.sceneids = []
self.timerange_start = datetime.min self.timerange_start = datetime.min
self.timerange_end = datetime.max self.timerange_end = datetime.max
self.bounds = box(-180, -90, 180, 90) self.bounds = box(-180, -90, 180, 90) # default, needed to create new job
self.distribution_index = None self.distribution_index = None
self.progress = None self.progress = None
self.feedback = None self.feedback = None
...@@ -633,20 +636,31 @@ class GMS_JOB(object): ...@@ -633,20 +636,31 @@ class GMS_JOB(object):
'timerange_end', 'bounds', 'distribution_index', 'progress', 'feedback', 'timerange_end', 'bounds', 'distribution_index', 'progress', 'feedback',
'failed_sceneids', 'datasetid_spatial_ref', 'failed_sceneids', 'datasetid_spatial_ref',
'virtualsensorid', 'ref_job_id', 'datacube_mgrs_tiles_proc', 'comment', 'virtualsensorid', 'ref_job_id', 'datacube_mgrs_tiles_proc', 'comment',
'status', 'statistics'] 'non_ref_datasetids', 'max_cloudcover', 'season_code', 'status',
self.virtualsensorid = None # set by self._set_target_sensor_specs() 'path_analysis_script', 'analysis_parameter', 'statistics', 'job_mode']
# FIXME notnull but not getable via virtual sensor id and not needed anymore in DB (gsd is given): self.datasetid_spatial_ref = 249 # this is overwritten if existing job is read from DB but needed to create new
self.datasetid_spatial_ref = 249 self.datasetname_spatial_ref = 'SENTINEL-2A' # same here
self.datasetname_spatial_ref = 'SENTINEL-2A'
self.status = None self.status = None
self.statistics = [] self.statistics = []
self.comment = None self.comment = None
self.epsg = None # set by self._set_target_sensor_specs() self.epsg = None # set by self._set_target_sensor_specs()
self.ground_spatial_sampling = None # set by self._set_target_sensor_specs() self.ground_spatial_sampling = None # set by self._set_target_sensor_specs()
self.analysis_parameter = None
def __repr__(self): def __repr__(self):
return 'GMS job:\n\n' + GeoSeries(self.db_entry).to_string() return 'GMS job:\n\n' + GeoSeries(self.db_entry).to_string()
@property
def virtualsensorid(self):
return self._virtualsensorid
@virtualsensorid.setter
def virtualsensorid(self, value):
"""Set virtual sensor ID but continue if no data value is received
NOTE: set by self._set_target_sensor_specs() and self.from_ID()"""
if value != -1: # no data value
self._virtualsensorid = value
def _set_target_sensor_specs(self, virtual_sensor_id, datasetid_spatial_ref): def _set_target_sensor_specs(self, virtual_sensor_id, datasetid_spatial_ref):
self.virtualsensorid = virtual_sensor_id self.virtualsensorid = virtual_sensor_id
res = get_info_from_postgreSQLdb(self.conn, 'virtual_sensors', ['spatial_resolution', res = get_info_from_postgreSQLdb(self.conn, 'virtual_sensors', ['spatial_resolution',
...@@ -671,11 +685,17 @@ class GMS_JOB(object): ...@@ -671,11 +685,17 @@ class GMS_JOB(object):
db_entry = collections.OrderedDict() db_entry = collections.OrderedDict()
for i in self.jobs_table_columns: for i in self.jobs_table_columns:
db_entry[i] = getattr(self, i) val = getattr(self, i)
if i == 'virtualsensorid' and val is None:
val = -1 # nodata value
db_entry[i] = val
return db_entry return db_entry
def from_dictlist(self, dictlist_data2process, virtual_sensor_id, datasetid_spatial_ref=249, comment=None): def from_dictlist(self, dictlist_data2process, virtual_sensor_id, datasetid_spatial_ref=249, comment=None):
# type: (list, int, int, str) -> object # type: (list, int, int, str) -> GMS_JOB
""" """
:param dictlist_data2process: <list> a list of dictionaries containing the keys "satellite", "sensor" and :param dictlist_data2process: <list> a list of dictionaries containing the keys "satellite", "sensor" and
"filenames", "filenames",
...@@ -910,7 +930,7 @@ class GMS_JOB(object): ...@@ -910,7 +930,7 @@ class GMS_JOB(object):
self.timerange_end = self.dataframe.acquisitiondate.max().to_pydatetime() self.timerange_end = self.dataframe.acquisitiondate.max().to_pydatetime()
def from_job_ID(self, job_ID): def from_job_ID(self, job_ID):
# type: (int) -> object # type: (int) -> GMS_JOB
""" """
Create a GMS_JOB instance by querying the database for a specific job ID. Create a GMS_JOB instance by querying the database for a specific job ID.
:param job_ID: <int> a valid id from the database table 'jobs' :param job_ID: <int> a valid id from the database table 'jobs'
......
...@@ -104,7 +104,7 @@ def get_mask_classdefinition(maskname, satellite): ...@@ -104,7 +104,7 @@ def get_mask_classdefinition(maskname, satellite):
'Snow': 60} # SICOR 'Snow': 60} # SICOR
} }
return legends[CFG.job.cloud_masking_algorithm[satellite]] return legends[CFG.usecase.cloud_masking_algorithm[satellite]]
else: else:
raise ValueError("'%s' is not a supported mask name." % maskname) raise ValueError("'%s' is not a supported mask name." % maskname)
......
...@@ -47,7 +47,7 @@ class GMSEnvironment(object): ...@@ -47,7 +47,7 @@ class GMSEnvironment(object):
# noinspection PyUnresolvedReferences # noinspection PyUnresolvedReferences
import fmask # noqa F401 unused import fmask # noqa F401 unused
except ImportError: except ImportError:
if 'FMASK' in list(CFG.job.cloud_masking_algorithm.values()): if 'FMASK' in list(CFG.usecase.cloud_masking_algorithm.values()):
msg = "FMASK library is not installed because it is not pip-installable and must be installed " \ msg = "FMASK library is not installed because it is not pip-installable and must be installed " \
"manually, e.g. for Anaconda by running 'conda install -c conda-forge python-fmask'!" "manually, e.g. for Anaconda by running 'conda install -c conda-forge python-fmask'!"
self.logger.warning(MissingNonPipLibraryWarning(msg)) self.logger.warning(MissingNonPipLibraryWarning(msg))
......
...@@ -302,7 +302,7 @@ class GMS_object(Dataset): ...@@ -302,7 +302,7 @@ class GMS_object(Dataset):
@property @property
def cloud_masking_algorithm(self): def cloud_masking_algorithm(self):
if not self._cloud_masking_algorithm: if not self._cloud_masking_algorithm:
self._cloud_masking_algorithm = CFG.job.cloud_masking_algorithm[self.satellite] self._cloud_masking_algorithm = CFG.usecase.cloud_masking_algorithm[self.satellite]
return self._cloud_masking_algorithm return self._cloud_masking_algorithm
@property @property
......
{ {
"job": { "exec_mode": "Python", /*"Python" or "Flink"*/
"exec_mode": "Python", /*"Python" or "Flink"*/ "db_host": "localhost",
"db_host": "localhost", "CPUs": "None", /*number of CPU cores to be used for processing (default: "None" -> use all available)*/
"CPUs": "None", /*number of CPU cores to be used for processing (default: "None" -> use all available)*/ "allow_subMultiprocessing": true, /*allow multiprocessing within workers*/
"allow_subMultiprocessing": true, /*allow multiprocessing within workers*/ "disable_exception_handler": false, /*enable/disable automatic handling of unexpected exceptions*/
"disable_exception_handler": false, /*enable/disable automatic handling of unexpected exceptions*/ "log_level": "INFO", /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';*/
"log_level": "INFO", /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';*/ "tiling_block_size_XY": [
"tiling_block_size_XY": [ 2048,
2048, 2048
2048 ], /*X/Y block size to be used for any tiling process*/
], /*X/Y block size to be used for any tiling process*/ "is_test": false, /*whether the current job represents a software test job (run by a test runner) or not*/
"is_test": false, /*whether the current job represents a software test job (run by a test runner) or not*/ "profiling": false, /*enable/disable code profiling*/
"profiling": false, /*enable/disable code profiling*/ "benchmark_global": false, /*enable/disable benchmark of the whole processing pipeline*/
"benchmark_global": false, /*enable/disable benchmark of the whole processing pipeline*/ "paths": {
"bench_cloudMask": false, /*enable/disable benchmark of the of the cloud mask generator module*/ "path_fileserver": "",
"paths": { "path_archive": "", /*input path where downloaded data are stored*/
"path_fileserver": "", "path_procdata_scenes": "", /*output path to store processed scenes*/
"path_archive": "", /*input path where downloaded data are stored*/ "path_procdata_MGRS": "", /*output path to store processed MGRS tiles*/
"path_procdata_scenes": "", /*output path to store processed scenes*/ "path_tempdir": "",
"path_procdata_MGRS": "", /*output path to store processed MGRS tiles*/ "path_benchmarks": "",
"path_tempdir": "", "path_job_logs": "", /*output path to store job log files*/
"path_testing": "", "path_spatIdxSrv": "",
"path_benchmarks": "", "path_ac_tables": "",
"path_job_logs": "", /*output path to store job log files*/ "path_SNR_models": "",
"path_spatIdxSrv": "", "path_SRFs": "",
"path_ac_tables": "", "path_dem_proc_srtm_90m": "",
"path_SNR_models": "", "path_earthSunDist": "",
"path_SRFs": "", "path_solar_irr": "",
"path_dem_proc_srtm_90m": "", "path_cloud_classif": "",
"path_earthSunDist": "", "path_ECMWF_db": ""
"path_solar_irr": "", },
"path_cloud_classif": "",
"path_ECMWF_db": "" "processors": {
"general_opts": { /*configuration parameters needed in all processors*/
"skip_thermal": true,
"skip_pan": true,
"sort_bands_by_cwl": true,
"conversion_type_optical": "BOA_Ref", /*'Rad' / 'TOA_Ref' / 'BOA_Ref'*/
"conversion_type_thermal": "Rad", /*'Rad' / 'Temp'*/
"scale_factor_TOARef": 10000,
"scale_factor_BOARef": 10000
}, },
"processors": { "L1A_P": { /*Level 1A processing: Data import and metadata homogenization*/
"L1A_P": { /*Level 1A processing: Data import and metadata homogenization*/ "run_processor": true,
"run_processor": true, "write_output": true,
"write_output": true, "delete_output": true,
"delete_output": true, "SZA_SAA_calculation_accurracy": "coarse", /*"coarse" or "fine"*/
"SZA_SAA_calculation_accurracy": "coarse", /*"coarse" or "fine"*/ "export_VZA_SZA_SAA_RAA_stats": true
"export_VZA_SZA_SAA_RAA_stats": true
}, },
"L1B_P": { /*Level 1B processing: calculation of geometric shift*/ "L1B_P": { /*Level 1B processing: calculation of geometric shift*/
"run_processor": true, "run_processor": true,
"write_output": true, "write_output": true,
"delete_output": true, "delete_output": true,
"skip_coreg": false "skip_coreg": false
}, },
"L1C_P": { /*Level 1C processing: atmospheric correction*/ "L1C_P": { /*Level 1C processing: atmospheric correction*/
"run_processor": true, "run_processor": true,
"write_output": true, "write_output": true,
"delete_output": true, "delete_output": true,
"cloud_masking_algorithm": { "cloud_masking_algorithm": {
"Landsat-4": "FMASK", "Landsat-4": "FMASK",
"Landsat-5": "FMASK", "Landsat-5": "FMASK",
"Landsat-7": "FMASK", "Landsat-7": "FMASK",
"Landsat-8": "FMASK", "Landsat-8": "FMASK",
"Sentinel-2A": "SICOR", "Sentinel-2A": "SICOR",
"Sentinel-2B": "SICOR" "Sentinel-2B": "SICOR"
}, /*'FMASK', 'Classical Bayesian', 'SICOR'*/ }, /*'FMASK', 'Classical Bayesian', 'SICOR'*/
"export_L1C_obj_dumps": false, "export_L1C_obj_dumps": false,
"auto_download_ecmwf": false "scale_factor_errors_ac": 255,
}, "auto_download_ecmwf": false
"L2A_P": { /*Level 2A processing: geometric homogenization*/ },
"run_processor": true, "L2A_P": { /*Level 2A processing: geometric homogenization*/
"write_output": true, "run_processor": true,
"delete_output": true "write_output": true,
}, "delete_output": true,
"L2B_P": { /*Level 2B processing: spectral homogenization*/ "align_coord_grids": true, /*allows to force deactivation of image resampling*/
"run_processor": true, "match_gsd": true
"write_output": true, },
"delete_output": false "L2B_P": { /*Level 2B processing: spectral homogenization*/
}, "run_processor": true,
"L2C_P": { /*Level 2C processing: accurracy assessment and MGRS tiling*/ "write_output": true,
"run_processor": true, "delete_output": false
"write_output": true, },
"delete_output": false "L2C_P": {
} /*Level 2C processing: accurracy assessment and MGRS tiling*/
"run_processor": true,
"write_output": true,
"delete_output": false
} }
}, },
"usecase": { "usecase": {
"skip_thermal": true, "virtual_sensor_id": 10, /*"None": use WebApp input; 1: Landsat-8, 10: Sentinel-2A 10m*/
"skip_pan": true,
"sort_bands_by_cwl": true,
"conversion_type_optical": "BOA_Ref", /*'Rad' / 'TOA_Ref' / 'BOA_Ref'*/
"conversion_type_thermal": "Rad", /*'Rad' / 'Temp'*/
"scale_factor_TOARef": 10000,
"scale_factor_BOARef": 10000,
"scale_factor_errors_ac": 255,
"virtual_sensor_id": "None", /*"None": use WebApp input; 1: Landsat-8, 10: Sentinel-2A 10m*/
"virtual_sensor_name": "None", /*"None": use WebApp input*/
"datasetid_spatial_ref": "None", /*"None": use WebApp input*/ "datasetid_spatial_ref": "None", /*"None": use WebApp input*/
"datasetid_spectral_ref": 249, /*249=Sentinel-2A*/ "datasetid_spectral_ref": 249, /*249=Sentinel-2A*/
"target_CWL": [ "target_CWL": [
...@@ -108,8 +109,6 @@ ...@@ -108,8 +109,6 @@
], /*target sensor x-coordinate-grid. e.g. [15, 45]*/ ], /*target sensor x-coordinate-grid. e.g. [15, 45]*/
"spatial_ref_gridy": [ "spatial_ref_gridy": [
], /*target sensor y-coordinate-grid. e.g. [15, 45]*/ ] /*target sensor y-coordinate-grid. e.g. [15, 45]*/
"align_coord_grids": true, /*allows to force deactivation of image resampling*/
"match_gsd": true
} }
} }
"""Definition of gms options schema (as used by cerberus library)."""
gms_schema = {
}
...@@ -11,8 +11,9 @@ Tests for gms_preprocessing.config ...@@ -11,8 +11,9 @@ Tests for gms_preprocessing.config
import os import os
from unittest import TestCase from unittest import TestCase
from gms_preprocessing.config import get_options
from gms_preprocessing import __path__ from gms_preprocessing import __path__
from gms_preprocessing.config import get_options
from gms_preprocessing.config import JobConfig
class Test_get_options(TestCase): class Test_get_options(TestCase):
...@@ -23,3 +24,13 @@ class Test_get_options(TestCase): ...@@ -23,3 +24,13 @@ class Test_get_options(TestCase):
def test_target_is_file_validation(self): def test_target_is_file_validation(self):
opts_dict = get_options(os.path.join(__path__[0], 'options_default.json')) opts_dict = get_options(os.path.join(__path__[0], 'options_default.json'))
self.assertIsInstance(opts_dict, dict) self.assertIsInstance(opts_dict, dict)
class Test_JobConfig(TestCase):
def setUp(self):
self.jobID = 26186662
self.db_host = 'localhost'
def test(self):
cfg = JobConfig(self.jobID, self.db_host)
print(cfg)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment