Commit 5c79b842 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Revised config and fixed unexpected behaviour of CLI parser (parameters did...

Revised config and fixed unexpected behaviour of CLI parser (parameters did not override  previously set WebUI configuration).
parent eb8781b1
......@@ -14,12 +14,41 @@ from gms_preprocessing.misc.database_tools import GMS_JOB # noqa: E402
from gms_preprocessing.options.config import get_conn_database # noqa: E402
from gms_preprocessing.options.config import path_options_default # noqa: E402
from gms_preprocessing.options.config import get_options # noqa: E402
from gms_preprocessing.options.options_schema import get_param_from_json_config # noqa: E402
options_default = get_options(path_options_default, validation=True) # type: dict
def parsedArgs_to_user_opts(cli_args):
return {k: v for k, v in vars(cli_args).items() if not k.startswith('_') and k != 'func'}
# type: (argparse.Namespace) -> dict
"""Convert argparse Namespace object to dictionary of explicitly given parameters.
NOTE: All options that have not been given explicitly (None values) are removed. Reason: all options to
passed set_config WILL OVERRIDE job settings read from the GMS database (e.g., specified by the WebUI)
=> only override job configuration defined by WebUI if CLI options are explicitly given
=> if json_opts are given: configuration defined by WebUI will be overridden by this json config in any case
:param cli_args: options as parsed by the argparse.ArgumentParser
"""
# convert argparse Namespace object to dictionary
opts = {k: v for k, v in vars(cli_args).items() if not k.startswith('_') and k != 'func'}
# remove those options that have not been given explicitly (None values)
user_opts = dict()
for k, v in opts.items():
# values are None if they are not given by the user -> don't pass to set_config
if v is None:
continue
# remove keys that are not to be passed to set_config
elif k in ['jobid', 'sceneids', 'entityids', 'filenames', 'comment', ]:
continue
else:
user_opts.update({k: v})
return user_opts
def run_from_jobid(args):
......@@ -52,8 +81,8 @@ def run_from_sceneids(args):
# create a new processing job from scene IDs
dbJob = GMS_JOB(get_conn_database(args.db_host))
dbJob.from_sceneIDlist(list_sceneIDs=args.sceneids,
virtual_sensor_id=args.virtual_sensor_id,
datasetid_spatial_ref=args.datasetid_spatial_ref,
virtual_sensor_id=get_user_input_or_default('virtual_sensor_id', args),
datasetid_spatial_ref=get_user_input_or_default('datasetid_spatial_ref', args),
comment=args.comment)
_run_job(dbJob, **parsedArgs_to_user_opts(args))
......@@ -66,8 +95,8 @@ def run_from_entityids(args):
"""
dbJob = GMS_JOB(get_conn_database(args.db_host))
dbJob.from_entityIDlist(list_entityids=args.entityids,
virtual_sensor_id=args.virtual_sensor_id,
datasetid_spatial_ref=args.datasetid_spatial_ref,
virtual_sensor_id=get_user_input_or_default('virtual_sensor_id', args),
datasetid_spatial_ref=get_user_input_or_default('datasetid_spatial_ref', args),
comment=args.comment)
_run_job(dbJob, **parsedArgs_to_user_opts(args))
......@@ -80,8 +109,8 @@ def run_from_filenames(args):
"""
dbJob = GMS_JOB(get_conn_database(args.db_host))
dbJob.from_filenames(list_filenames=args.filenames,
virtual_sensor_id=args.virtual_sensor_id,
datasetid_spatial_ref=args.datasetid_spatial_ref,
virtual_sensor_id=get_user_input_or_default('virtual_sensor_id', args),
datasetid_spatial_ref=get_user_input_or_default('datasetid_spatial_ref', args),
comment=args.comment)
_run_job(dbJob, **parsedArgs_to_user_opts(args))
......@@ -117,6 +146,13 @@ def _run_job(dbJob, **config_kwargs):
PC.run_all_processors()
def get_user_input_or_default(paramname, argsparse_ns):
user_input = getattr(argsparse_ns, paramname)
return user_input if user_input is not None else \
get_param_from_json_config(paramname, options_default)
def get_gms_argparser():
"""Return argument parser for run_gms.py program."""
......@@ -157,20 +193,19 @@ def get_gms_argparser():
default='localhost', # hardcoded here because default json is read from database and host must be available
help='host name of the server that runs the postgreSQL database')
gop_p('-DOO', '--delete_old_output', nargs='?', type=bool,
default=options_default["global_opts"]["delete_old_output"],
# NOTE: don't define any defaults here for parameters that are passed to set_config!
# -> otherwise, we cannot distinguish between explicity given parameters and default values
# => see docs in parsedArgs_to_user_opts() for explanation
gop_p('-DOO', '--delete_old_output', nargs='?', type=bool, default=None,
help='delete previously created output of the given job ID before running the job')
gop_p('-vid', '--virtual_sensor_id', type=int,
default=options_default["usecase"]["virtual_sensor_id"],
gop_p('-vid', '--virtual_sensor_id', type=int, default=None,
help='ID of the target (virtual) sensor')
gop_p('-dsid_spat', '--datasetid_spatial_ref', type=int,
default=options_default["usecase"]["datasetid_spatial_ref"],
gop_p('-dsid_spat', '--datasetid_spatial_ref', type=int, default=None,
help='dataset ID of the spatial reference')
gop_p('--CPUs', type=int,
default=options_default["global_opts"]["CPUs"],
gop_p('--CPUs', type=int, default=None,
help='number of CPU cores to be used for processing (default: "None" -> use all available')
gop_p('-c', '--comment', nargs='?', type=str,
......
......@@ -11,7 +11,7 @@ import psycopg2
import psycopg2.extras
from collections import OrderedDict, Mapping
import multiprocessing
from inspect import getargvalues, stack, getfullargspec, signature, _empty
from inspect import getargvalues, stack, signature, _empty
import json
from json import JSONDecodeError
from jsmin import jsmin
......@@ -23,7 +23,7 @@ import psutil
from pprint import pformat
from typing import TYPE_CHECKING
from .options_schema import gms_schema_input, gms_schema_config_output
from .options_schema import gms_schema_input, gms_schema_config_output, parameter_mapping
from ..version import __version__, __versionalias__
if TYPE_CHECKING:
......@@ -62,6 +62,8 @@ def set_config(job_ID, json_config='', reset_status=False, **kwargs):
:param reset_status: whether to reset the job status or not (default=False)
:param kwargs: keyword arguments to be passed to JobConfig
NOTE: All keyword arguments given here WILL OVERRIDE configurations that have been previously set via WebUI!
:Keyword Arguments:
- inmem_serialization: False: write intermediate results to disk in order to save memory
True: keep intermediate results in memory in order to save IO time
......@@ -101,6 +103,12 @@ def set_config(job_ID, json_config='', reset_status=False, **kwargs):
- datasetid_spatial_ref: 249 Sentinel-2A
:rtype: JobConfig
"""
# validate kwargs
for k in kwargs:
if k not in parameter_mapping and k != 'db_host':
raise ValueError("'%s' is not a valid parameter. Valid parameters are: \n%s"
% (k, list(parameter_mapping.keys())))
#################################
# set GMS_JobConfig in builtins #
#################################
......@@ -143,11 +151,6 @@ def get_conn_database(hostname='localhost', timeout=3):
% (hostname, timeout)
def get_config_kwargs_default():
a = getfullargspec(set_config)
return dict(zip(a.args[-len(a.defaults):], a.defaults))
class JobConfig(object):
def __init__(self, ID, json_config='', **user_opts):
"""Create a job configuration
......
......@@ -167,3 +167,131 @@ def get_updated_schema(source_schema, key2update, new_value):
gms_schema_config_output = get_updated_schema(gms_schema_input, key2update='required', new_value=True)
parameter_mapping = dict(
# global opts
inmem_serialization=('global_opts', 'inmem_serialization'),
parallelization_level=('global_opts', 'parallelization_level'),
spatial_index_server_host=('global_opts', 'spatial_index_server_host'),
spatial_index_server_port=('global_opts', 'spatial_index_server_port'),
CPUs=('global_opts', 'CPUs'),
CPUs_all_jobs=('global_opts', 'CPUs_all_jobs'),
max_mem_usage=('global_opts', 'max_mem_usage'),
critical_mem_usage=('global_opts', 'critical_mem_usage'),
max_parallel_reads_writes=('global_opts', 'max_parallel_reads_writes'),
allow_subMultiprocessing=('global_opts', 'allow_subMultiprocessing'),
delete_old_output=('global_opts', 'delete_old_output'),
disable_exception_handler=('global_opts', 'disable_exception_handler'),
disable_IO_locks=('global_opts', 'disable_IO_locks'),
disable_CPU_locks=('global_opts', 'disable_CPU_locks'),
disable_memory_locks=('global_opts', 'disable_memory_locks'),
min_version_mem_usage_stats=('global_opts', 'min_version_mem_usage_stats'),
log_level=('global_opts', 'log_level'),
tiling_block_size_XY=('global_opts', 'tiling_block_size_XY'),
is_test=('global_opts', 'is_test'),
profiling=('global_opts', 'profiling'),
benchmark_global=('global_opts', 'benchmark_global'),
# paths
path_fileserver=('paths', 'path_fileserver'),
path_archive=('paths', 'path_archive'),
path_procdata_scenes=('paths', 'path_procdata_scenes'),
path_procdata_MGRS=('paths', 'path_procdata_MGRS'),
path_tempdir=('paths', 'path_tempdir'),
path_benchmarks=('paths', 'path_benchmarks'),
path_job_logs=('paths', 'path_job_logs'),
path_spatIdxSrv=('paths', 'path_spatIdxSrv'),
path_SNR_models=('paths', 'path_SNR_models'),
path_SRFs=('paths', 'path_SRFs'),
path_dem_proc_srtm_90m=('paths', 'path_dem_proc_srtm_90m'),
path_earthSunDist=('paths', 'path_earthSunDist'),
path_solar_irr=('paths', 'path_solar_irr'),
path_cloud_classif=('paths', 'path_cloud_classif'),
path_custom_sicor_options=('paths', 'path_custom_sicor_options'),
path_ECMWF_db=('paths', 'path_ECMWF_db'),
path_spechomo_classif=('paths', 'path_spechomo_classif'),
# processors > general opts
skip_thermal=('processors', 'general_opts', 'skip_thermal'),
skip_pan=('processors', 'general_opts', 'skip_pan'),
sort_bands_by_cwl=('processors', 'general_opts', 'sort_bands_by_cwl'),
target_radunit_optical=('processors', 'general_opts', 'target_radunit_optical'),
target_radunit_thermal=('processors', 'general_opts', 'target_radunit_thermal'),
scale_factor_TOARef=('processors', 'general_opts', 'scale_factor_TOARef'),
scale_factor_BOARef=('processors', 'general_opts', 'scale_factor_BOARef'),
mgrs_pixel_buffer=('processors', 'general_opts', 'mgrs_pixel_buffer'),
output_data_compression=('processors', 'general_opts', 'output_data_compression'),
write_ENVIclassif_cloudmask=('processors', 'general_opts', 'write_ENVIclassif_cloudmask'),
# processors > L1A
exec_L1AP=('processors', 'L1A', ['run_processor', 'write_output', 'delete_output']),
SZA_SAA_calculation_accurracy=('processors', 'L1A', 'SZA_SAA_calculation_accurracy'),
export_VZA_SZA_SAA_RAA_stats=('processors', 'L1A', 'export_VZA_SZA_SAA_RAA_stats'),
# processors > L1B
exec_L1BP=('processors', 'L1B', ['run_processor', 'write_output', 'delete_output']),
skip_coreg=('processors', 'L1B', 'skip_coreg'),
spatial_ref_min_overlap=('processors', 'L1B', 'spatial_ref_min_overlap'),
spatial_ref_max_cloudcov=('processors', 'L1B', 'spatial_ref_max_cloudcov'),
spatial_ref_plusminus_days=('processors', 'L1B', 'spatial_ref_plusminus_days'),
spatial_ref_plusminus_years=('processors', 'L1B', 'spatial_ref_plusminus_years'),
coreg_band_wavelength_for_matching=('processors', 'L1B', 'coreg_band_wavelength_for_matching'),
coreg_max_shift_allowed=('processors', 'L1B', 'coreg_max_shift_allowed'),
coreg_window_size=('processors', 'L1B', 'coreg_window_size'),
# processors > L1C
exec_L1CP=('processors', 'L1C', ['run_processor', 'write_output', 'delete_output']),
cloud_masking_algorithm=('processors', 'L1C', 'cloud_masking_algorithm'),
export_L1C_obj_dumps=('processors', 'L1C', 'export_L1C_obj_dumps'),
auto_download_ecmwf=('processors', 'L1C', 'auto_download_ecmwf'),
ac_fillnonclear_areas=('processors', 'L1C', 'ac_fillnonclear_areas'),
ac_clear_area_labels=('processors', 'L1C', 'ac_clear_area_labels'),
ac_scale_factor_errors=('processors', 'L1C', 'ac_scale_factor_errors'),
ac_max_ram_gb=('processors', 'L1C', 'ac_max_ram_gb'),
ac_estimate_accuracy=('processors', 'L1C', 'ac_estimate_accuracy'),
ac_bandwise_accuracy=('processors', 'L1C', 'ac_bandwise_accuracy'),
# processors > L2A
exec_L2AP=('processors', 'L2A', ['run_processor', 'write_output', 'delete_output']),
align_coord_grids=('processors', 'L2A', 'align_coord_grids'),
match_gsd=('processors', 'L2A', 'match_gsd'),
spatial_resamp_alg=('processors', 'L2A', 'spatial_resamp_alg'),
clip_to_extent=('processors', 'L2A', 'clip_to_extent'),
spathomo_estimate_accuracy=('processors', 'L2A', 'spathomo_estimate_accuracy'),
# processors > L2B
exec_L2BP=('processors', 'L2B', ['run_processor', 'write_output', 'delete_output']),
spechomo_method=('processors', 'L2B', 'spechomo_method'),
spechomo_estimate_accuracy=('processors', 'L2B', 'spechomo_estimate_accuracy'),
spechomo_bandwise_accuracy=('processors', 'L2B', 'spechomo_bandwise_accuracy'),
# processors > L2C
exec_L2CP=('processors', 'L2C', ['run_processor', 'write_output', 'delete_output']),
# usecase
virtual_sensor_id=('usecase', 'virtual_sensor_id'),
datasetid_spatial_ref=('usecase', 'datasetid_spatial_ref'),
virtual_sensor_name=('usecase', 'virtual_sensor_name'),
datasetid_spectral_ref=('usecase', 'datasetid_spectral_ref'),
target_CWL=('usecase', 'target_CWL'),
target_FWHM=('usecase', 'target_FWHM'),
target_gsd=('usecase', 'target_gsd'),
target_epsg_code=('usecase', 'target_epsg_code'),
spatial_ref_gridx=('usecase', 'spatial_ref_gridx'),
spatial_ref_gridy=('usecase', 'spatial_ref_gridy'),
)
def get_param_from_json_config(paramname, json_config):
keymap = parameter_mapping[paramname] # tuple
dict2search = json_config
for i, k in enumerate(keymap):
if i < len(keymap) - 1:
# not the last element of the tuple -> contains a sub-dictionary
dict2search = dict2search[k]
elif isinstance(k, list):
return [dict2search[sk] for sk in k]
else:
return dict2search[k]
......@@ -63,6 +63,10 @@ class Base_CLITester:
parsed_args.func(parsed_args)
self.assertEqual(self.current_CFG.virtual_sensor_id, vid)
# test if parameter fallbacks are working ('CPUs' has a fallback)
self.assertNotEquals(self.current_CFG.CPUs, None)
self.assertNotIsInstance(self.current_CFG.CPUs, str)
def test_json_opts(self):
parsed_args = self.parser_run.parse_args(
self.baseargs + ['--db_host', db_host,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment