Commit f347434a authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Revised command line argparser.


Former-commit-id: f85d2a7b
parent 45b3a414
......@@ -10,6 +10,17 @@ matplotlib.use('Agg', warn=False) # switch matplotlib backend to 'Agg' and disa
from gms_preprocessing import process_controller, __version__ # noqa: E402
from gms_preprocessing.misc.database_tools import GMS_JOB # noqa: E402
from gms_preprocessing.options.config import get_conn_database # noqa: E402
from gms_preprocessing.options.config import path_options_default # noqa: E402
from gms_preprocessing.options.config import get_options # noqa: E402
from gms_preprocessing.options.config import get_config_kwargs_default # noqa: E402
options_default = get_options(path_options_default, validation=True) # type: dict
config_kwargs_default = get_config_kwargs_default() # type: dict
def get_config_kwargs_from_cli_args(cli_args):
return {k: v for k, v in cli_args.__dict__.items() if k in config_kwargs_default.keys()}
def run_from_jobid(args):
......@@ -19,9 +30,7 @@ def run_from_jobid(args):
# TODO download: run only the downloader
# set up process controller instance
PC = process_controller(args.jobid, parallelization_level='scenes', db_host='geoms') # FIXME hardcoded host
# PC.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
# PC.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
PC = process_controller(args.jobid, **get_config_kwargs_from_cli_args(args))
# run the job
PC.run_all_processors()
......@@ -32,19 +41,12 @@ def run_from_sceneids(args):
warnings.warn('Currently the console argument parser expects the given scenes as already downloaded.') # TODO
# create a new processing job from scene IDs
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection)
dbJob = GMS_JOB(get_conn_database(args.database_hostname))
dbJob.from_sceneIDlist(list_sceneIDs=args.sceneids,
virtual_sensor_id=virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref,
comment='')
_run_job(dbJob)
virtual_sensor_id=args.virtual_sensor_id,
datasetid_spatial_ref=args.datasetid_spatial_ref,
comment=args.comment)
_run_job(dbJob, **get_config_kwargs_from_cli_args(args))
def run_from_entityids(args):
......@@ -53,20 +55,12 @@ def run_from_entityids(args):
:param args:
:return:
"""
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection)
dbJob = GMS_JOB(get_conn_database(args.database_hostname))
dbJob.from_entityIDlist(list_entityids=args.entityids,
virtual_sensor_id=virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref,
comment='')
_run_job(dbJob)
virtual_sensor_id=args.virtual_sensor_id,
datasetid_spatial_ref=args.datasetid_spatial_ref,
comment=args.comment)
_run_job(dbJob, **get_config_kwargs_from_cli_args(args))
def run_from_filenames(args):
......@@ -75,20 +69,12 @@ def run_from_filenames(args):
:param args:
:return:
"""
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection)
dbJob = GMS_JOB(get_conn_database(args.database_hostname))
dbJob.from_filenames(list_filenames=args.filenames,
virtual_sensor_id=virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref,
comment='')
_run_job(dbJob)
virtual_sensor_id=args.virtual_sensor_id,
datasetid_spatial_ref=args.datasetid_spatial_ref,
comment=args.comment)
_run_job(dbJob, **get_config_kwargs_from_cli_args(args))
def run_from_constraints(args):
......@@ -97,23 +83,18 @@ def run_from_constraints(args):
raise NotImplementedError
def _run_job(dbJob, parallelization_level='scenes'):
def _run_job(dbJob, **config_kwargs):
# type: (GMS_JOB) -> None
"""
:param dbJob:
:return:
"""
# create a database record for the given job
dbJob.create()
jobid = dbJob.id
# set up process controller instance
warnings.warn("Currently the console argument parser sets the parallelization level to 'scenes'.") # TODO
PC = process_controller(jobid, parallelization_level=parallelization_level)
# PC.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
# PC.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
PC = process_controller(dbJob.id, **config_kwargs)
# run the job
PC.run_all_processors()
......@@ -122,7 +103,10 @@ def _run_job(dbJob, parallelization_level='scenes'):
def get_gms_argparser():
"""Return argument parser for run_gms.py program."""
# CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN
##################################################################
# CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN #
##################################################################
parser = argparse.ArgumentParser(
prog='run_gms.py',
description='=' * 70 + '\n' + 'GeoMultiSens preprocessing console argument parser. '
......@@ -133,40 +117,86 @@ def get_gms_argparser():
parser.add_argument('--version', action='version', version=__version__)
#################################################################
# CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN #
#################################################################
##############################################
# define parsers containing common arguments #
##############################################
general_opts_parser = argparse.ArgumentParser(add_help=False)
gop_p = general_opts_parser.add_argument
gop_p('-jc', '--json_config', nargs='?', type=str,
help='file path of a JSON file containing options. See here for an example:'
'https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/'
'blob/master/gms_preprocessing/options/options_default.json')
# '-exec_L1AP': dict(nargs=3, type=bool, help="L1A Processor configuration",
# metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]),
gop_p('-DH', '--database_hostname', nargs='?', type=str,
default=options_default["global_opts"]["db_host"],
help='host name of the server that runs the postgreSQL database')
gop_p('-vid', '--virtual_sensor_id', type=int,
default=options_default["usecase"]["virtual_sensor_id"],
help='ID of the target (virtual) sensor')
gop_p('-dsid_spat', '--datasetid_spatial_ref', type=int,
default=options_default["usecase"]["datasetid_spatial_ref"],
help='dataset ID of the spatial reference')
gop_p('-c', '--comment', nargs='?', type=str,
default='',
help='comment concerning the job')
##################
# add subparsers #
##################
subparsers = parser.add_subparsers()
# CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN
parser_jobid = subparsers\
.add_parser('jobid', description='Run a GeoMultiSens preprocessing job using an already existing job ID.',
parser_jobid = subparsers.add_parser(
'jobid', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job using an already existing job ID.',
help="Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser).")
parser_sceneids = subparsers\
.add_parser('sceneids', description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
parser_sceneids = subparsers.add_parser(
'sceneids', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser).")
parser_entityids = subparsers\
.add_parser('entityids', description='Run a GeoMultiSens preprocessing job for a given list of entity IDs.',
parser_entityids = subparsers.add_parser(
'entityids', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job for a given list of entity IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser).")
parser_filenames = subparsers\
.add_parser('filenames', description='Run a GeoMultiSens preprocessing job for a given list of filenames of '
parser_filenames = subparsers.add_parser(
'filenames', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job for a given list of filenames of '
'downloaded satellite image archives!',
help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
"image archives! (Sub-Parser).")
parser_constraints = subparsers\
.add_parser('constraints', description='Run a GeoMultiSens preprocessing job matching the given constraints.',
parser_constraints = subparsers.add_parser(
'constraints', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job matching the given constraints.',
help="Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser).")
# parse_from_sceneids = subparsers.add_parser('from_sceneids',
# description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
# help="use '>>> python /path/to/GeMultiSens/run_gms.py from_sceneids -h' for documentation and usage hints")
#################
# ADD ARGUMENTS #
#################
##########################
# add indivial arguments #
##########################
# ADD ARGUMENTS
# add arguments to parser_jobid
jid_p = parser_jobid.add_argument
jid_p('jobid', type=int, help='job ID of an already created GeoMultiSens preprocessing job (must be present in the '
'jobs table of the database)')
jid_p('jobid', type=int, help='job ID of an already created GeoMultiSens preprocessing job '
'(must be present in the jobs table of the database)')
# add arguments to parser_sceneids
sid_p = parser_sceneids.add_argument
......@@ -191,26 +221,10 @@ def get_gms_argparser():
# con_p('constraints', nargs='+', type=str, help="list of entity IDs corresponding to valid records within the "
# "'scenes' table of the database")
# add general arguments # TODO add these configs to each subparser
general_opts = {
'-db_host': dict(),
'-exec_mode': dict(nargs=3, type=bool, help="L1A Processor configuration",
metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]),
'-exec_L1AP': dict(),
'-exec_L1BP': dict(),
'-exec_L1CP': dict(),
'-exec_L2AP': dict(),
'-exec_L2BP': dict(),
'-exec_L2CP': dict(),
'-sub_multiProc': dict(),
'-exc_handler': dict(),
'-blocksize': dict(),
'-profiling': dict(),
'-bench_all': dict(),
}
# LINK PARSERS TO RUN FUNCTIONS
#################################
# LINK PARSERS TO RUN FUNCTIONS #
#################################
parser_jobid.set_defaults(func=run_from_jobid)
parser_sceneids.set_defaults(func=run_from_sceneids)
parser_entityids.set_defaults(func=run_from_entityids)
......
......@@ -632,6 +632,11 @@ class GMS_JOB(object):
self.failed_sceneids = []
self.ref_job_id = None
self.datacube_mgrs_tiles_proc = []
self.non_ref_datasetids = []
self.max_cloudcover = None
self.season_code = None # type: int
self.path_analysis_script = '' # TODO
self.job_mode = 'processing_only' # FIXME download/processing/...
self.jobs_table_columns = ['id', 'creationtime', 'finishtime', 'sceneids', 'timerange_start',
'timerange_end', 'bounds', 'distribution_index', 'progress', 'feedback',
'failed_sceneids', 'datasetid_spatial_ref',
......@@ -663,6 +668,10 @@ class GMS_JOB(object):
def _set_target_sensor_specs(self, virtual_sensor_id, datasetid_spatial_ref):
self.virtualsensorid = virtual_sensor_id
if not isinstance(datasetid_spatial_ref, int):
raise ValueError(datasetid_spatial_ref)
res = get_info_from_postgreSQLdb(self.conn, 'virtual_sensors', ['spatial_resolution',
"projection_epsg"], {'id': virtual_sensor_id})
assert res, \
......@@ -671,7 +680,6 @@ class GMS_JOB(object):
self.ground_spatial_sampling = [target_gsd, target_gsd] if type(target_gsd) in [int, float] else target_gsd
self.epsg = int(res[0][1])
assert isinstance(datasetid_spatial_ref, int)
self.datasetid_spatial_ref = datasetid_spatial_ref
res = get_info_from_postgreSQLdb(self.conn, 'datasets', ['name'], {'id': datasetid_spatial_ref})
assert res, \
......
......@@ -98,6 +98,23 @@ def set_config(job_ID, json_config='', exec_mode='Python', parallelization_level
return getattr(builtins, 'GMS_JobConfig')
def get_conn_database(hostname='localhost', timeout=3):
# type: (str, int) -> str
"""Return database connection string.
:param hostname: the host that runs the GMS postgreSQL database
:param timeout: connection timeout in seconds
:return:
"""
return "dbname='geomultisens' user='gmsdb' password='gmsdb' host='%s' connect_timeout=%d" \
% (hostname, timeout)
def get_config_kwargs_default():
a = getfullargspec(set_config)
return dict(zip(a.args[-len(a.defaults):], a.defaults))
class JobConfig(object):
def __init__(self, ID, **user_opts):
"""Create a job configuration
......@@ -136,8 +153,7 @@ class JobConfig(object):
# database connection
self.db_host = user_opts['db_host']
self.conn_database = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='%s' connect_timeout=3" \
% self.db_host
self.conn_database = get_conn_database(hostname=self.db_host)
# get validated options dict from JSON-options
json_opts = self.get_json_opts(validate=True)
......@@ -377,8 +393,7 @@ class JobConfig(object):
@property
def kwargs_defaults(self):
if not self._kwargs_defaults:
a = getfullargspec(set_config)
self._kwargs_defaults = dict(zip(a.args[-len(a.defaults):], a.defaults))
self._kwargs_defaults = get_config_kwargs_default()
return self._kwargs_defaults
......
......@@ -125,7 +125,7 @@
"usecase": { /*NOTE: These options will be not respected in the WebApp! Use the WebApp GUI instead.*/
"virtual_sensor_id": 10, /*"None": use WebApp input; 1: Landsat-8, 10: Sentinel-2A 10m*/
"datasetid_spatial_ref": "None", /*"None": use WebApp input*/
"datasetid_spatial_ref": 249, /*"None": use WebApp input*/
"datasetid_spectral_ref": 249, /*249=Sentinel-2A*/
"target_CWL": [
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment