Commit 069dba5f authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Revised command line argparser.

Former-commit-id: f85d2a7b
Former-commit-id: f347434a
parent 7efdf2df
...@@ -10,6 +10,17 @@ matplotlib.use('Agg', warn=False) # switch matplotlib backend to 'Agg' and disa ...@@ -10,6 +10,17 @@ matplotlib.use('Agg', warn=False) # switch matplotlib backend to 'Agg' and disa
from gms_preprocessing import process_controller, __version__ # noqa: E402 from gms_preprocessing import process_controller, __version__ # noqa: E402
from gms_preprocessing.misc.database_tools import GMS_JOB # noqa: E402 from gms_preprocessing.misc.database_tools import GMS_JOB # noqa: E402
from gms_preprocessing.options.config import get_conn_database # noqa: E402
from gms_preprocessing.options.config import path_options_default # noqa: E402
from gms_preprocessing.options.config import get_options # noqa: E402
from gms_preprocessing.options.config import get_config_kwargs_default # noqa: E402
options_default = get_options(path_options_default, validation=True) # type: dict
config_kwargs_default = get_config_kwargs_default() # type: dict
def get_config_kwargs_from_cli_args(cli_args):
return {k: v for k, v in cli_args.__dict__.items() if k in config_kwargs_default.keys()}
def run_from_jobid(args): def run_from_jobid(args):
...@@ -19,9 +30,7 @@ def run_from_jobid(args): ...@@ -19,9 +30,7 @@ def run_from_jobid(args):
# TODO download: run only the downloader # TODO download: run only the downloader
# set up process controller instance # set up process controller instance
PC = process_controller(args.jobid, parallelization_level='scenes', db_host='geoms') # FIXME hardcoded host PC = process_controller(args.jobid, **get_config_kwargs_from_cli_args(args))
# PC.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
# PC.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
# run the job # run the job
PC.run_all_processors() PC.run_all_processors()
...@@ -32,19 +41,12 @@ def run_from_sceneids(args): ...@@ -32,19 +41,12 @@ def run_from_sceneids(args):
warnings.warn('Currently the console argument parser expects the given scenes as already downloaded.') # TODO warnings.warn('Currently the console argument parser expects the given scenes as already downloaded.') # TODO
# create a new processing job from scene IDs # create a new processing job from scene IDs
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO dbJob = GMS_JOB(get_conn_database(args.database_hostname))
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection)
dbJob.from_sceneIDlist(list_sceneIDs=args.sceneids, dbJob.from_sceneIDlist(list_sceneIDs=args.sceneids,
virtual_sensor_id=virtual_sensor_id, virtual_sensor_id=args.virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref, datasetid_spatial_ref=args.datasetid_spatial_ref,
comment='') comment=args.comment)
_run_job(dbJob) _run_job(dbJob, **get_config_kwargs_from_cli_args(args))
def run_from_entityids(args): def run_from_entityids(args):
...@@ -53,20 +55,12 @@ def run_from_entityids(args): ...@@ -53,20 +55,12 @@ def run_from_entityids(args):
:param args: :param args:
:return: :return:
""" """
dbJob = GMS_JOB(get_conn_database(args.database_hostname))
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection)
dbJob.from_entityIDlist(list_entityids=args.entityids, dbJob.from_entityIDlist(list_entityids=args.entityids,
virtual_sensor_id=virtual_sensor_id, virtual_sensor_id=args.virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref, datasetid_spatial_ref=args.datasetid_spatial_ref,
comment='') comment=args.comment)
_run_job(dbJob) _run_job(dbJob, **get_config_kwargs_from_cli_args(args))
def run_from_filenames(args): def run_from_filenames(args):
...@@ -75,20 +69,12 @@ def run_from_filenames(args): ...@@ -75,20 +69,12 @@ def run_from_filenames(args):
:param args: :param args:
:return: :return:
""" """
dbJob = GMS_JOB(get_conn_database(args.database_hostname))
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection)
dbJob.from_filenames(list_filenames=args.filenames, dbJob.from_filenames(list_filenames=args.filenames,
virtual_sensor_id=virtual_sensor_id, virtual_sensor_id=args.virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref, datasetid_spatial_ref=args.datasetid_spatial_ref,
comment='') comment=args.comment)
_run_job(dbJob) _run_job(dbJob, **get_config_kwargs_from_cli_args(args))
def run_from_constraints(args): def run_from_constraints(args):
...@@ -97,23 +83,18 @@ def run_from_constraints(args): ...@@ -97,23 +83,18 @@ def run_from_constraints(args):
raise NotImplementedError raise NotImplementedError
def _run_job(dbJob, parallelization_level='scenes'): def _run_job(dbJob, **config_kwargs):
# type: (GMS_JOB) -> None # type: (GMS_JOB) -> None
""" """
:param dbJob: :param dbJob:
:return: :return:
""" """
# create a database record for the given job
dbJob.create() dbJob.create()
jobid = dbJob.id
# set up process controller instance # set up process controller instance
warnings.warn("Currently the console argument parser sets the parallelization level to 'scenes'.") # TODO PC = process_controller(dbJob.id, **config_kwargs)
PC = process_controller(jobid, parallelization_level=parallelization_level)
# PC.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
# PC.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
# run the job # run the job
PC.run_all_processors() PC.run_all_processors()
...@@ -122,7 +103,10 @@ def _run_job(dbJob, parallelization_level='scenes'): ...@@ -122,7 +103,10 @@ def _run_job(dbJob, parallelization_level='scenes'):
def get_gms_argparser(): def get_gms_argparser():
"""Return argument parser for run_gms.py program.""" """Return argument parser for run_gms.py program."""
# CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN ##################################################################
# CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN #
##################################################################
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog='run_gms.py', prog='run_gms.py',
description='=' * 70 + '\n' + 'GeoMultiSens preprocessing console argument parser. ' description='=' * 70 + '\n' + 'GeoMultiSens preprocessing console argument parser. '
...@@ -133,40 +117,86 @@ def get_gms_argparser(): ...@@ -133,40 +117,86 @@ def get_gms_argparser():
parser.add_argument('--version', action='version', version=__version__) parser.add_argument('--version', action='version', version=__version__)
subparsers = parser.add_subparsers() #################################################################
# CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN #
#################################################################
##############################################
# define parsers containing common arguments #
##############################################
general_opts_parser = argparse.ArgumentParser(add_help=False)
gop_p = general_opts_parser.add_argument
gop_p('-jc', '--json_config', nargs='?', type=str,
help='file path of a JSON file containing options. See here for an example:'
'https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/'
'blob/master/gms_preprocessing/options/options_default.json')
# CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN # '-exec_L1AP': dict(nargs=3, type=bool, help="L1A Processor configuration",
parser_jobid = subparsers\ # metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]),
.add_parser('jobid', description='Run a GeoMultiSens preprocessing job using an already existing job ID.',
help="Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser).")
parser_sceneids = subparsers\ gop_p('-DH', '--database_hostname', nargs='?', type=str,
.add_parser('sceneids', description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.', default=options_default["global_opts"]["db_host"],
help="Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser).") help='host name of the server that runs the postgreSQL database')
parser_entityids = subparsers\ gop_p('-vid', '--virtual_sensor_id', type=int,
.add_parser('entityids', description='Run a GeoMultiSens preprocessing job for a given list of entity IDs.', default=options_default["usecase"]["virtual_sensor_id"],
help="Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser).") help='ID of the target (virtual) sensor')
parser_filenames = subparsers\ gop_p('-dsid_spat', '--datasetid_spatial_ref', type=int,
.add_parser('filenames', description='Run a GeoMultiSens preprocessing job for a given list of filenames of ' default=options_default["usecase"]["datasetid_spatial_ref"],
'downloaded satellite image archives!', help='dataset ID of the spatial reference')
help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
"image archives! (Sub-Parser).")
parser_constraints = subparsers\ gop_p('-c', '--comment', nargs='?', type=str,
.add_parser('constraints', description='Run a GeoMultiSens preprocessing job matching the given constraints.', default='',
help="Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser).") help='comment concerning the job')
# parse_from_sceneids = subparsers.add_parser('from_sceneids', ##################
# description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.', # add subparsers #
# help="use '>>> python /path/to/GeMultiSens/run_gms.py from_sceneids -h' for documentation and usage hints") ##################
subparsers = parser.add_subparsers()
parser_jobid = subparsers.add_parser(
'jobid', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job using an already existing job ID.',
help="Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser).")
parser_sceneids = subparsers.add_parser(
'sceneids', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser).")
parser_entityids = subparsers.add_parser(
'entityids', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job for a given list of entity IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser).")
parser_filenames = subparsers.add_parser(
'filenames', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job for a given list of filenames of '
'downloaded satellite image archives!',
help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
"image archives! (Sub-Parser).")
parser_constraints = subparsers.add_parser(
'constraints', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job matching the given constraints.',
help="Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser).")
#################
# ADD ARGUMENTS #
#################
##########################
# add indivial arguments #
##########################
# ADD ARGUMENTS
# add arguments to parser_jobid # add arguments to parser_jobid
jid_p = parser_jobid.add_argument jid_p = parser_jobid.add_argument
jid_p('jobid', type=int, help='job ID of an already created GeoMultiSens preprocessing job (must be present in the ' jid_p('jobid', type=int, help='job ID of an already created GeoMultiSens preprocessing job '
'jobs table of the database)') '(must be present in the jobs table of the database)')
# add arguments to parser_sceneids # add arguments to parser_sceneids
sid_p = parser_sceneids.add_argument sid_p = parser_sceneids.add_argument
...@@ -191,26 +221,10 @@ def get_gms_argparser(): ...@@ -191,26 +221,10 @@ def get_gms_argparser():
# con_p('constraints', nargs='+', type=str, help="list of entity IDs corresponding to valid records within the " # con_p('constraints', nargs='+', type=str, help="list of entity IDs corresponding to valid records within the "
# "'scenes' table of the database") # "'scenes' table of the database")
# add general arguments # TODO add these configs to each subparser #################################
general_opts = { # LINK PARSERS TO RUN FUNCTIONS #
'-db_host': dict(), #################################
'-exec_mode': dict(nargs=3, type=bool, help="L1A Processor configuration",
metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]),
'-exec_L1AP': dict(),
'-exec_L1BP': dict(),
'-exec_L1CP': dict(),
'-exec_L2AP': dict(),
'-exec_L2BP': dict(),
'-exec_L2CP': dict(),
'-sub_multiProc': dict(),
'-exc_handler': dict(),
'-blocksize': dict(),
'-profiling': dict(),
'-bench_all': dict(),
}
# LINK PARSERS TO RUN FUNCTIONS
parser_jobid.set_defaults(func=run_from_jobid) parser_jobid.set_defaults(func=run_from_jobid)
parser_sceneids.set_defaults(func=run_from_sceneids) parser_sceneids.set_defaults(func=run_from_sceneids)
parser_entityids.set_defaults(func=run_from_entityids) parser_entityids.set_defaults(func=run_from_entityids)
......
...@@ -632,6 +632,11 @@ class GMS_JOB(object): ...@@ -632,6 +632,11 @@ class GMS_JOB(object):
self.failed_sceneids = [] self.failed_sceneids = []
self.ref_job_id = None self.ref_job_id = None
self.datacube_mgrs_tiles_proc = [] self.datacube_mgrs_tiles_proc = []
self.non_ref_datasetids = []
self.max_cloudcover = None
self.season_code = None # type: int
self.path_analysis_script = '' # TODO
self.job_mode = 'processing_only' # FIXME download/processing/...
self.jobs_table_columns = ['id', 'creationtime', 'finishtime', 'sceneids', 'timerange_start', self.jobs_table_columns = ['id', 'creationtime', 'finishtime', 'sceneids', 'timerange_start',
'timerange_end', 'bounds', 'distribution_index', 'progress', 'feedback', 'timerange_end', 'bounds', 'distribution_index', 'progress', 'feedback',
'failed_sceneids', 'datasetid_spatial_ref', 'failed_sceneids', 'datasetid_spatial_ref',
...@@ -663,6 +668,10 @@ class GMS_JOB(object): ...@@ -663,6 +668,10 @@ class GMS_JOB(object):
def _set_target_sensor_specs(self, virtual_sensor_id, datasetid_spatial_ref): def _set_target_sensor_specs(self, virtual_sensor_id, datasetid_spatial_ref):
self.virtualsensorid = virtual_sensor_id self.virtualsensorid = virtual_sensor_id
if not isinstance(datasetid_spatial_ref, int):
raise ValueError(datasetid_spatial_ref)
res = get_info_from_postgreSQLdb(self.conn, 'virtual_sensors', ['spatial_resolution', res = get_info_from_postgreSQLdb(self.conn, 'virtual_sensors', ['spatial_resolution',
"projection_epsg"], {'id': virtual_sensor_id}) "projection_epsg"], {'id': virtual_sensor_id})
assert res, \ assert res, \
...@@ -671,7 +680,6 @@ class GMS_JOB(object): ...@@ -671,7 +680,6 @@ class GMS_JOB(object):
self.ground_spatial_sampling = [target_gsd, target_gsd] if type(target_gsd) in [int, float] else target_gsd self.ground_spatial_sampling = [target_gsd, target_gsd] if type(target_gsd) in [int, float] else target_gsd
self.epsg = int(res[0][1]) self.epsg = int(res[0][1])
assert isinstance(datasetid_spatial_ref, int)
self.datasetid_spatial_ref = datasetid_spatial_ref self.datasetid_spatial_ref = datasetid_spatial_ref
res = get_info_from_postgreSQLdb(self.conn, 'datasets', ['name'], {'id': datasetid_spatial_ref}) res = get_info_from_postgreSQLdb(self.conn, 'datasets', ['name'], {'id': datasetid_spatial_ref})
assert res, \ assert res, \
......
...@@ -98,6 +98,23 @@ def set_config(job_ID, json_config='', exec_mode='Python', parallelization_level ...@@ -98,6 +98,23 @@ def set_config(job_ID, json_config='', exec_mode='Python', parallelization_level
return getattr(builtins, 'GMS_JobConfig') return getattr(builtins, 'GMS_JobConfig')
def get_conn_database(hostname='localhost', timeout=3):
# type: (str, int) -> str
"""Return database connection string.
:param hostname: the host that runs the GMS postgreSQL database
:param timeout: connection timeout in seconds
:return:
"""
return "dbname='geomultisens' user='gmsdb' password='gmsdb' host='%s' connect_timeout=%d" \
% (hostname, timeout)
def get_config_kwargs_default():
a = getfullargspec(set_config)
return dict(zip(a.args[-len(a.defaults):], a.defaults))
class JobConfig(object): class JobConfig(object):
def __init__(self, ID, **user_opts): def __init__(self, ID, **user_opts):
"""Create a job configuration """Create a job configuration
...@@ -136,8 +153,7 @@ class JobConfig(object): ...@@ -136,8 +153,7 @@ class JobConfig(object):
# database connection # database connection
self.db_host = user_opts['db_host'] self.db_host = user_opts['db_host']
self.conn_database = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='%s' connect_timeout=3" \ self.conn_database = get_conn_database(hostname=self.db_host)
% self.db_host
# get validated options dict from JSON-options # get validated options dict from JSON-options
json_opts = self.get_json_opts(validate=True) json_opts = self.get_json_opts(validate=True)
...@@ -377,8 +393,7 @@ class JobConfig(object): ...@@ -377,8 +393,7 @@ class JobConfig(object):
@property @property
def kwargs_defaults(self): def kwargs_defaults(self):
if not self._kwargs_defaults: if not self._kwargs_defaults:
a = getfullargspec(set_config) self._kwargs_defaults = get_config_kwargs_default()
self._kwargs_defaults = dict(zip(a.args[-len(a.defaults):], a.defaults))
return self._kwargs_defaults return self._kwargs_defaults
......
...@@ -125,7 +125,7 @@ ...@@ -125,7 +125,7 @@
"usecase": { /*NOTE: These options will be not respected in the WebApp! Use the WebApp GUI instead.*/ "usecase": { /*NOTE: These options will be not respected in the WebApp! Use the WebApp GUI instead.*/
"virtual_sensor_id": 10, /*"None": use WebApp input; 1: Landsat-8, 10: Sentinel-2A 10m*/ "virtual_sensor_id": 10, /*"None": use WebApp input; 1: Landsat-8, 10: Sentinel-2A 10m*/
"datasetid_spatial_ref": "None", /*"None": use WebApp input*/ "datasetid_spatial_ref": 249, /*"None": use WebApp input*/
"datasetid_spectral_ref": 249, /*249=Sentinel-2A*/ "datasetid_spectral_ref": 249, /*249=Sentinel-2A*/
"target_CWL": [ "target_CWL": [
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment