run_gms.py 12.8 KB
Newer Older
1
2
# -*- coding: utf-8 -*-
__author__ = 'Daniel Scheffler'
3

4
5
import argparse
import warnings
6
import os
7

8
import matplotlib
9

10
matplotlib.use('Agg', warn=False)  # switch matplotlib backend to 'Agg' and disable warning in case its already 'Agg'
11

12
from gms_preprocessing import ProcessController, __version__  # noqa: E402
13
from gms_preprocessing.misc.database_tools import GMS_JOB  # noqa: E402
14
15
16
from gms_preprocessing.options.config import get_conn_database  # noqa: E402
from gms_preprocessing.options.config import path_options_default  # noqa: E402
from gms_preprocessing.options.config import get_options  # noqa: E402
17
from gms_preprocessing.options.options_schema import get_param_from_json_config  # noqa: E402
18
19
20
21

options_default = get_options(path_options_default, validation=True)  # type: dict


22
def parsedArgs_to_user_opts(cli_args):
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
    # type: (argparse.Namespace) -> dict
    """Convert argparse Namespace object to dictionary of explicitly given parameters.

    NOTE:   All options that have not been given explicitly (None values) are removed. Reason: all options to
            passed set_config WILL OVERRIDE job settings read from the GMS database (e.g., specified by the WebUI)
            => only override job configuration defined by WebUI if CLI options are explicitly given
            => if json_opts are given: configuration defined by WebUI will be overridden by this json config in any case

    :param cli_args:    options as parsed by the argparse.ArgumentParser
    """

    # convert argparse Namespace object to dictionary
    opts = {k: v for k, v in vars(cli_args).items() if not k.startswith('_') and k != 'func'}

    # remove those options that have not been given explicitly (None values)
    user_opts = dict()
    for k, v in opts.items():
        # values are None if they are not given by the user -> don't pass to set_config
        if v is None:
            continue

        # remove keys that are not to be passed to set_config
        elif k in ['jobid', 'sceneids', 'entityids', 'filenames', 'comment', ]:
            continue

        else:
            user_opts.update({k: v})

    return user_opts
52
53


54
def run_from_jobid(args):
55
56
57
58
59
    # TODO distinguish between ID of a master, processing or download job
    # TODO master: find corresponding sub-jobs and run them
    # TODO processing: check for not downloaded scenes and run processing after download
    # TODO download: run only the downloader

60
    # set up process controller instance
61
    kwargs = parsedArgs_to_user_opts(args)
62

63
    if 'GMS_IS_TEST' in os.environ and os.environ['GMS_IS_TEST'] == 'True':
64
65
        kwargs['is_test'] = True

66
    PC = ProcessController(args.jobid, **kwargs)
67
68
69

    # run the job
    if 'GMS_IS_TEST_CONFIG' in os.environ and os.environ['GMS_IS_TEST_CONFIG'] == 'True':
70
71
72
73
74
        # in case of software test, it is enough to get an instance of process controller because all inputs are
        # validated within options.config.Job_Config (indirectly called by ProcessController.__init__() )
        pass
    else:
        PC.run_all_processors()
75

76
77
78

def run_from_sceneids(args):
    # create and run a download job
79
    warnings.warn('Currently the console argument parser expects the given scenes as already downloaded.')  # TODO
80
81

    # create a new processing job from scene IDs
82
    dbJob = GMS_JOB(get_conn_database(args.db_host))
83
    dbJob.from_sceneIDlist(list_sceneIDs=args.sceneids,
84
85
                           virtual_sensor_id=get_user_input_or_default('virtual_sensor_id', args),
                           datasetid_spatial_ref=get_user_input_or_default('datasetid_spatial_ref', args),
86
                           comment=args.comment)
87
    _run_job(dbJob, **parsedArgs_to_user_opts(args))
88
89
90


def run_from_entityids(args):
91
    """Create a new job from entity IDs.
92

93
94
95
    :param args:
    :return:
    """
96
    dbJob = GMS_JOB(get_conn_database(args.db_host))
97
    dbJob.from_entityIDlist(list_entityids=args.entityids,
98
99
                            virtual_sensor_id=get_user_input_or_default('virtual_sensor_id', args),
                            datasetid_spatial_ref=get_user_input_or_default('datasetid_spatial_ref', args),
100
                            comment=args.comment)
101
    _run_job(dbJob, **parsedArgs_to_user_opts(args))
102
103
104


def run_from_filenames(args):
105
    """Create a new GMS job from filenames of downloaded archives and run it!
106

107
108
109
    :param args:
    :return:
    """
110
    dbJob = GMS_JOB(get_conn_database(args.db_host))
111
    dbJob.from_filenames(list_filenames=args.filenames,
112
113
                         virtual_sensor_id=get_user_input_or_default('virtual_sensor_id', args),
                         datasetid_spatial_ref=get_user_input_or_default('datasetid_spatial_ref', args),
114
                         comment=args.comment)
115
    _run_job(dbJob, **parsedArgs_to_user_opts(args))
116
117
118
119
120


def run_from_constraints(args):
    # create a new job from constraints
    # TODO
121
122
123
    raise NotImplementedError


124
def _run_job(dbJob, **config_kwargs):
125
126
    # type: (GMS_JOB) -> None
    """
127

128
129
130
    :param dbJob:
    :return:
    """
131
    # create a database record for the given job
132
    dbJob.create()
133
134

    # set up process controller instance
135
136
137
    if 'GMS_IS_TEST' in os.environ and os.environ['GMS_IS_TEST'] == 'True':
        config_kwargs['is_test'] = True

138
    PC = ProcessController(dbJob.id, **config_kwargs)
139
140

    # run the job
141
    if 'GMS_IS_TEST_CONFIG' in os.environ and os.environ['GMS_IS_TEST_CONFIG'] == 'True':
142
143
144
145
146
        # in case of software test, it is enough to get an instance of process controller because all inputs are
        # validated within options.config.Job_Config (indirectly called by ProcessController.__init__() )
        pass
    else:
        PC.run_all_processors()
147
148


149
150
151
152
153
154
155
def get_user_input_or_default(paramname, argsparse_ns):
    user_input = getattr(argsparse_ns, paramname)

    return user_input if user_input is not None else \
        get_param_from_json_config(paramname, options_default)


156
157
def get_gms_argparser():
    """Return argument parser for run_gms.py program."""
158

159
160
161
162
    ##################################################################
    # CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN #
    ##################################################################

163
164
    parser = argparse.ArgumentParser(
        prog='run_gms.py',
165
166
        description='=' * 70 + '\n' + 'GeoMultiSens preprocessing console argument parser. '
                                      'Python implementation by Daniel Scheffler (daniel.scheffler@gfz-potsdam.de)',
167
        epilog="The argument parser offers multiple sub-argument parsers (jobid, sceneids, ...) for starting GMS jobs. "
168
169
               "use '>>> python /path/to/gms_preprocessing/run_gms.py <sub-parser> -h' for detailed documentation and "
               "usage hints.")
170
171
172

    parser.add_argument('--version', action='version', version=__version__)

173
174
175
176
177
178
179
180
181
182
183
184
    #################################################################
    # CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN #
    #################################################################

    ##############################################
    # define parsers containing common arguments #
    ##############################################

    general_opts_parser = argparse.ArgumentParser(add_help=False)
    gop_p = general_opts_parser.add_argument

    gop_p('-jc', '--json_config', nargs='?', type=str,
185
          help='file path of a JSON file containing options. See here for an example: '
186
187
               'https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/'
               'blob/master/gms_preprocessing/options/options_default.json')
188

189
190
    # '-exec_L1AP': dict(nargs=3, type=bool, help="L1A Processor configuration",
    #                   metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]),
191

192
    gop_p('-DH', '--db_host', nargs='?', type=str,
193
          default='localhost',  # hardcoded here because default json is read from database and host must be available
194
          help='host name of the server that runs the postgreSQL database')
195

196
197
198
199
    # NOTE: don't define any defaults here for parameters that are passed to set_config!
    #       -> otherwise, we cannot distinguish between explicity given parameters and default values
    #       => see docs in parsedArgs_to_user_opts() for explanation
    gop_p('-DOO', '--delete_old_output', nargs='?', type=bool, default=None,
200
201
          help='delete previously created output of the given job ID before running the job')

202
    gop_p('-vid', '--virtual_sensor_id', type=int, default=None,
203
          help='ID of the target (virtual) sensor')
204

205
    gop_p('-dsid_spat', '--datasetid_spatial_ref', type=int, default=None,
206
          help='dataset ID of the spatial reference')
207

208
    gop_p('--CPUs', type=int, default=None,
209
210
          help='number of CPU cores to be used for processing (default: "None" -> use all available')

211
212
213
    gop_p('-c', '--comment', nargs='?', type=str,
          default='',
          help='comment concerning the job')
214

215
216
217
218
219
220
221
222
223
    ##################
    # add subparsers #
    ##################

    subparsers = parser.add_subparsers()

    parser_jobid = subparsers.add_parser(
        'jobid', parents=[general_opts_parser],
        description='Run a GeoMultiSens preprocessing job using an already existing job ID.',
224
225
        help="Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser).",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
226
227
228
229

    parser_sceneids = subparsers.add_parser(
        'sceneids', parents=[general_opts_parser],
        description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
230
231
        help="Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser).",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
232
233
234
235

    parser_entityids = subparsers.add_parser(
        'entityids', parents=[general_opts_parser],
        description='Run a GeoMultiSens preprocessing job for a given list of entity IDs.',
236
237
        help="Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser).",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
238
239
240
241
242
243

    parser_filenames = subparsers.add_parser(
        'filenames', parents=[general_opts_parser],
        description='Run a GeoMultiSens preprocessing job for a given list of filenames of '
                    'downloaded satellite image archives!',
        help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
244
             "image archives! (Sub-Parser).",
245
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
246
247
248
249

    parser_constraints = subparsers.add_parser(
        'constraints', parents=[general_opts_parser],
        description='Run a GeoMultiSens preprocessing job matching the given constraints.',
250
251
        help="Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser).",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
252
253
254
255
256
257
258
259

    #################
    # ADD ARGUMENTS #
    #################

    ##########################
    # add indivial arguments #
    ##########################
260

261
262
    # add arguments to parser_jobid
    jid_p = parser_jobid.add_argument
263
264
    jid_p('jobid', type=int, help='job ID of an already created GeoMultiSens preprocessing job '
                                  '(must be present in the jobs table of the database)')
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285

    # add arguments to parser_sceneids
    sid_p = parser_sceneids.add_argument
    sid_p('sceneids', nargs='+', type=int,
          help="list of scene IDs corresponding to valid records within the 'scenes' table of the database")

    # add arguments to parser_entityids
    eid_p = parser_entityids.add_argument
    eid_p('entityids', nargs='+', type=str,
          help="list of entity IDs corresponding to valid records within the 'scenes' table of the database")
    # FIXME satellite and sensor are required

    # add arguments to parser_filenames
    eid_p = parser_filenames.add_argument
    eid_p('filenames', nargs='+', type=str,
          help="list of filenames of satellite image archives corresponding to valid records within the 'scenes' "
               "table of the database")

    # add arguments to parse_constraints
    con_p = parser_constraints.add_argument
    # TODO
286
    # con_p('constraints', nargs='+', type=str, help="list of entity IDs corresponding to valid records within the "
287
    #                                            "'scenes' table of the database")
288

289
290
291
292
    #################################
    # LINK PARSERS TO RUN FUNCTIONS #
    #################################

293
294
295
    parser_jobid.set_defaults(func=run_from_jobid)
    parser_sceneids.set_defaults(func=run_from_sceneids)
    parser_entityids.set_defaults(func=run_from_entityids)
296
    parser_filenames.set_defaults(func=run_from_filenames)
297
    parser_constraints.set_defaults(func=run_from_constraints)
298

299
300
301
    return parser


302
if __name__ == '__main__':
303
    parsed_args = get_gms_argparser().parse_args()
304
305
    parsed_args.func(parsed_args)

306
    print('\nready.')