run_gms.py 11.3 KB
Newer Older
1
2
# -*- coding: utf-8 -*-
__author__ = 'Daniel Scheffler'
3

4
5
import argparse
import warnings
6
import os
7

8
import matplotlib
9

10
matplotlib.use('Agg', warn=False)  # switch matplotlib backend to 'Agg' and disable warning in case its already 'Agg'
11

12
13
from gms_preprocessing import process_controller, __version__  # noqa: E402
from gms_preprocessing.misc.database_tools import GMS_JOB  # noqa: E402
14
15
16
17
18
19
20
21
22
23
24
from gms_preprocessing.options.config import get_conn_database  # noqa: E402
from gms_preprocessing.options.config import path_options_default  # noqa: E402
from gms_preprocessing.options.config import get_options  # noqa: E402
from gms_preprocessing.options.config import get_config_kwargs_default  # noqa: E402

options_default = get_options(path_options_default, validation=True)  # type: dict
config_kwargs_default = get_config_kwargs_default()  # type: dict


def get_config_kwargs_from_cli_args(cli_args):
    return {k: v for k, v in cli_args.__dict__.items() if k in config_kwargs_default.keys()}
25
26


27
def run_from_jobid(args):
28
29
30
31
32
    # TODO distinguish between ID of a master, processing or download job
    # TODO master: find corresponding sub-jobs and run them
    # TODO processing: check for not downloaded scenes and run processing after download
    # TODO download: run only the downloader

33
    # set up process controller instance
34
    kwargs = get_config_kwargs_from_cli_args(args)
35

36
    if 'GMS_IS_TEST' in os.environ and os.environ['GMS_IS_TEST'] == 'True':
37
38
39
40
41
42
        kwargs['is_test'] = True

    PC = process_controller(args.jobid, **kwargs)

    # run the job
    if 'GMS_IS_TEST_CONFIG' in os.environ and os.environ['GMS_IS_TEST_CONFIG'] == 'True':
43
44
45
46
47
        # in case of software test, it is enough to get an instance of process controller because all inputs are
        # validated within options.config.Job_Config (indirectly called by ProcessController.__init__() )
        pass
    else:
        PC.run_all_processors()
48

49
50
51

def run_from_sceneids(args):
    # create and run a download job
52
    warnings.warn('Currently the console argument parser expects the given scenes as already downloaded.')  # TODO
53
54

    # create a new processing job from scene IDs
55
    dbJob = GMS_JOB(get_conn_database(args.db_host))
56
    dbJob.from_sceneIDlist(list_sceneIDs=args.sceneids,
57
58
59
60
                           virtual_sensor_id=args.virtual_sensor_id,
                           datasetid_spatial_ref=args.datasetid_spatial_ref,
                           comment=args.comment)
    _run_job(dbJob, **get_config_kwargs_from_cli_args(args))
61
62
63


def run_from_entityids(args):
64
    """Create a new job from entity IDs.
65

66
67
68
    :param args:
    :return:
    """
69
    dbJob = GMS_JOB(get_conn_database(args.db_host))
70
    dbJob.from_entityIDlist(list_entityids=args.entityids,
71
72
73
74
                            virtual_sensor_id=args.virtual_sensor_id,
                            datasetid_spatial_ref=args.datasetid_spatial_ref,
                            comment=args.comment)
    _run_job(dbJob, **get_config_kwargs_from_cli_args(args))
75
76
77


def run_from_filenames(args):
78
    """Create a new GMS job from filenames of downloaded archives and run it!
79

80
81
82
    :param args:
    :return:
    """
83
    dbJob = GMS_JOB(get_conn_database(args.db_host))
84
    dbJob.from_filenames(list_filenames=args.filenames,
85
86
87
88
                         virtual_sensor_id=args.virtual_sensor_id,
                         datasetid_spatial_ref=args.datasetid_spatial_ref,
                         comment=args.comment)
    _run_job(dbJob, **get_config_kwargs_from_cli_args(args))
89
90
91
92
93


def run_from_constraints(args):
    # create a new job from constraints
    # TODO
94
95
96
    raise NotImplementedError


97
def _run_job(dbJob, **config_kwargs):
98
99
    # type: (GMS_JOB) -> None
    """
100

101
102
103
    :param dbJob:
    :return:
    """
104
    # create a database record for the given job
105
    dbJob.create()
106
107

    # set up process controller instance
108
109
110
    if 'GMS_IS_TEST' in os.environ and os.environ['GMS_IS_TEST'] == 'True':
        config_kwargs['is_test'] = True

111
    PC = process_controller(dbJob.id, **config_kwargs)
112
113

    # run the job
114
    if 'GMS_IS_TEST_CONFIG' in os.environ and os.environ['GMS_IS_TEST_CONFIG'] == 'True':
115
116
117
118
119
        # in case of software test, it is enough to get an instance of process controller because all inputs are
        # validated within options.config.Job_Config (indirectly called by ProcessController.__init__() )
        pass
    else:
        PC.run_all_processors()
120
121


122
123
def get_gms_argparser():
    """Return argument parser for run_gms.py program."""
124

125
126
127
128
    ##################################################################
    # CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN #
    ##################################################################

129
130
    parser = argparse.ArgumentParser(
        prog='run_gms.py',
131
132
        description='=' * 70 + '\n' + 'GeoMultiSens preprocessing console argument parser. '
                                      'Python implementation by Daniel Scheffler (daniel.scheffler@gfz-potsdam.de)',
133
        epilog="The argument parser offers multiple sub-argument parsers (jobid, sceneids, ...) for starting GMS jobs. "
134
135
               "use '>>> python /path/to/gms_preprocessing/run_gms.py <sub-parser> -h' for detailed documentation and "
               "usage hints.")
136
137
138

    parser.add_argument('--version', action='version', version=__version__)

139
140
141
142
143
144
145
146
147
148
149
150
    #################################################################
    # CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN #
    #################################################################

    ##############################################
    # define parsers containing common arguments #
    ##############################################

    general_opts_parser = argparse.ArgumentParser(add_help=False)
    gop_p = general_opts_parser.add_argument

    gop_p('-jc', '--json_config', nargs='?', type=str,
151
          help='file path of a JSON file containing options. See here for an example: '
152
153
               'https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/'
               'blob/master/gms_preprocessing/options/options_default.json')
154

155
156
    # '-exec_L1AP': dict(nargs=3, type=bool, help="L1A Processor configuration",
    #                   metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]),
157

158
    gop_p('-DH', '--db_host', nargs='?', type=str,
159
          default='localhost',  # hardcoded here because default json is read from database and host must be available
160
          help='host name of the server that runs the postgreSQL database')
161

162
163
164
165
    gop_p('-DOO', '--delete_old_output', nargs='?', type=bool,
          default=options_default["global_opts"]["delete_old_output"],
          help='delete previously created output of the given job ID before running the job')

166
167
168
    gop_p('-vid', '--virtual_sensor_id', type=int,
          default=options_default["usecase"]["virtual_sensor_id"],
          help='ID of the target (virtual) sensor')
169

170
171
172
    gop_p('-dsid_spat', '--datasetid_spatial_ref', type=int,
          default=options_default["usecase"]["datasetid_spatial_ref"],
          help='dataset ID of the spatial reference')
173

174
175
176
177
    gop_p('--CPUs', type=int,
          default=options_default["global_opts"]["CPUs"],
          help='number of CPU cores to be used for processing (default: "None" -> use all available')

178
179
180
    gop_p('-c', '--comment', nargs='?', type=str,
          default='',
          help='comment concerning the job')
181

182
183
184
185
186
187
188
189
190
    ##################
    # add subparsers #
    ##################

    subparsers = parser.add_subparsers()

    parser_jobid = subparsers.add_parser(
        'jobid', parents=[general_opts_parser],
        description='Run a GeoMultiSens preprocessing job using an already existing job ID.',
191
192
        help="Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser).",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
193
194
195
196

    parser_sceneids = subparsers.add_parser(
        'sceneids', parents=[general_opts_parser],
        description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
197
198
        help="Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser).",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
199
200
201
202

    parser_entityids = subparsers.add_parser(
        'entityids', parents=[general_opts_parser],
        description='Run a GeoMultiSens preprocessing job for a given list of entity IDs.',
203
204
        help="Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser).",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
205
206
207
208
209
210

    parser_filenames = subparsers.add_parser(
        'filenames', parents=[general_opts_parser],
        description='Run a GeoMultiSens preprocessing job for a given list of filenames of '
                    'downloaded satellite image archives!',
        help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
211
             "image archives! (Sub-Parser).",
212
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
213
214
215
216

    parser_constraints = subparsers.add_parser(
        'constraints', parents=[general_opts_parser],
        description='Run a GeoMultiSens preprocessing job matching the given constraints.',
217
218
        help="Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser).",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
219
220
221
222
223
224
225
226

    #################
    # ADD ARGUMENTS #
    #################

    ##########################
    # add indivial arguments #
    ##########################
227

228
229
    # add arguments to parser_jobid
    jid_p = parser_jobid.add_argument
230
231
    jid_p('jobid', type=int, help='job ID of an already created GeoMultiSens preprocessing job '
                                  '(must be present in the jobs table of the database)')
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252

    # add arguments to parser_sceneids
    sid_p = parser_sceneids.add_argument
    sid_p('sceneids', nargs='+', type=int,
          help="list of scene IDs corresponding to valid records within the 'scenes' table of the database")

    # add arguments to parser_entityids
    eid_p = parser_entityids.add_argument
    eid_p('entityids', nargs='+', type=str,
          help="list of entity IDs corresponding to valid records within the 'scenes' table of the database")
    # FIXME satellite and sensor are required

    # add arguments to parser_filenames
    eid_p = parser_filenames.add_argument
    eid_p('filenames', nargs='+', type=str,
          help="list of filenames of satellite image archives corresponding to valid records within the 'scenes' "
               "table of the database")

    # add arguments to parse_constraints
    con_p = parser_constraints.add_argument
    # TODO
253
    # con_p('constraints', nargs='+', type=str, help="list of entity IDs corresponding to valid records within the "
254
    #                                            "'scenes' table of the database")
255

256
257
258
259
    #################################
    # LINK PARSERS TO RUN FUNCTIONS #
    #################################

260
261
262
    parser_jobid.set_defaults(func=run_from_jobid)
    parser_sceneids.set_defaults(func=run_from_sceneids)
    parser_entityids.set_defaults(func=run_from_entityids)
263
    parser_filenames.set_defaults(func=run_from_filenames)
264
    parser_constraints.set_defaults(func=run_from_constraints)
265

266
267
268
    return parser


269
if __name__ == '__main__':
270
    parsed_args = get_gms_argparser().parse_args()
271
272
    parsed_args.func(parsed_args)

273
    print('\nready.')