test_gms_preprocessing.py 34.7 KB
Newer Older
1
2
3
#!/usr/bin/env python
# -*- coding: utf-8 -*-

4
5
# gms_preprocessing, spatial and spectral homogenization of satellite remote sensing data
#
6
# Copyright (C) 2020  Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
7
8
9
10
11
12
#
# This software was developed within the context of the GeoMultiSens project funded
# by the German Federal Ministry of Education and Research
# (project grant code: 01 IS 14 010 A-C).
#
# This program is free software: you can redistribute it and/or modify it under
13
# the terms of the GNU General Public License as published by the Free Software
14
15
# Foundation, either version 3 of the License, or (at your option) any later version.
# Please note the following exception: `gms_preprocessing` depends on tqdm, which
16
17
18
# is distributed under the Mozilla Public Licence (MPL) v2.0 except for the files
# "tqdm/_tqdm.py", "setup.py", "README.rst", "MANIFEST.in" and ".gitignore".
# Details can be found here: https://github.com/tqdm/tqdm/blob/master/LICENCE.
19
20
21
22
23
24
25
26
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program.  If not, see <http://www.gnu.org/licenses/>.
27

28
"""
29
test_gms_preprocessing
30
----------------------------------
31

32
The testcases contained in this testscript, are parametrized testcases. They test
33
34
the level-processing steps defined in the 'gms_preprocessing' module in the
"gms_preprocessing"-project with the help of the test datasets:
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
- Landsat-5, Pre-Collection Data,
- Landsat-5, Collection Data,
- Landsat-7, SLC on, Pre-Collection Data,
- Landsat-7, SLC off, Pre-Collection Data,
- Landsat-7, SLC off, Collection Data,
- Landsat-8, Pre-Collection Data,
- Landsat-8, Collection Data,
- Sentinel-2A, Pre-Collection Data and
- Sentinel-2A, Collection Data.
The test datasets can be found in the directory "tests/data/archive_data/...". The
respective SRTM-datasets needed in the data-processing can be found in the directory
"tests/data/archive_data/Endeavor".

The tests, defined in a base-testcase (not executed), are triggered by creating
jobs (based on given job-IDs) in individual testcases that inherit the tests
from the base-testcase. The exception: The job-ID used in the last testclass
contains 3 different test datasets of the above listed datasets.

Note that the testresults are outputted in the console as well as a log-textfile
that can be found in the directory "tests/logs".

Program edited in July 2017.
"""

59
# Import python standard libraries.
60
61
62
63
64
65
import itertools
import logging
import os
import pandas
import sys
import time
66
67
import unittest

68
# Imports regarding the 'gms_preprocessing' module.
69
from gms_preprocessing import ProcessController, __file__
70
from gms_preprocessing.model.gms_object import GMS_object
71
72
73
74
75
from gms_preprocessing.algorithms.L1A_P import L1A_object
from gms_preprocessing.algorithms.L1B_P import L1B_object
from gms_preprocessing.algorithms.L1C_P import L1C_object
from gms_preprocessing.algorithms.L2A_P import L2A_object
from gms_preprocessing.algorithms.L2B_P import L2B_object
76
# from gms_preprocessing.algorithms.L2C_P import L2C_object
77
from gms_preprocessing.misc.database_tools import get_info_from_postgreSQLdb
78
from gms_preprocessing.model.gms_object import GMS_object_2_dataset_dict
79

80
from . import db_host, index_host
81

82
__author__ = 'Daniel Scheffler'  # edited by Jessica Palka.
83

84
# Rootpath of the gms_preprocessing-repository.
85
86
87
88
gmsRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))

# Defining the configurations needed to start a job containing the different dataset scenes.
# TODO Change the job-configurations for selected datasets.
89
job_config_kwargs = dict(parallelization_level='scenes', db_host=db_host, spatial_index_server_host=index_host,
90
                         delete_old_output=True, is_test=True, reset_status=True,
91
92
                         inmem_serialization=False,
                         exec_L1AP=[True, True, True], exec_L1BP=[True, True, True], exec_L1CP=[True, True, True],
93
                         exec_L2AP=[True, True, True], exec_L2BP=[True, True, False], exec_L2CP=[True, True, False])
94

Daniel Scheffler's avatar
Daniel Scheffler committed
95
##########################
96
# Test case: BaseTestCases
Daniel Scheffler's avatar
Daniel Scheffler committed
97
98
##########################

99
100
101
102

class BaseTestCases:
    """
    General testclass. The tests defined in this testclass test the processing steps Level-1A, Level-1B, Level-1C,
103
    Level-2A, Level-2B and Level-2C defined in the "gms_preprocessing"-repository.
104
105
106
107
    Note that the tests in this testclass are not executed directly. They are re-used in the other classes defined
    in this test-script.
    """
    class TestAll(unittest.TestCase):
108
        PC = None  # default
109
110
111

        @classmethod
        def tearDownClass(cls):
112
            cls.PC.config.DB_job_record.delete_procdata_of_entire_job(force=True)
113
114
115

        @classmethod
        def validate_db_entry(cls, filename):
116
117
            sceneID_res = get_info_from_postgreSQLdb(cls.PC.config.conn_database, 'scenes', ['id'],
                                                     {'filename': filename})
118
119
120
121
            assert sceneID_res and isinstance(sceneID_res[0][0], int), 'Invalid database entry.'

        @classmethod
        def create_job(cls, jobID, config):
122
            cls.PC = ProcessController(jobID, **config)
123
124

            # update attributes of DB_job_record and related DB entry
125
            cls.PC.config.DB_job_record.reset_job_progress()
126

127
            GMS_object.proc_status_all_GMSobjs.clear()  # reset
128

Daniel Scheffler's avatar
Bugfix.    
Daniel Scheffler committed
129
            cls.PC.config.data_list = cls.PC.add_local_availability(cls.PC.config.data_list)
130

131
            [cls.validate_db_entry(ds['filename']) for ds in cls.PC.config.data_list]
132

133
134
135
            cls.PC.config.ac_estimate_accuracy = True
            cls.PC.config.spathomo_estimate_accuracy = True
            cls.PC.config.spechomo_estimate_accuracy = True
136

137
138
139
140
        def check_availability(self, GMS_objs, tgt_procL):
            dss = self.PC.add_local_availability([GMS_object_2_dataset_dict(obj) for obj in GMS_objs])
            for ds in dss:
                self.assertEqual(ds['proc_level'], tgt_procL,
Daniel Scheffler's avatar
Daniel Scheffler committed
141
142
                                 msg='Written %s dataset cannot be found by PC.add_local_availability().'
                                     % (' '.join([ds['satellite'], ds['sensor'], ds['subsystem'], tgt_procL])))
143

144
145
146
        def test_L1A_processing(self):
            self.L1A_newObjects = self.PC.L1A_processing()
            self.assertIsInstance(self.L1A_newObjects, list)
147
            self.assertNotEqual(len(self.L1A_newObjects), 0, msg='L1A_processing did not output an L1A object.')
148
149
            self.assertIsInstance(self.L1A_newObjects[0], L1A_object)

150
            # check if PC.add_local_availability finds the written dataset
Daniel Scheffler's avatar
Daniel Scheffler committed
151
152
            if self.PC.config.exec_L1AP[1]:
                self.check_availability(self.L1A_newObjects, 'L1A')
153

154
155
156
        def test_L1B_processing(self):
            self.L1B_newObjects = self.PC.L1B_processing()
            self.assertIsInstance(self.L1B_newObjects, list)
157
            self.assertNotEqual(len(self.L1B_newObjects), 0, msg='L1B_processing did not output an L1B object.')
158
159
            self.assertIsInstance(self.L1B_newObjects[0], L1B_object)

160
            # check if PC.add_local_availability finds the written dataset
Daniel Scheffler's avatar
Daniel Scheffler committed
161
162
            if self.PC.config.exec_L1BP[1]:
                self.check_availability(self.L1B_newObjects, 'L1B')
163

164
165
166
        def test_L1C_processing(self):
            self.L1C_newObjects = self.PC.L1C_processing()
            self.assertIsInstance(self.L1C_newObjects, list)
167
            self.assertNotEqual(len(self.L1C_newObjects), 0, msg='L1C_processing did not output an L1C object.')
168
169
            self.assertIsInstance(self.L1C_newObjects[0], L1C_object)

170
            # check if PC.add_local_availability finds the written dataset
Daniel Scheffler's avatar
Daniel Scheffler committed
171
172
            # if self.PC.config.exec_L1CP[1]:
            #     self.check_availability(self.L1C_newObjects, 'L1C')
173

174
175
176
        def test_L2A_processing(self):
            self.L2A_newObjects = self.PC.L2A_processing()
            self.assertIsInstance(self.L2A_newObjects, list)
177
            self.assertNotEqual(len(self.L2A_newObjects), 0, msg='L2A_processing did not output an L2A object.')
178
179
            self.assertIsInstance(self.L2A_newObjects[0], L2A_object)

180
            # check if PC.add_local_availability finds the written dataset
181
            # FIXME this will fail because AC outputs TOA-Ref if ECMWF data are missing
Daniel Scheffler's avatar
Daniel Scheffler committed
182
183
            # if self.PC.config.exec_L2AP[1]:
            #     self.check_availability(self.L2A_newObjects, 'L2A')
184

185
186
187
        def test_L2B_processing(self):
            self.L2B_newObjects = self.PC.L2B_processing()
            self.assertIsInstance(self.L2B_newObjects, list)
188
            self.assertNotEqual(len(self.L2B_newObjects), 0, msg='L2B_processing did not output an L2B object.')
189
190
            self.assertIsInstance(self.L2B_newObjects[0], L2B_object)

191
            # check if PC.add_local_availability finds the written dataset
192
            # FIXME this will fail because AC outputs TOA-Ref if ECMWF data are missing
Daniel Scheffler's avatar
Daniel Scheffler committed
193
194
            # if self.PC.config.exec_L2BP[1]:
            #     self.check_availability(self.L2B_newObjects, 'L2B')
195

196
197
198
        def test_L2C_processing(self):
            self.L2C_newObjects = self.PC.L2C_processing()
            self.assertIsInstance(self.L2C_newObjects, list)
199
            self.assertNotEqual(len(self.L2C_newObjects), 0, msg='L2C_processing did not output an L2C object.')
200
            # self.assertIsInstance(self.L2C_newObjects[0], L2C_object)
201
202

            # check if PC.add_local_availability finds the written dataset
203
            # FIXME this will fail because AC outputs TOA-Ref if ECMWF data are missing
Daniel Scheffler's avatar
Daniel Scheffler committed
204
205
            # if self.PC.config.exec_L2CP[1]:
            #     self.check_availability(self.L2C_newObjects, 'L2C')  # FIXME fails (not yet working)
206

Daniel Scheffler's avatar
Daniel Scheffler committed
207
            # Setting the config.status manually.
208
            # if self.L2C_newObjects:
209
            #     self.PC.config.status = "finished"
210
211
            # FIXME after updating the job.status-attribute for the level-processes, delete the code that is commented
            # FIXME out.
212

Daniel Scheffler's avatar
Daniel Scheffler committed
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
    class TestCompletePipeline(unittest.TestCase):
        PC = None  # default

        @classmethod
        def tearDownClass(cls):
            cls.PC.config.DB_job_record.delete_procdata_of_entire_job(force=True)

        @classmethod
        def validate_db_entry(cls, filename):
            sceneID_res = get_info_from_postgreSQLdb(cls.PC.config.conn_database, 'scenes', ['id'],
                                                     {'filename': filename})
            assert sceneID_res and isinstance(sceneID_res[0][0], int), 'Invalid database entry.'

        @classmethod
        def create_job(cls, jobID, config):
228
            cls.PC = ProcessController(jobID, **config)
Daniel Scheffler's avatar
Daniel Scheffler committed
229
230
231

            [cls.validate_db_entry(ds['filename']) for ds in cls.PC.config.data_list]

232
233
            cls.PC.config.CPUs_all_jobs = 3
            cls.PC.config.max_parallel_reads_writes = 3
234
235
236
            cls.PC.config.spathomo_estimate_accuracy = True
            cls.PC.config.ac_estimate_accuracy = True
            cls.PC.config.spechomo_estimate_accuracy = True
237
238
            # cls.PC.config.exec_L1CP = [1, 1, 0]
            # cls.PC.config.exec_2ACP = [1, 1, 0]
239

Daniel Scheffler's avatar
Bugfix.    
Daniel Scheffler committed
240
241
        def test_run_all_processors(self):
            self.PC.run_all_processors()
Daniel Scheffler's avatar
Daniel Scheffler committed
242
            self.assertIsInstance(self.PC.L2C_newObjects, list)
243
244
245
246
            self.assertIsInstance(self.PC.summary_detailed, pandas.DataFrame)
            self.assertFalse(self.PC.summary_detailed.empty)
            self.assertIsInstance(self.PC.summary_quick, pandas.DataFrame)
            self.assertFalse(self.PC.summary_quick.empty)
247
248

###################################################################################
249
# Test cases 1-9: Test_<Satellite-Dataset>_<PreCollection or Collection>Data
250
251
252
253
254
255
256
# Test case 10: Test_MultipleDatasetsInOneJob


# TESTDATA-CLASSES.
class Test_Landsat5_PreCollectionData(BaseTestCases.TestAll):
    """
    Parametrized testclass. Tests the level-processes on a Landsat-5 TM scene (pre-collection data).
257
    More information on the dataset will be output after the tests-classes are executed.
258
259
260
261
262
    """
    @classmethod
    def setUpClass(cls):
        cls.create_job(26186263, job_config_kwargs)

263
264
265
266
267
268
269
270
271
272
273

class Test_Landsat5_PreCollectionData_CompletePipeline(BaseTestCases.TestCompletePipeline):
    """
    Parametrized testclass. Tests the level-processes on a Landsat-5 TM scene (pre-collection data).
    More information on the dataset will be output after the tests-classes are executed.
    """
    @classmethod
    def setUpClass(cls):
        cls.create_job(26186263, job_config_kwargs)


274
275
276
277
278
279
280
281
282
# class Test_Landsat5_CollectionData(BaseTestCases.TestAll):
#     """
#     Parametrized testclass. Tests the level-processes on a Landsat-5 TM scene (collection data).
#     More information on the dataset will be outputted after the tests-classes are executed.
#     """
#     @classmethod
#     def setUpClass(cls):
#         cls.create_job(26186263, job_config_kwargs) # FIXME job_ID!

Daniel Scheffler's avatar
Daniel Scheffler committed
283

284
285
286
class Test_Landsat7_SLC_on_PreCollectionData(BaseTestCases.TestAll):
    """
    Parametrized testclass. Tests the level-processes on a Landsat-7 ETM+_SLC_ON scene (pre-collection data).
287
    More information on the dataset will be output after after the tests-classes are executed.
288
289
290
291
292
    """
    @classmethod
    def setUpClass(cls):
        cls.create_job(26186262, job_config_kwargs)

Daniel Scheffler's avatar
Daniel Scheffler committed
293

294
295
296
class Test_Landsat7_SLC_off_PreCollectionData(BaseTestCases.TestAll):
    """
    Parametrized testclass. Tests the level-processes on a Landsat-7 ETM+_SLC_OFF scene (pre-collection data).
297
    More information on the dataset will be output after the tests-classes are executed.
298
299
300
301
302
    """
    @classmethod
    def setUpClass(cls):
        cls.create_job(26186267, job_config_kwargs)

Daniel Scheffler's avatar
Daniel Scheffler committed
303

304
305
306
307
308
309
310
311
312
# class Test_Landsat7_SLC_off_CollectionData(BaseTestCases.TestAll):
#     """
#     Parametrized testclass. Tests the level-processes on a Landsat-7 ETM+_SLC_OFF scene (collection data).
#     More information on the dataset will be outputted after the tests-classes are executed.
#     """
#     @classmethod
#     def setUpClass(cls):
#         cls.create_job(26186267, job_config_kwargs) # FIXME job_ID!

313
#
314
315
316
class Test_Landsat8_PreCollectionData(BaseTestCases.TestAll):
    """
    Parametrized testclass. Tests the level-processes on a Landsat-8 OLI_TIRS scene (pre-collection data).
317
    More information on the dataset will be output after the tests-classes are executed.
318
319
320
    """
    @classmethod
    def setUpClass(cls):
321
322
323
        # cls.create_job(26186196, job_config_kwargs) # LC80010702013141LGN01.tar.gz: in the west of South America, fails due to missing MGRS tile specifications  # noqa
        cls.create_job(26192300, job_config_kwargs)  # LC81900252013105LGN01.tar.gz
        # cls.create_job(26192301, job_config_kwargs)  # LC82000292015245LGN00.tar.gz: fails within SpatialIndexMediator
324

Daniel Scheffler's avatar
Daniel Scheffler committed
325

326
327
328
class Test_Landsat8_CollectionData(BaseTestCases.TestAll):
    """
    Parametrized testclass. Tests the level-processes on a Landsat-8 OLI_TIRS scene (collection data).
329
    More information on the dataset will be output after the tests-classes are executed.
330
331
332
    """
    @classmethod
    def setUpClass(cls):
333
        cls.create_job(26188372, job_config_kwargs)
334

Daniel Scheffler's avatar
Daniel Scheffler committed
335

336
337
338
339
340
341
342
343
344
class Test_Landsat8_CollectionData_CompletePipeline(BaseTestCases.TestCompletePipeline):
    """
    Parametrized testclass. Tests the level-processes on a Landsat-8 OLI_TIRS scene (collection data).
    More information on the dataset will be output after the tests-classes are executed.
    """
    @classmethod
    def setUpClass(cls):
        cfg = job_config_kwargs
        # cfg.update(dict(inmem_serialization=True))
345
        cls.create_job(26188372, cfg)
346
347


348
class Test_Sentinel2A_SingleGranuleFormat(BaseTestCases.TestAll):
349
    """
350
351
    Parametrized testclass. Tests the level-processes on a Sentinel-2A MSI scene (1 granule in archive: > 2017).
    More information on the dataset will be output after the tests-classes are executed.
352
353
354
355
356
    """
    @classmethod
    def setUpClass(cls):
        cls.create_job(26186268, job_config_kwargs)

Daniel Scheffler's avatar
Daniel Scheffler committed
357

Daniel Scheffler's avatar
Daniel Scheffler committed
358
359
360
361
362
363
364
365
366
class Test_Sentinel2A_SingleGranuleFormat_CompletePipeline(BaseTestCases.TestCompletePipeline):
    """
    Parametrized testclass. Tests the level-processes on a Sentinel-2A MSI scene (1 granule in archive: > 2017).
    More information on the dataset will be output after the tests-classes are executed.
    """
    @classmethod
    def setUpClass(cls):
        cls.create_job(26186268, job_config_kwargs)

367
368
369
    # @classmethod
    # def tearDownClass(cls):
    #     super().tearDownClass()
370
371
        # PC = cls.PC

Daniel Scheffler's avatar
Daniel Scheffler committed
372

373
class Test_Sentinel2A_MultiGranuleFormat(BaseTestCases.TestAll):
374
    """
375
376
    Parametrized testclass. Tests the level-processes on a Sentinel-2A MSI scene (multiple granules in archive: < 2017).
    More information on the dataset will be output after the tests-classes are executed.
377
378
379
380
381
    """
    @classmethod
    def setUpClass(cls):
        cls.create_job(26186272, job_config_kwargs)

Daniel Scheffler's avatar
Daniel Scheffler committed
382

383
384
385
386
387
388
389
390
391
392
class Test_Sentinel2B_SingleGranuleFormat(BaseTestCases.TestAll):
    """
    Parametrized testclass. Tests the level-processes on a Sentinel-2B MSI scene (1 granule in archive: > 2017).
    More information on the dataset will be output after the tests-classes are executed.
    """
    @classmethod
    def setUpClass(cls):
        cls.create_job(26186937, job_config_kwargs)


393
394
395
396
397
398
399
400
401
402
class Test_MultipleDatasetsInOneJob(BaseTestCases.TestAll):
    """
    Parametrized testclass. Tests the level-processes on a job containing a Landsat-5 (pre-collection data),
    Landsat-7 SLC_off (pre-collection data) and a Sentinel-2A (collection data) scene.
    """
    @classmethod
    def setUpClass(cls):
        cls.create_job(26186273, job_config_kwargs)


Daniel Scheffler's avatar
Daniel Scheffler committed
403
404
405
406
407
408
409
410
411
class Test_MultipleDatasetsInOneJob_CompletePipeline(BaseTestCases.TestCompletePipeline):
    """
    Parametrized testclass. Tests the level-processes on a job containing a Landsat-5 (pre-collection data),
    Landsat-7 SLC_off (pre-collection data) and a Sentinel-2A (collection data) scene.
    """
    @classmethod
    def setUpClass(cls):
        cls.create_job(26186273, job_config_kwargs)

412
413
414
    # @classmethod
    # def tearDownClass(cls):
    #     super().tearDownClass()
415
416
        # PC = cls.PC

Daniel Scheffler's avatar
Daniel Scheffler committed
417

418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
class Test_ProcessContinuing_CompletePipeline(unittest.TestCase):
    """
    Parametrized testclass. Tests the level-processes on a job containing a Landsat-5 (pre-collection data),
    Landsat-7 SLC_off (pre-collection data) and a Sentinel-2A (collection data) scene.
    """
    PC = None  # default

    @classmethod
    def tearDownClass(cls):
        cls.PC.config.DB_job_record.delete_procdata_of_entire_job(force=True)

    @classmethod
    def validate_db_entry(cls, filename):
        sceneID_res = get_info_from_postgreSQLdb(cls.PC.config.conn_database, 'scenes', ['id'],
                                                 {'filename': filename})
        assert sceneID_res and isinstance(sceneID_res[0][0], int), 'Invalid database entry.'

    @classmethod
    def create_job(cls, jobID, config):
437
        cls.PC = ProcessController(jobID, **config)
438
439
440
441
442
443
444

        cls.PC.logger.info('Execution of entire GeoMultiSens pre-processing chain started for job ID %s...'
                           % cls.PC.config.ID)

        [cls.validate_db_entry(ds['filename']) for ds in cls.PC.config.data_list]

    def setUp(self):
Daniel Scheffler's avatar
Daniel Scheffler committed
445
        self.cfg_kw = job_config_kwargs.copy()  # copy, because job_config_kwargs is modified otherwise
446
447
448
449
450
451
452
453
        self.cfg_kw.update(dict(
            exec_L1BP=[False, False, False],
            exec_L1CP=[False, False, False],
            exec_L2AP=[False, False, False],
            exec_L2BP=[False, False, False],
            exec_L2CP=[False, False, False]
        ))

454
455
456
    def test_continue_from_L1A(self):
        cfg_kw = self.cfg_kw

457
        # produce L1A data and stop processing there
458
        self.create_job(26186263, cfg_kw)  # 1x L5 pre-collection
459
460
461
462
463
464
465
466
        self.PC.run_all_processors()

        # create a new job and try to continue from L1A
        cfg_kw.update(dict(
            exec_L1BP=[True, True, False],
            delete_old_output=False
        ))
        self.create_job(26186263, cfg_kw)  # 1x L5 pre-collection
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
        self.PC.run_all_processors()

    def test_continue_from_L2C(self):
        cfg_kw = self.cfg_kw
        cfg_kw.update(dict(
            exec_L1AP=[True, True, True],
            exec_L1BP=[True, True, True],
            exec_L1CP=[True, True, True],
            exec_L2AP=[True, True, True],
            exec_L2BP=[True, True, False],
            exec_L2CP=[True, True, False],
        ))

        # produce L1A data and stop processing there
        self.create_job(26186263, cfg_kw)  # 1x L5 pre-collection
        self.PC.run_all_processors()

        # create a new job and try to continue from L2C
        cfg_kw.update(dict(
            delete_old_output=False
        ))
        self.create_job(26186263, cfg_kw)  # 1x L5 pre-collection
489
490
491
        self.PC.run_all_processors()


492
493
494
495
496
497
498
499
###################################################################################
# Summarizing the information regarding the test datasets.

# The information: 'country' (3-letter country code, UN), 'characteristic features of the shown scene', 'cloud cover
# present' and 'overlap area present' of each dataset are summarized in the dictionary "testdata_scenes". The
# information are sorted according to the testdata.
# 3-letter code:
# UKR-Ukraine, KGZ-Kyrgyztan, POL-Poland, AUT-Austria, JPN-Japan, BOL-Bolivia, TUR-Turkey, DEU-Germany, CHE-Switzerland.
500
501
502
503
504
505
506
507
508
509
510
testdata_scenes = \
    {'Landsat5_PreCollectionData': list(['UKR', 'City region, forest', 'Sparsely', 'Zone 34/35']),
     # 'Landsat5_CollectionData': list(['KGZ', 'Snowy Mountains', 'Yes', 'None']),
     'Landsat7_SLC_on_PreCollectionData': list(['POL', 'City region, lakes', 'Yes', 'None']),
     'Landsat7_SLC_off_PreCollectionData': list(['AUT', 'Stripes (partly), Mountains', 'None', 'None']),
     # 'Landsat7_SLC_off_CollectionData': list(['JPN', 'Stripes (completly), Mountains', 'Yes', 'Zone 53/54']),
     'Landsat8_PreCollectionData': list(['BOL', 'Forest', 'Yes', 'None']),
     'Landsat8_CollectionData': list(['TUR', 'Snowy Mountains', 'Yes', 'None']),
     'Sentinel2A_PreCollectionData': list(['DEU', 'Potsdam', 'Sparsely', 'None']),
     'Sentinel2A_CollectionData': list(['CHE', 'City region, on the Rhine', 'Yes', 'None'])
     }
511
512
513
514
515
516
517
518

# The key of the dictionary is the key-value to parametrize the testclasses so that each testclass is executed
# automatically.
testdata = list(testdata_scenes.keys())
testdata.append('MultipleDatasetsInOneJob')


###################################################################################
519
# Parameterizing the test cases and creating a summary of the test results.
520
521
522

summary_testResults, summary_errors, summary_failures, summary_skipped, jobstatus = [[] for _ in range(5)]

523

524
@unittest.SkipTest
525
526
class Test_in_normal_mode(unittest.TestCase):
    def setUp(self):
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
        # self.job_id = 26184107
        # self.job_id = 26185175   # 1x TM5
        # self.job_id = 26185176   # 1x Landsat
        # self.job_id = 26185177  # 1. Sentinel-2-Testszene
        # self.job_id = 26185189   # direkt benachbarte Granules von 1. Sentinel-2-Testszene
        # self.job_id = 26185237  # 4 x Landsat-8 -> Job per database tools erstellt
        # self.job_id = 26185239  # 50 x Landsat-8 -> Job per database tools erstellt - 1. L8 Beta-Testjob
        # self.job_id = 26185242  # 1 x Landsat-8 - Bug files_in_archive=None
        # self.job_id = 26185250  # Beta-Job - 219 x L8, 172 x L7, 111 x S2, spatref S2
        # self.job_id = 26185251  # 1x L8, Zielsensor L8
        # self.job_id = 26185252  # 1x L8, Zielsensor L8, spat.ref L8
        # self.job_id = 26185253  # 25x L8, Zielsensor L8, spat.ref L8
        # self.job_id = 26185254  # 10x L8, Zielsensor L8, spat.ref L8
        # Grund=Schreibfehler L1A im tiled Python-mode bei mehr als 1 Szene im Job:
        # self.job_id = 26185255  # 1x L8 Bug 5 corners found
        # self.job_id = 26185256  # 1x L7 SLC off, Zielsensor L8, spat.ref L8
        # self.job_id = 26185257  # Beta-Job - 219 x L8, 172 x L7, 111 x S2, spatref L8
        # self.job_id = 26185258  # Beta-Job - 219 x L8, spatref L8
        # self.job_id = 26185259  # Beta-Job - 172 x L7, spatref L8
        # self.job_id = 26185260  # Beta-Job - 111 x S2, spatref L8
        # self.job_id = 26185268  # 25x L7 SLC off, Zielsensor L8, spat.ref L8
        # self.job_id = 26185269  # 1x L7 SLC off, Bug SpatialIndexMediator
        # self.job_id = 26185270  # 5x L7 SLC off, Bug SpatialIndexMediator
        # self.job_id = 26185275  # 1x L8, spat. Ref. L8 Bug L1B_mask not found
        # self.job_id = 26185264  # 1x L8, Bug L1B_masks not found
        # self.job_id = 26185265  # 1x L8, Bug L2B_masks not found
        # self.job_id = 26185268  # "2x L8, Bug L2B_masks not found, incl. 1x bad archive"
        # self.job_id = 26185269 # "10x L8, Bug L2B_masks not found"
        # self.job_id = 26185272 # "1x S2A Sips"
        # self.job_id = 26185273  # "1x L7, target L8, spat.ref L8"
        # self.job_id = 26185275 # "1x L7, target L8, spat.ref L8 L1B Matching failed"
        # self.job_id = 26185276 # "1x L7, target L8, spat.ref L8 L1B Matching window became too small."
        # self.job_id = 26185279 # "GEOMS: 25x L7, target L8, spat.ref L8"
        # "GEOMS: 1x L7, target L8, spat.ref L8, debugging NoneType object is not subscriptable within
        # mapinfo2geotransform":
        # self.job_id = 26185280
        # self.job_id = 26185281 # "GEOMS: 4x L7, target L8, spat.ref L8, freeze of pool.map"
        # self.job_id = 26185283 # "GEOMS: 10x L7, target L8, spat.ref L8, freeze of pool.map"
        # self.job_id = 26185284 # "GEOMS: 11x L7, target L8, spat.ref L8, freeze of pool.map"
        # self.job_id = 26185321 # "GEOMS: 1x L7, target L8, spat.ref L8, debugging L1B_P"
        # "GEOMS: 1x L7, target L8, spat.ref L8, Bug calc_shifted_cross_power_spectrum: NoneType object not iterable":
        # self.job_id = 26185322
        # self.job_id = 26185277 # "GMS41: 10x L7, target L8, spat.ref L8, Permission errors during logging"
        # self.job_id = 26185278 # "Beta-Job - 172 x L7, spatref L8"
        # self.job_id = 26185284 # "GMS41: "all beta-L8 with cloud cover <30% (74 scenes)"
        # self.job_id = 26185285 # "GMS41: "all beta-L7 with cloud cover <30% (49 scenes)"
        # self.job_id = 26185396 # "GEOMS: 1x S2A multi GSD testing"
        # self.job_id = 26185398  # "GEOMS: 1x S2A granule multi GSD testing"

576
        # self.job_id = 26186740  # Testjob Landsat-8
577
        # self.job_id = 26186906  # Bug Input Validator
578
        # self.job_id = 26186925  # 1 Sentinel-2A, Bug NoneType' object has no attribute 'find'
579
580
        # self.job_id = 26187051  # GMS41: 1 Landsat, FileNotFoundError
        # self.job_id = 26187052  # GMS41: 1 Landsat, DB query returns no DEM
581
        # self.job_id = 26187053  # GMS41: AC: The input 'list_GMS_objs' contains duplicates: ['', '']
582
        # self.job_id = 26187750  # GEOMS: [AC]: RuntimeWarning: All-NaN slice encountered
583
        # self.job_id = 26187760  # GEOMS: [L2C]: ValueError: 'axis' entry is out of bounds
584
        # self.job_id = 26187804  # GEOMS: Spatial homogenization leaves resampling artifacs at the image edges.
585
        # self.job_id = 26187922  # GEOMS: AssertionError (self.job_id = 26187922  # GEOMS: AssertionError)
Daniel Scheffler's avatar
Daniel Scheffler committed
586
587
        # self.job_id = 26188163  # GEOMS: pandas.errors.ParserError: Expected 2 fields in line 31, saw 3
        self.job_id = 26189301  # GEOMS: process continuation
588

589
590
        self.PC = ProcessController(self.job_id, **dict(is_test=False, parallelization_level='scenes', db_host=db_host,
                                                        delete_old_output=True, disable_exception_handler=True))
591
592
593
        # self.PC.config.spathomo_estimate_accuracy = True
        # self.PC.config.ac_estimate_accuracy = True
        # self.PC.config.spechomo_estimate_accuracy = True
594
595
596
        # self.PC.config.exec_L1CP = [1, 1, 0]
        # self.PC.config.exec_2ACP = [1, 1, 0]
        # self.PC.config.path_procdata_scenes = '/storage/gms/processed_scenes/20180227_MGRS33UUU_S2_L8_L7/'
597
598
599
600
601

    def test(self):
        self.PC.run_all_processors()


602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
if __name__ == '__main__':
    # Part 1: Creating and running a testsuite for each dataset-testcase, and querying the job.status of the job.
    for items in testdata:
        suite = unittest.TestLoader().loadTestsFromTestCase(eval("Test_"+items))
        alltests = unittest.TestSuite(suite)

        # Part 2: Saving the results of each testsuite and the query for the job.status in individual variables.
        testResult = unittest.TextTestRunner(verbosity=2).run(alltests)

        summary_testResults.append([testResult.testsRun, testResult.wasSuccessful(),
                                    len(testResult.errors), len(testResult.failures),
                                    len(testResult.skipped)])
        summary_errors.append(testResult.errors)
        summary_failures.append(testResult.failures)
        summary_skipped.append(testResult.skipped)

618
        # FIXME: If the job.status-issue is fixed, the commented out section can be nullified.
619
        # jobstatus.append(eval("Test_"+items).PC.status)
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635

    # Part 3: Summarizing the testresults of each testsuite and outputting the results in an orderly fashion on the
    # console and in a textfile.
    # Note that the testresults are outputted as usual after each test is executed. Since the output of each
    # level-process is rather long, the output of the testresults become lost. Therefore, the purpose to output the
    # testresults again is simply to summarize the testresults in one place and to give an overview over the results.

    # Output: a) Information on the test datasets (table), b) testresults summarized in a table, c)if existing,
    # a list of errors, failures and skips in the testcases and d) the job.status that is not set to "finished".

    time.sleep(0.5)

    # Path of the textfile the results will be logged to.
    test_log_path = os.path.join(gmsRepo_rootpath, 'tests', 'data', 'logs', time.strftime('%Y%m%d_%H%M%S_log.txt'))

    # Creating a logging system for the testresults.
636
637
    # Source: The "GMS_logger"-function in the "gms_preprocessing" --> "misc" --> "logging.py"-script was used and
    # slightly altered to meet the needs of the current problem.
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
    logger = logging.getLogger("log_Test")
    logger.setLevel(logging.INFO)

    # Defining the format of the console and the file-output.
    formatter_fileH = logging.Formatter('')
    formatter_ConsoleH = logging.Formatter('')

    # Creating a handler for the file for the logging level "INFO".
    fileHandler = logging.FileHandler(test_log_path)
    fileHandler.setFormatter(formatter_fileH)
    fileHandler.setLevel(logging.INFO)

    # Creating a handler for the console for the logging level "INFO". "sys.stdout" is used for the logging output.
    consoleHandler_out = logging.StreamHandler(stream=sys.stdout)
    consoleHandler_out.setFormatter(formatter_ConsoleH)
    consoleHandler_out.set_name('console handler stdout')
    consoleHandler_out.setLevel(logging.INFO)

    # Adding the defined handlers to the instantiated logger.
    logger.addHandler(fileHandler)
    logger.addHandler(consoleHandler_out)

    # OUPUT, START.
    # Header of the file.
662
    logger.info("\ntest_gms_preprocessing.py"
663
664
665
666
667
668
                "\nREVIEW OF ALL TEST RESULTS, SUMMARY:"
                "\n***************************************************************************************"
                "\n--> SPECIFIC FEATURES OF DATA:")

    # Adding a table displaying the characteristic features of each dataset.
    logger.info(pandas.DataFrame.from_items(testdata_scenes.items(),
669
670
                                            orient='index',
                                            columns=['Country', 'Characteristic', 'Clouds', 'Overlap_area']))
671
672
673
674
675
676
677
678
679
680
    logger.info("\nThe jobID used in Test_" + testdata[-1] + " contains the datasets: "
                "\n-Landsat5_PreCollectionData,\n-Landsat7_SLC_off_PreCollectionData and "
                "\n-Sentinel2A_CollectionData.")

    # Adding a table displaying the testresults.
    logger.info("\n***************************************************************************************"
                "\n--> TESTRESULTS:")

    results = ["Run", "Success", "Errors", "Failures", "Skips"]
    testdata_index = ["Test_" + item for item in testdata]
681
    logger.info(pandas.DataFrame(summary_testResults, columns=results, index=testdata_index))
682
683
684
685
686

    # If errors, failures or skips (there is yet nothing to skip in the code) occurres, the respective message will
    # be printed.
    logger.info("\n***************************************************************************************")
    if list(itertools.chain(*summary_errors)) or list(itertools.chain(*summary_failures)) or \
687
       list(itertools.chain(*summary_skipped)):
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
        logger.info("--> ERRORS/FAILURES/SKIPS:")
        logger.info("\n---------------------------------------------------------------------------------------")

        for index, test in enumerate(testdata):
            logger.info("Test_" + test + ", ERRORS:")
            if summary_errors[index]:
                logger.info(summary_errors[index][0][1])
            else:
                logger.info("None. \n")

            logger.info("Test_" + test + ", FAILURES:")
            if summary_failures[index]:
                logger.info(summary_failures[index][0][1])
            else:
                logger.info("None. \n")
703

704
705
706
707
708
            logger.info("Test_" + test + ", SKIPS:")
            if summary_skipped[index]:
                logger.info(summary_skipped[index][0][1])
            else:
                logger.info("None.")
709

710
711
            if not index == (len(testdata) - 1):
                logger.info("\n---------------------------------------------------------------------------------------")
712

713
        logger.info("\n***************************************************************************************")
714

715
716
    else:
        pass
717

718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
    # Checking, if the job.status of each job is set to "finished". Is it not set to "finished", a dataframe is created
    # containing the test-name with and the different job.status itself.
    # FIXME: If the job.status-issue is fixed, the commented out section can be nullified.
    # jobstatus_table, index_table = [[] for _ in range(2)]
    # for index, test in enumerate(testdata):
    #     if jobstatus[index] != "finished":
    #         jobstatus_table.append(jobstatus[index])
    #         index_table.append("Test_" + test)
    #
    # if jobstatus_table:
    #     logger.info("--> WARNING!!! JOBSTATUS of the following testcase(s) is not set to 'finished': \n")
    #     logger.info(pandas.DataFrame(jobstatus_table, columns=["jobstatus"], index=index_table))
    #     logger.info("\n***************************************************************************************")
    # else:
    #     pass
733

734
    logger.info("END.")  # OUTPUT, END.
735

736
    # Delete the handlers added to the "log_Test"-logger to ensure that no message is output twice in a row, when
737
738
    # the logger is used again.
    logger.handlers = []