gms_object.py 71.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
# -*- coding: utf-8 -*-
__author__='Daniel Scheffler'

import collections
import copy
import datetime
import glob
import json
import os
import re
import shutil
import sys
import warnings
14
import logging
15
from itertools import chain
16
17
18

import numpy as np
import spectral
19
from spectral.io import envi
20
from numba import jit
21
from pandas import DataFrame, read_csv
22
23
24
25
try:
    from osgeo import gdalnumeric
except ImportError:
    import gdalnumeric
26

27
from geoarray import GeoArray
28
29
30
31
from py_tools_ds.geo.coord_grid import is_coord_grid_equal
from py_tools_ds.geo.map_info import geotransform2mapinfo, mapinfo2geotransform
from py_tools_ds.geo.coord_calc import calc_FullDataset_corner_positions
from py_tools_ds.geo.coord_trafo import pixelToLatLon, pixelToMapYX
32
from sicor.options import get_options as get_ac_options
33

34
35
36
37
38
39
40
41
42
43
44
45
from ..misc.logging import GMS_logger as DatasetLogger
from ..misc.mgrs_tile import MGRS_tile
from ..model.METADATA import METADATA, get_dict_LayerOptTherm, metaDict_to_metaODict
from ..model.dataset import Dataset
from ..misc import path_generator as PG
from ..misc import database_tools as DB_T
from ..config import GMS_config as CFG
from ..algorithms import GEOPROCESSING as GEOP
from ..io import Input_reader as INP_R
from ..io import Output_writer as OUT_W
from ..misc import helper_functions as HLP_F
from ..misc import definition_dicts as DEF_D
46
47
48
49
50
51
52
53


class GMS_object(Dataset):

    def __init__(self, pathImage=''):
        # get all attributes of base class "Dataset"
        super(GMS_object, self).__init__()

54
        # add private attributes
55
        self._dict_LayerOptTherm = None
56
57
        self._cloud_masking_algorithm = None
        self._meta_odict = None
58
59
60

        self.job_ID = CFG.job.ID
        #self.dataset_ID = int(DB_T.get_info_from_postgreSQLdb(CFG.job.conn_database, 'scenes', ['datasetid'],
61
        #                                {'id': self.scene_ID})[0][0]) if self.scene_ID !=-9999 else -9999 # FIXME not needed anymore?
62
63
64
        self.scenes_proc_ID  = None  # set by Output writer after creation/update of db record in table scenes_proc
        self.mgrs_tiles_proc_ID = None  # set by Output writer after creation/update of db record in table mgrs_tiles_proc
        self.MGRS_info = None
65
66

        # set pathes
67
68
69
70
71
72
73
74
75
76
        self.path_cloud_class_obj = ''

        # handle initialization arguments
        if pathImage:
            self.arr = pathImage  # run the setter for 'arr' of the base class 'Dataset' which creates an Instance of GeoArray


    def __getstate__(self):
        """Defines how the attributes of GMS object are pickled."""

Daniel Scheffler's avatar
Bugfix    
Daniel Scheffler committed
77
78
        self.close_loggers()
        del self.pathGen # path generator can only be used for the current processing level
79
80

        # delete arrays if their in-mem size is to big to be pickled
81
        # => (avoids MaybeEncodingError: Error sending result: '[<gms_preprocessing.algorithms.L2C_P.L2C_object
82
83
84
85
        #    object at 0x7fc44f6399e8>]'. Reason: 'error("'i' format requires -2147483648 <= number <= 2147483647",)')
        if self.proc_level=='L2C' and CFG.job.exec_mode=='Flink':
            if self.mask_nodata is not None and self.masks.bands>1 and self.mask_clouds is not None: # FIXME check by bandname
                del self.masks
86

Daniel Scheffler's avatar
Bugfix    
Daniel Scheffler committed
87
88
        return self.__dict__

89
90

    def set_pathes(self):
91
92
93
94
        self.baseN                = self.pathGen.get_baseN()
        self.path_procdata        = self.pathGen.get_path_procdata()
        self.ExtractedFolder      = self.pathGen.get_path_tempdir()
        self.path_logfile         = self.pathGen.get_path_logfile()
95
        self.pathGen              = PG.path_generator(self.__dict__) # passes a logger in addition to previous attributes
96
        self.path_archive         = self.pathGen.get_local_archive_path_baseN()
97
98
        self.path_cloud_class_obj = PG.get_path_cloud_class_obj(self.GMS_identifier,
                                                                get_all=True if CFG.job.bench_cloudMask else False)
99
        if CFG.job.exec_mode=='Python':
Daniel Scheffler's avatar
Daniel Scheffler committed
100
101
            self.path_InFilePreprocessor = os.path.join(self.ExtractedFolder, '%s%s_DN.bsq'
                                                %(self.entity_ID, ('_%s'%self.subsystem if self.subsystem else '')))
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
        else: # Flink
            self.path_InFilePreprocessor = None # None: keeps all produced data in memory (numpy array attributes)

        self.path_MetaPreprocessor = self.path_archive


    def validate_pathes(self):
        if not os.path.isfile(self.path_archive) and not os.path.isdir(self.path_archive):
            self.logger.info("The %s dataset '%s' has not been processed earlier and no corresponding raw data archive"
                "has been found at %s." %(self.sensor,self.entity_ID,self.path_archive))
            self.logger.info('Trying to download the dataset...')
            self.path_archive_valid = self._data_downloader(self.sensor,self.entity_ID)
        else:
            self.path_archive_valid = True

        if CFG.job.exec_mode=='Python' and self.ExtractedFolder and not os.path.isdir(self.ExtractedFolder):
            os.makedirs(self.ExtractedFolder)

        assert os.path.exists(self.path_archive), 'Invalid path to RAW data. File %s does not exist at %s.' \
                                                  % (os.path.basename(self.path_archive),
                                                     os.path.dirname(self.path_archive))
        assert isinstance(self.path_archive, str), 'Invalid path to RAW data. Got %s instead of string or unicode.' \
                                                   % type(self.path_archive)
        if CFG.job.exec_mode == 'Python' and self.ExtractedFolder: assert os.path.exists(self.path_archive), \
            'Invalid path for temporary files. Directory %s does not exist.' % self.ExtractedFolder


    @property
    def logger(self):
        if self._loggers_disabled:
            return None
        if self._logger and self._logger.handlers[:]:
            return self._logger
        else:
136
137
            self._logger = DatasetLogger('log__' + self.baseN, fmt_suffix=self.scene_ID, path_logfile=self.path_logfile,
                                         log_level=CFG.job.log_level, append=True)
138
139
140
141
142
            return self._logger


    @logger.setter
    def logger(self, logger):
143
144
145
146
147
148
        assert isinstance(logger, logging.Logger) or logger in ['not set', None], \
            "GMS_obj.logger can not be set to %s." %logger

        # save prior logs
        #if logger is None and self._logger is not None:
        #    self.log += self.logger.captured_stream
149
150
151
        self._logger = logger


152

153
154
    @property
    def GMS_identifier(self):
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
        return collections.OrderedDict(zip(
            ['image_type', 'Satellite', 'Sensor', 'Subsystem', 'proc_level', 'dataset_ID', 'logger'],
            [self.image_type, self.satellite, self.sensor, self.subsystem, self.proc_level, self.dataset_ID, self.logger]))


    @property
    def MetaObj(self):
        if self._meta_odict:
            # if there is already a meta_odict -> create a new MetaObj from it (ensures synchronization!)
            self._MetaObj = METADATA(self.GMS_identifier).from_odict(self._meta_odict)
            del self.meta_odict
        elif not self._MetaObj:
            # if there is no meta_odict and no MetaObj -> create MetaObj by reading metadata from disk
            pass  # reading from disk should use L1A_P.L1A_object.import_metadata -> so just return None

        return self._MetaObj


    @MetaObj.setter
    def MetaObj(self, MetaObj):
        assert isinstance(MetaObj, METADATA), "'MetaObj' can only be set to an instance of METADATA class. " \
                                              "Got %s." %type(MetaObj)
        self._MetaObj = MetaObj

        # update meta_odict
        del self.meta_odict # it is recreated if getter is used the next time


    @MetaObj.deleter
    def MetaObj(self):
        self._MetaObj = None


    @property
    def meta_odict(self):
        if self._MetaObj:
191
            # if there is already a MetaObj -> create new meta_odict from it (ensures synchronization!)
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
            self._meta_odict = self._MetaObj.to_odict()
            del self.MetaObj
        elif not self._meta_odict:
            # if there is no MetaObj and no meta_odict -> use MetaObj getter to read metadata from disk
            pass # reading from disk should use L1A_P.L1A_object.import_metadata -> so just return None
            self._meta_odict = None

        return self._meta_odict


    @meta_odict.setter
    def meta_odict(self, odict):
        assert isinstance(odict, (collections.OrderedDict, dict)),"'meta_odict' can only be set to an instance of " \
                                                          "collections.OrderedDict. Got %s." %type(odict)
        self._meta_odict = odict

        # update MetaObj
        del self.MetaObj # it is recreated if getter is used the next time


    @meta_odict.deleter
    def meta_odict(self):
        self._meta_odict = None


217
218
219
220
221
    @property
    def dict_LayerOptTherm(self):
        if self._dict_LayerOptTherm:
            return self._dict_LayerOptTherm
        elif self.LayerBandsAssignment:
222
            self._dict_LayerOptTherm = get_dict_LayerOptTherm(self.identifier,self.LayerBandsAssignment)
223
224
225
226
227
228
229
230
            return self._dict_LayerOptTherm
        else:
            return None


    @property
    def georef(self):
        """Returns True if the current dataset can serve as spatial reference."""
Daniel Scheffler's avatar
Daniel Scheffler committed
231

232
233
234
235
236
        return True if self.image_type == 'RSD' and re.search('OLI', self.sensor, re.I) else False


    @property
    def coreg_needed(self):
237
238
239
        if self._coreg_needed is None:
            self._coreg_needed = not (self.dataset_ID == CFG.usecase.datasetid_spatial_ref)
        return self._coreg_needed
240
241
242
243
244
245
246


    @coreg_needed.setter
    def coreg_needed(self, value):
        self._coreg_needed = value


247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
    @property
    def resamp_needed(self):
        if self._resamp_needed is None:
            gt = mapinfo2geotransform(self.meta_odict['map info'])
            self._resamp_needed = not is_coord_grid_equal(gt, CFG.usecase.spatial_ref_gridx, CFG.usecase.spatial_ref_gridy)
        return self._resamp_needed


    @resamp_needed.setter
    def resamp_needed(self, value):
        self._resamp_needed = value


    @property
    def masks(self):
262
263
264
265
266
        #if self.mask_nodata is not None and self.mask_clouds is not None and \
        #     self._masks is not None and self._masks.bands==1:

        #     self.build_combined_masks_array()

267
268
269
270
        return self._masks


    @masks.setter
271
    def masks(self, *geoArr_initArgs):
272
273
274
275
        """
        NOTE: This does not automatically update mask_nodata and mask_clouds BUT if mask_nodata and mask_clouds are
        None their getters will automatically synchronize!
        """
Daniel Scheffler's avatar
Daniel Scheffler committed
276

277
278
279
280
281
282
283
        if geoArr_initArgs[0] is not None:
            self._masks        = GeoArray(*geoArr_initArgs)
            self._masks.nodata = 0
            self._masks.gt     = self.arr.gt
            self._masks.prj    = self.arr.prj
        else:
            del self.masks
284
285


286
287
288
289
290
    @masks.deleter
    def masks(self):
        self._masks = None


291
292
293
294
295
296
297
    @property
    def cloud_masking_algorithm(self):
        if not self._cloud_masking_algorithm:
            self._cloud_masking_algorithm = CFG.job.cloud_masking_algorithm[self.satellite]
        return self._cloud_masking_algorithm


298
299
300
    @property
    def ac_options(self):
        """
301
302
        Returns the options dictionary needed as input for atmospheric correction. If an empty dictionary is returned,
        atmospheric correction is not yet available for the current sensor and will later be skipped.
303
        """
Daniel Scheffler's avatar
Daniel Scheffler committed
304

305
        if not self._ac_options:
306
307
            path_ac_options = PG.get_path_ac_options(self.GMS_identifier)

308
            if path_ac_options and os.path.exists(path_ac_options):
Daniel Scheffler's avatar
Daniel Scheffler committed
309
                opt_dict = get_ac_options(path_ac_options, validation=False)  # don't validate because options contain pathes that do not exist on another server
310

Daniel Scheffler's avatar
Daniel Scheffler committed
311
                # update some file paths depending on the current environment
312
313
314
315
316
                opt_dict['DEM']['fn'] = CFG.job.path_dem_proc_srtm_90m
                opt_dict['ECMWF']['path_db'] = CFG.job.path_ECMWF_db
                for key in opt_dict['RTFO']:
                    if 'atm_tables_fn' in opt_dict['RTFO'][key]:
                        opt_dict['RTFO'][key]['atm_tables_fn'] = PG.get_path_ac_table(key)
Daniel Scheffler's avatar
Daniel Scheffler committed
317
                opt_dict['S2Image']['S2_MSI_granule_path'] = None  # only a placeholder -> will always be None for GMS usage
318
                opt_dict['cld_mask']['persistence_file'] = PG.get_path_cloud_class_obj(self.GMS_identifier)
319
                opt_dict['cld_mask']['novelty_detector'] = None # FIXME update this after switching to SICOR
320
321
                opt_dict['output'] = [] # outputs are not needed for GMS -> so
                opt_dict['report']['report_path'] = os.path.join(self.pathGen.get_path_procdata(), '[TYPE]')
322
323
                if 'uncertainties' in opt_dict:
                    opt_dict['uncertainties']['snr_model'] = PG.get_path_snr_model(self.GMS_identifier)
324

Daniel Scheffler's avatar
Daniel Scheffler committed
325
                # opt_dict['AC']['n_cores'] = CFG.job.CPUs if CFG.job.allow_subMultiprocessing else 1
326

327
                self._ac_options = opt_dict
328
329
330
            else:
                self.logger.warning('There is no options file available for atmospheric correction. '
                                    'Atmospheric correction must be skipped.')
331

332
333
334
        return self._ac_options


335
    def get_copied_dict_and_props(self, remove_privates=False):
336
        # type: (bool) -> dict
337
        """Returns a copy of the current object dictionary including the current values of all object properties."""
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357

        # loggers must be closed
        self.close_GMS_loggers()
        self._loggers_disabled = True # this disables automatic recreation of loggers (otherwise loggers are created by using getattr())

        out_dict = self.__dict__.copy()

        # add properties
        property_names = [p for p in dir(self.__class__) if isinstance(getattr(self.__class__, p), property)]
        [out_dict.update({propK:copy.copy(getattr(self,propK))}) for propK in property_names]

        # remove private attributes
        if remove_privates:
            out_dict = {k: v for k, v in out_dict.items() if not k.startswith('_')}

        self._loggers_disabled = False  # enables automatic recreation of loggers

        return out_dict


358
359
    def attributes2dict(self, remove_privates=False):
        # type: (bool) -> dict
360
        """Returns a copy of the current object dictionary including the current values of all object properties."""
361
362
363

        # loggers must be closed
        self.close_GMS_loggers()
364
        self._loggers_disabled = True  # this disables automatic recreation of loggers (otherwise loggers are created by using getattr())
365
366
367
368

        out_dict = self.__dict__.copy()

        # add some selected property values
369
        for i in ['GMS_identifier', 'LayerBandsAssignment', 'coreg_needed', 'resamp_needed', 'dict_LayerOptTherm',
370
                  'georef', 'meta_odict']:
371
372
373
374
375
376
            out_dict[i] = getattr(self,i)

        # remove private attributes
        if remove_privates:
            out_dict = {k: v for k, v in out_dict.items() if not k.startswith('_')}

377
        self._loggers_disabled = False  # enables automatic recreation of loggers
378
379
380
        return out_dict


381
382
383
384
385
386
    def _data_downloader(self,sensor, entity_ID):
        self.logger.info('Data downloader started.')
        success = False
        " > download source code for Landsat here < "
        if not success:
            self.logger.critical("Download for %s dataset '%s' failed. No further processing possible." %(sensor,entity_ID))
387
            raise RuntimeError('Archive download failed.')
388
389
390
        return success


391
392
393
394
395
    def from_disk(self, tuple_GMS_subset):
        """Fills an already instanced GMS object with data from disk. Excludes array attributes in Python mode.

        :param tuple_GMS_subset:    <tuple> e.g. ('/path/gms_file.gms', ['cube', None])
        """
396

397
398
        path_GMS_file = tuple_GMS_subset[0]
        GMSfileDict   = INP_R.GMSfile2dict(path_GMS_file)
399
400

        # copy all attributes from GMS file (private attributes are not touched since they are not included in GMS file)
401
        self.meta_odict = GMSfileDict['meta_odict'] # set that first in order to make some getters and setters work
402
403
404
405
406
        for key, value in GMSfileDict.items():
            if key in ['GMS_identifier', 'georef', 'dict_LayerOptTherm']:
                continue # properties that should better be created on the fly
            try:
                setattr(self, key, value)
407
            except:
408
409
                raise AttributeError("Can't set attribute %s." %key)

410
        self.acq_datetime        = datetime.datetime.strptime(self.acq_datetime, '%Y-%m-%d %H:%M:%S.%f%z')
411
412
        self.arr_shape, self.arr_pos = tuple_GMS_subset[1]

413
414
415
        self.arr        = self.pathGen.get_path_imagedata()
        self.masks      = self.pathGen.get_path_maskdata() # self.mask_nodata and self.mask_clouds are auto-synchronized via self.masks (see their getters)

416
417
418
        return copy.copy(self)


419
420
421
422
423
424
425
426
    def from_sensor_subsystems(self, list_GMS_objs):
        # type: (list) -> GMS_object
        """Merge separate GMS objects belonging to the same scene-ID into ONE GMS object.

        :param list_GMS_objs:   <list> of GMS objects covering the same geographic area but representing different
                                sensor subsystems (e.g. 3 GMS_objects for Sentinel-2 10m/20m/60m bands)
        """

427
        # assertions
428
429
430
431
432
433
434
435
436
437
438
        assert len(list_GMS_objs)>1, "'GMS_object.from_sensor_subsystems()' expects multiple input GMS objects. " \
                                     "Got %d." %len(list_GMS_objs)
        assert all([is_coord_grid_equal(list_GMS_objs[0].arr.gt, *obj.arr.xygrid_specs) for obj in list_GMS_objs[1:]]),\
            "The input GMS objects must have the same pixel grid. Received: %s"\
            %np.array([obj.arr.xygrid_specs for obj in list_GMS_objs])
        assert len(list(set([GMS_obj.proc_level for GMS_obj in list_GMS_objs])))==1, \
            "The input GMS objects for GMS_object.from_sensor_subsystems() must have the same processing level."
        subsystems = [GMS_obj.subsystem for GMS_obj in list_GMS_objs]
        assert len(subsystems) == len(list(set(subsystems))), \
            "The input 'list_GMS_objs' contains duplicates: %s" %subsystems

439
        # log
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
        list_GMS_objs[0].logger.info('Merging the subsystems %s to a single GMS object...'
                                     %', '.join([GMS_obj.subsystem for GMS_obj in list_GMS_objs]))


        # find the common extent. NOTE: boundsMap is expected in the order [xmin,xmax,ymin,ymax]
        geoExtents    = np.array([GMS_obj.arr.box.boundsMap for GMS_obj in list_GMS_objs])
        common_extent = (min(geoExtents[:,0]), max(geoExtents[:,1]), min(geoExtents[:,2]), max(geoExtents[:,3]))


        # MERGE METADATA
        # copy all attributes from the first input GMS file (private attributes are not touched)
        for key, value in list_GMS_objs[0].__dict__.copy().items():
            if key in ['GMS_identifier', 'georef', 'dict_LayerOptTherm']:
                continue # properties that should better be created on the fly
            try:
                setattr(self, key, value)
            except:
                raise AttributeError("Can't set attribute %s." %key)


         # update LayerBandsAssignment and get full list of output bandnames
        from .METADATA import get_LayerBandsAssignment
        gms_idf = list_GMS_objs[0].GMS_identifier # use identifier of first input GMS object for getting LBA (respects current proc_level)
        self.LayerBandsAssignment = get_LayerBandsAssignment(gms_idf, return_fullLBA=True)
        bandnames                 = ['B%s'%i if len(i)==2 else 'B0%s'%i for i in self.LayerBandsAssignment]



        # update layer-dependent metadata with respect to remaining input GMS objects
        self.meta_odict.update({
            'band names': [('Band %s' %i) for i in self.LayerBandsAssignment],
            'LayerBandsAssignment' : self.LayerBandsAssignment,
            'Subsystem':'',
            'PhysUnit':self.meta_odict['PhysUnit'],# TODO can contain val_optical / val_thermal
        })
        self.subsystem = ''
        del self.pathGen # must be refreshed because subsystem is now ''
477
        self.close_GMS_loggers() # must also be refreshed because it depends on pathGen
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499

        for attrN in ['SolIrradiance','CWL','FWHM','Offsets','OffsetsRef','Gains','GainsRef',
                         'ThermalConstK1','ThermalConstK2', 'ViewingAngle_arrProv', 'IncidenceAngle_arrProv']:

            # combine values from separate subsystems to a single value
            attrDic_fullLBA = {}
            for GMS_obj in list_GMS_objs:
                attr_val = getattr(GMS_obj.MetaObj, attrN)
                if isinstance(attr_val, list):
                    attrDic_fullLBA.update(dict(zip(GMS_obj.LayerBandsAssignment, attr_val)))
                elif isinstance(attr_val,(dict,collections.OrderedDict)):
                    attrDic_fullLBA.update(attr_val)
                else:
                    raise ValueError(attrN)

            # update the attribute in self.MetaObj
            if attrDic_fullLBA:
                val2set = [attrDic_fullLBA[bN] for bN in self.LayerBandsAssignment] \
                    if isinstance(getattr(list_GMS_objs[0].MetaObj, attrN),list) else attrDic_fullLBA
                setattr(self.MetaObj, attrN, val2set)

        # merge logfiles (read all logs into DataFrame, sort it by the first column and write to new logfile
500
        [GMS_obj.close_GMS_loggers() for GMS_obj in list_GMS_objs] # close the loggers of the input objects
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
        paths_inLogs = [GMS_obj.pathGen.get_path_logfile() for GMS_obj in list_GMS_objs]
        allLogs_df   = DataFrame()
        for log in paths_inLogs:
            df = read_csv(log, sep='\n', delimiter=':   ', header=None, engine='python') # engine suppresses a pandas warning
            allLogs_df = allLogs_df.append(df) # FIXME this will log e.g. atm. corr 3 times for S2A -> use captured streams instead?

        allLogs_df = allLogs_df.sort_values(0)
        np.savetxt(self.pathGen.get_path_logfile(), np.array(allLogs_df), delimiter=':   ', fmt="%s")


        # MERGE ARRAY DATA
        # overwrite array data with merged arrays, clipped to common_extent and reordered according to FullLayerBandsAss.
        for attrname in ['arr', 'ac_errors', 'dem', 'mask_nodata', 'mask_clouds', 'mask_clouds_confidence', 'masks']:

            # get current attribute of each subsystem without running property getters
            all_arrays = [getattr(GMS_obj, '_%s' %attrname) for GMS_obj in list_GMS_objs]

            # get the same geographical extent for each input GMS object
            if len(set(tuple(ext) for ext in geoExtents.tolist())) > 1:
                # in case of different extents
                geoArrs_same_extent = []

                for geoArr in all_arrays:

                    if geoArr is not None:
                        geoArr_same_extent = \
                            GeoArray(*geoArr.get_mapPos(
                                mapBounds       = np.array(common_extent)[[0,2,1,3]], # pass (xmin, ymin, xmax, ymax)
                                mapBounds_prj   = geoArr.prj),
                                    bandnames = list(geoArr.bandnames.keys()),
                                    nodata    = geoArr.nodata) # FIXME mask_clouds_confidence is until here no GeoArray -> has no nodata value -> calculation throughs warning
                        geoArrs_same_extent.append(geoArr_same_extent)
                    else:
                        # e.g. in case of cloud mask that is only appended to the GMS object with the same spatial resolution)
                        geoArrs_same_extent.append(None)

            else:
                # skip get_mapPos() if all input GMS objects have the same extent
                geoArrs_same_extent = all_arrays


            # validate output GeoArrays
            if len([gA for gA in geoArrs_same_extent if gA is not None])>1:
                equal_bounds = all([geoArrs_same_extent[0].box.boundsMap==gA.box.boundsMap
                                    for gA in geoArrs_same_extent[1:]])
                equal_epsg   = all([geoArrs_same_extent[0].epsg==gA.epsg for gA in geoArrs_same_extent[1:]])
                equal_xydims = all([geoArrs_same_extent[0].shape[:2]==gA.shape[:2] for gA in geoArrs_same_extent[1:]])
                if not all([equal_bounds, equal_epsg, equal_xydims]):
                    raise RuntimeError('Something went wrong during getting the same geographical extent for all the '
                                       'input GMS objects. The extents, projections or pixel dimensions of the '
                                       'calculated input GMS objects are not equal.')


            # set output arrays
            if attrname in ['arr', 'ac_errors'] and list(set(geoArrs_same_extent)) != [None]:
                # the bands of these arrays have to be reordered according to FullLayerBandsAssignment

                # check that each desired band name for the current attribute is provided by one of the input GMS objects
                available_bandNs = list(chain.from_iterable([list(gA.bandnames) for gA in geoArrs_same_extent]))
                for bN in bandnames:
                    if not bN in available_bandNs:
                        raise ValueError("The given input GMS objects (subsystems) do not provide a bandname '%s' for "
                                         "the attribute '%s'. Available band names amongst all input GMS objects are: %s"
                                         %(bN, attrname, str(available_bandNs)))

                # merge arrays
                get_band = lambda bandN: [gA[bandN] for gA in geoArrs_same_extent if gA and bandN in gA.bandnames][0]
                full_geoArr = GeoArray(np.dstack((get_band(bandN) for bandN in bandnames)),
                                       geoArrs_same_extent[0].gt, geoArrs_same_extent[0].prj,
                                       bandnames = bandnames,
                                       nodata    = geoArrs_same_extent[0].nodata)
                setattr(self, attrname, full_geoArr)

            else:
                # masks, dem, mask_nodata, mask_clouds, mask_clouds_confidence
                if attrname == 'dem':
                    # use the DEM of the first input object (if the grid is the same, the DEMs should be the same anyway)
                    self.dem = geoArrs_same_extent[0]
                elif attrname == 'mask_nodata':
                    # must not be merged -> self.arr is already merged, so just recalculate it (np.all)
                    self.mask_nodata = self.calc_mask_nodata(overwrite=True)
                elif attrname == 'mask_clouds':
                    # possibly only present in ONE subsystem (set by atm. Corr.)
                    mask_clouds = [msk for msk in geoArrs_same_extent if msk is not None]
                    if len(mask_clouds)>1:
                        raise ValueError('Expected mask clouds in only one subsystem. Got %s.' %len(mask_clouds))
                    self.mask_clouds = mask_clouds[0] if mask_clouds else None
                elif attrname == 'mask_clouds_confidence':
                    # possibly only present in ONE subsystem (set by atm. Corr.)
                    mask_clouds_conf = [msk for msk in geoArrs_same_extent if msk is not None]
                    if len(mask_clouds_conf)>1:
                        raise ValueError('Expected mask_clouds_conf in only one subsystem. Got %s.' %len(mask_clouds_conf))
                    self.mask_clouds_confidence = mask_clouds_conf[0] if mask_clouds_conf else None
                elif attrname =='masks':
                    # self.mask_nodata and self.mask_clouds will already be set here -> so just recreate it from there
                    self.masks = None

        # recreate self.masks
        self.build_combined_masks_array()

        # update array-dependent metadata
        self.meta_odict.update({
            'samples':self.arr.cols, 'lines':self.arr.rows, 'bands':self.arr.bands,
            'map info':geotransform2mapinfo(self.arr.gt, self.arr.prj), 'coordinate system string':self.arr.prj,})
605
606

        # set shape of full array
607
        self.shape_fullArr = self.arr.shape
608
609
610
611

        return copy.copy(self)


612
613
614
615
616
617
    def from_tiles(self, list_GMS_tiles):
        # type: (list) -> self
        """Merge separate GMS objects with different spatial coverage but belonging to the same scene-ID to ONE GMS object.

        :param list_GMS_tiles: <list> of GMS objects that have been created by cut_GMS_obj_into_blocks()
        """
Daniel Scheffler's avatar
Daniel Scheffler committed
618

619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
        if 'IMapUnorderedIterator' in str(type(list_GMS_tiles)):
            list_GMS_tiles = list(list_GMS_tiles)


        # copy all attributes except of array attributes
        tile1 = list_GMS_tiles[0]
        [setattr(self, i, getattr(tile1, i)) for i in tile1.__dict__ \
            if not callable(getattr(tile1, i)) and not isinstance(getattr(tile1, i), (np.ndarray, GeoArray))]


        # MERGE ARRAY-ATTRIBUTES
        list_arraynames = [i for i in tile1.__dict__ if not callable(getattr(tile1, i)) and \
                           isinstance(getattr(tile1, i), (np.ndarray, GeoArray))]
        list_arraynames = ['_arr'] + [i for i in list_arraynames if i !='_arr'] # list must start with _arr, otherwise setters will not work

        for arrname in list_arraynames:
            samplearray = getattr(tile1, arrname)
            assert isinstance(samplearray, (np.ndarray, GeoArray)), \
                'Received a %s object for attribute %s. Expected a numpy array or an instance of GeoArray.'\
                % (type(samplearray), arrname)
            is_3d        = samplearray.ndim == 3
            bands        = (samplearray.shape[2],) if is_3d else ()  # dynamic -> works for arr, cld_arr,...
            target_shape = tuple(self.shape_fullArr[:2]) + bands
            target_dtype = samplearray.dtype
            merged_array = self._numba_array_merger(list_GMS_tiles, arrname, target_shape, target_dtype)

            setattr(self, arrname if not arrname.startswith('_') else arrname[1:], merged_array) # use setters if possible
            # NOTE: this asserts that each attribute starting with '_' has also a property with a setter!


        # UPDATE ARRAY-DEPENDENT ATTRIBUTES
        self.arr_shape = 'cube'
        self.arr_pos   = None

        # update MetaObj attributes
        self.meta_odict.update({
            'samples':self.arr.cols, 'lines':self.arr.rows, 'bands':self.arr.bands,
            'map info':geotransform2mapinfo(self.arr.gt, self.arr.prj), 'coordinate system string':self.arr.prj,})

        # calculate data_corners_imXY (mask_nodata is always an array here because get_mapPos always returns an array)
        corners_imYX = calc_FullDataset_corner_positions(
            self.mask_nodata, assert_four_corners=False, algorithm='shapely')
        self.trueDataCornerPos = [(YX[1], YX[0]) for YX in corners_imYX] # [UL, UR, LL, LR]

        # calculate trueDataCornerLonLat
        data_corners_LatLon = pixelToLatLon(self.trueDataCornerPos, geotransform=self.arr.gt, projection=self.arr.prj)
        self.trueDataCornerLonLat = [(YX[1], YX[0]) for YX in data_corners_LatLon]

        # calculate trueDataCornerUTM
        data_corners_utmYX = pixelToMapYX(self.trueDataCornerPos, geotransform = self.arr.gt, projection = self.arr.prj)  # FIXME asserts gt in UTM coordinates
        self.trueDataCornerUTM = [(YX[1], YX[0]) for YX in data_corners_utmYX]

        return copy.copy(self)


674
675
676
    @staticmethod
    @jit
    def _numba_array_merger(list_GMS_tiles, arrname2merge, target_shape, target_dtype):
Daniel Scheffler's avatar
Daniel Scheffler committed
677
        # type: (list, str, tuple, np.dtype) -> np.ndarray
678
679
680
681
682
683
684
685
686
        """
        private function, e.g. called by merge_GMS_tiles_to_GMS_obj() in order to fasten array merging

        :param list_GMS_tiles:
        :param arrname2merge:
        :param target_shape:
        :param target_dtype:
        :return:
        """
Daniel Scheffler's avatar
Daniel Scheffler committed
687

688
689
690
691
692
693
694
695
        out_arr = np.empty(target_shape, dtype=target_dtype)
        for idx, tile in enumerate(list_GMS_tiles):
            rowStart, rowEnd = tile.arr_pos[0]
            colStart, colEnd = tile.arr_pos[1]
            out_arr[rowStart:rowEnd + 1, colStart:colEnd + 1] = getattr(tile, arrname2merge)
        return out_arr


Daniel Scheffler's avatar
Daniel Scheffler committed
696
    def log_for_fullArr_or_firstTile(self, log_msg, subset=None):
697
698
699
700
701
702
703
        """Send a message to the logger only if full array or the first tile is currently processed.
        This function can be called when processing any tile but log message will only be sent from first tile.

        :param log_msg:  the log message to be logged
        :param subset:   subset argument as sent to e.g. DN2TOARadRefTemp that indicates which tile is to be processed.
                         Not needed if self.arr_pos is not None.
        """
Daniel Scheffler's avatar
Daniel Scheffler committed
704

705
706
707
708
        if subset is None and\
                (self.arr_shape=='cube' or self.arr_pos is None or [self.arr_pos[0][0],self.arr_pos[1][0]]==[0,0]) or \
            subset==['cube',None] or (subset and [subset[1][0][0], subset[1][1][0]]==[0,0]) or \
            hasattr(self,'logAtThisTile') and getattr(self,'logAtThisTile'): # cube or 1st tile
Daniel Scheffler's avatar
Daniel Scheffler committed
709
            self.logger.info(log_msg)
710
711
712
713
714
715
716
717
718
719
720
721
        else:
            pass


    def calc_mask_nodataOLD(self, subset):
        rasObj = GEOP.GEOPROCESSING(self.MetaObj.Dataname, self.logger, subset=subset)
        data = rasObj.calc_mask_data_nodataOLD(custom_nodataVal=-9999)
        return {'desc': 'mask_nodata', 'row_start': rasObj.rowStart, 'row_end': rasObj.rowEnd,
                'col_start': rasObj.colStart, 'col_end': rasObj.colEnd,'data': data}


    def apply_nodata_mask_to_ObjAttr(self, attrname, out_nodata_val=None):
722
        # type: (str,int) -> None
723
        """Applies self.mask_nodata to the specified array attribute by setting all values where mask_nodata is 0 to the
724
725
726
727
728
729
730
731
        given nodata value.

        :param attrname:         The attribute to apply the nodata mask to. Must be an array attribute or
                                 a string path to a previously saved ENVI-file.
        :param out_nodata_val:   set the values of the given attribute to this value.
        """

        assert hasattr(self,attrname)
732

733
        if getattr(self,attrname) is not None:
734

735
736
737
738
            if isinstance(getattr(self,attrname),str):
                update_spec_vals = True if attrname=='arr' else False
                self.apply_nodata_mask_to_saved_ENVIfile(getattr(self,attrname), out_nodata_val, update_spec_vals)
            else:
739
740
741
                assert isinstance(getattr(self,attrname),(np.ndarray, GeoArray)), \
                    'L1A_obj.%s must be a numpy array or an instance of GeoArray. Got type %s.' \
                    %(attrname,type(getattr(self,attrname)))
742
                assert hasattr(self,'mask_nodata') and self.mask_nodata is not None
743

744
                self.log_for_fullArr_or_firstTile('Applying nodata mask to L1A_object.%s...' %attrname)
745

746
                nodata_val = out_nodata_val if out_nodata_val else \
747
                    DEF_D.get_outFillZeroSaturated(getattr(self, attrname).dtype)[0]
748
749
750
751
                getattr(self,attrname)[self.mask_nodata.astype(np.int8) == 0] = nodata_val

                if attrname=='arr':
                    self.MetaObj.spec_vals['fill']=nodata_val
752
753
754
755
756
757
758


    def build_combined_masks_array(self):
        # type: () -> dict
        """Generates self.masks attribute (unsigned integer 8bit) from by concatenating all masks included in GMS obj.
        The corresponding metadata is assigned to L1A_obj.masks_meta. Empty mask attributes are skipped."""

759
        arrays2combine = [aN for aN in ['mask_nodata', 'mask_clouds'] \
760
                          if hasattr(self, aN) and isinstance(getattr(self, aN), (GeoArray, np.ndarray))]
761
762
763
764
765
766
767
768
769
        if arrays2combine:
            self.log_for_fullArr_or_firstTile('Combining masks...')
            get_data   = lambda arrName: getattr(self,arrName).astype(np.uint8)[:,:,None]

            for aN in arrays2combine:
                if False in np.equal(getattr(self,aN),getattr(self,aN).astype(np.uint8)):
                    warnings.warn('Some pixel values of attribute %s changed during data type '
                                  'conversion within build_combined_masks_array().')

770
            # set self.masks
771
772
            self.masks = get_data(arrays2combine[0]) if len(arrays2combine)==1 else \
                                    np.concatenate([get_data(aN) for aN in arrays2combine], axis=2)
773
            self.masks.bandnames = arrays2combine # set band names of GeoArray (allows later indexing by band name)
774

775
            # set self.masks_meta
776
            nodataVal = DEF_D.get_outFillZeroSaturated(self.masks.dtype)[0]
777
            self.masks_meta = {'map info': self.MetaObj.map_info, 'coordinate system string': self.MetaObj.projection,
778
779
780
781
782
783
784
                               'bands':len(arrays2combine),'band names':arrays2combine, 'data ignore value':nodataVal}

            return {'desc': 'masks', 'row_start': 0, 'row_end': self.shape_fullArr[0],
                    'col_start': 0, 'col_end': self.shape_fullArr[1], 'data': self.masks} # usually not needed


    def apply_nodata_mask_to_saved_ENVIfile(self, path_saved_ENVIhdr, custom_nodata_val=None, update_spec_vals=False):
785
        # type: (str,int,bool) -> None
786
787
        """Applies self.mask_nodata to a saved ENVI file with the same X/Y dimensions like self.mask_nodata by setting all
         values where mask_nodata is 0 to the given nodata value.
788
789
790
791
792
793
794
795

        :param path_saved_ENVIhdr:  <str> The path of the ENVI file to apply the nodata mask to.
        :param custom_nodata_val:   <int> set the values of the given attribute to this value.
        :param update_spec_vals:    <bool> whether to update self.MetaObj.spec_vals['fill']
        """

        self.log_for_fullArr_or_firstTile('Applying nodata mask to saved ENVI file...')
        assert os.path.isfile(path_saved_ENVIhdr)
796
        assert hasattr(self,'mask_nodata') and self.mask_nodata is not None
797
798
799
800
        if not path_saved_ENVIhdr.endswith('.hdr') and os.path.isfile(os.path.splitext(path_saved_ENVIhdr)[0]+'.hdr'):
            path_saved_ENVIhdr = os.path.splitext(path_saved_ENVIhdr)[0]+'.hdr'
        if custom_nodata_val is None:
            dtype_IDL  = int(INP_R.read_ENVIhdr_to_dict(path_saved_ENVIhdr)['data type'])
801
            nodata_val = DEF_D.get_outFillZeroSaturated(DEF_D.dtype_lib_IDL_Python[dtype_IDL])[0]
802
803
804
805
        else:
            nodata_val = custom_nodata_val
        FileObj     = spectral.open_image(path_saved_ENVIhdr)
        File_memmap = FileObj.open_memmap(writable=True)
806
        File_memmap[self.mask_nodata == 0] = nodata_val
807
808
809
810
        if update_spec_vals: self.MetaObj.spec_vals['fill'] = nodata_val


    def combine_tiles_to_ObjAttr(self, tiles, target_attr):
811
        # type: (list,str) -> None
812
813
814
815
816
817
818
        """Combines tiles, e.g. produced by L1A_P.L1A_object.DN2TOARadRefTemp() to a single attribute.
        If CFG.usecase.CFG.job.exec_mode == 'Python' the produced attribute is additionally written to disk.

        :param tiles:           <list> a list of dictionaries with the keys 'desc', 'data', 'row_start','row_end',
                                'col_start' and 'col_end'
        :param target_attr:     <str> the name of the attribute to be produced
        """
Daniel Scheffler's avatar
Daniel Scheffler committed
819

820
        warnings.warn("'combine_tiles_to_ObjAttr' is deprecated.", DeprecationWarning)
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
        assert tiles[0] and isinstance(tiles,list) and isinstance(tiles[0],dict),\
            "The 'tiles' argument has to be list of dictionaries with the keys 'desc', 'data', 'row_start'," \
            "'row_end', 'col_start' and 'col_end'."
        self.logger.info("Building L1A attribute '%s' by combining given tiles..." % target_attr)
        tiles        = [tiles] if not isinstance(tiles, list) else tiles
        sampleTile   = dict(tiles[0])
        target_shape = self.shape_fullArr if len(sampleTile['data'].shape) == 3 else self.shape_fullArr[:2]
        setattr(self, target_attr, np.empty(target_shape, dtype=sampleTile['data'].dtype))
        for tile in tiles:
            rS,rE,cS,cE = tile['row_start'],tile['row_end'],tile['col_start'],tile['col_end']
            if len(target_shape) == 3:  getattr(self,target_attr)[rS:rE+1,cS:cE+1,:] = tile['data']
            else:                       getattr(self,target_attr)[rS:rE+1,cS:cE+1]   = tile['data']
        if target_attr == 'arr':
            self.arr_desc    = sampleTile['desc']
            self.arr_shape   = 'cube' if len(self.arr.shape)==3 else 'band' if len(self.arr.shape)==2 else 'unknown'

            if CFG.job.exec_mode=='Python': # and not 'Flink'
                path_radref_file = os.path.join(self.ExtractedFolder, self.baseN+'__' + self.arr_desc)
                # path_radref_file = os.path.abspath('./testing/out/%s_TOA_Ref' % self.baseN)
                while not os.path.isdir(os.path.dirname(path_radref_file)):
                    try:
                        os.makedirs(os.path.dirname(path_radref_file))
                    except OSError as e:  # maybe not neccessary anymore in python 3
                        if not e.errno == 17:
                            raise
                GEOP.ndarray2gdal(self.arr, path_radref_file, importFile=self.MetaObj.Dataname, direction=3)
                self.MetaObj.Dataname = path_radref_file


    def write_tiles_to_ENVIfile(self, tiles, overwrite=True):
851
        # type: (list,bool) -> None
852
853
854
855
856
857
858
859
860
861
        """Writes tiles, e.g. produced by L1A_P.L1A_object.DN2TOARadRefTemp() to a single output ENVI file.

        :param tiles:           <list> a list of dictionaries with the keys 'desc', 'data', 'row_start','row_end',
                                'col_start' and 'col_end'
        :param overwrite:       whether to overwrite files that have been produced earlier
        """

        self.logger.info("Writing tiles '%s' temporarily to disk..." % tiles[0]['desc'])
        outpath = os.path.join(self.ExtractedFolder, '%s__%s.%s' %(self.baseN, tiles[0]['desc'], self.outInterleave))
        if CFG.usecase.conversion_type_optical in tiles[0]['desc'] or CFG.usecase.conversion_type_thermal in tiles[0]['desc']:
862
            self.meta_odict = self.MetaObj.to_odict() # important in order to keep geotransform/projection
863
864
865
866
            self.arr_desc = tiles[0]['desc']
            self.arr = outpath
            # self.arr = os.path.abspath('./testing/out/%s_TOA_Ref.bsq' % self.baseN)
            self.MetaObj.Dataname = self.arr
867
868
            self.arr_shape = \
                'cube' if len(tiles[0]['data'].shape)==3 else 'band' if len(tiles[0]['data'].shape)==2 else 'unknown'
869
870
871
872
873
874
875
        elif tiles[0]['desc'] == 'masks':
            self.masks = outpath
        elif tiles[0]['desc'] == 'lonlat_arr':
            # outpath = os.path.join(os.path.abspath('./testing/out/'),'%s__%s.%s' %(self.baseN, tiles[0]['desc'], self.outInterleave))
            self.lonlat_arr = outpath # FIXME
        outpath   = os.path.splitext(outpath)[0]+'.hdr' if not outpath.endswith('.hdr') else outpath
        out_shape = self.shape_fullArr[:2]+([tiles[0]['data'].shape[2]] if len(tiles[0]['data'].shape)==3 else [1])
876
877
        OUT_W.Tiles_Writer(tiles, outpath, out_shape, tiles[0]['data'].dtype, self.outInterleave, self.meta_odict,
                           overwrite=overwrite)
878
879
880


    def to_MGRS_tiles(self, pixbuffer=10, v=False):
881
        # type: (int) -> self
882
        """Returns a generator object where items represent the MGRS tiles for the GMS object.
883
884
885
886
887

        :param pixbuffer:   <int> a buffer in pixel values used to generate an overlap between the returned MGRS tiles
        :param v:           <bool> verbose mode
        :return:            <list> of MGRS_tile objects
        """
Daniel Scheffler's avatar
Daniel Scheffler committed
888

889
        assert self.arr_shape == 'cube', "Only 'cube' objects can be cut into MGRS tiles. Got %s." % self.arr_shape
890
        self.logger.info('Cutting scene %s (entity ID %s) into MGRS tiles...' % (self.scene_ID, self.entity_ID))
891
892

        # get GeoDataFrame containing all overlapping MGRS tiles (MGRS geometries completely within nodata area are excluded)
Daniel Scheffler's avatar
Daniel Scheffler committed
893
        GDF_MGRS_tiles = DB_T.get_overlapping_MGRS_tiles(CFG.job.conn_database,
894
                                                         tgt_corners_lonlat=self.trueDataCornerLonLat)
895
896

        # calculate image coordinate bounds of the full GMS object for each MGRS tile within the GeoDataFrame
897
        gt, prj       = mapinfo2geotransform(self.meta_odict['map info']), self.meta_odict['coordinate system string']
898
899
900
901
902
903
904
905
906
907
908

        get_arrBounds = lambda MGRStileObj: \
            list(np.array(MGRStileObj.poly_utm.buffer(pixbuffer*gt[1]).bounds)[[0, 2, 1, 3]])
        GDF_MGRS_tiles['MGRStileObj']     = [*GDF_MGRS_tiles['granuleid']  .map(lambda mgrsTileID: MGRS_tile(mgrsTileID))]
        GDF_MGRS_tiles['map_bounds_MGRS'] = [*GDF_MGRS_tiles['MGRStileObj'].map(get_arrBounds)]  # xmin,xmax,ymin,ymax

        # find first tile to log and assign 'logAtThisTile' later
        dictIDxminymin = {(b[0] + b[2]): ID for ID, b in
                          zip(GDF_MGRS_tiles['granuleid'], GDF_MGRS_tiles['map_bounds_MGRS'])}
        firstTile_ID = dictIDxminymin[min(dictIDxminymin.keys())]

909
        # ensure self.masks exists (does not exist in Flink mode because in that case self.fill_from_disk() is skipped)
910
911
912
913
        if not hasattr(self, 'masks') or self.masks is None:
            self.build_combined_masks_array()  # creates self.masks and self.masks_meta

        #read whole dataset into RAM in order to fasten subsetting
914
915
        self.arr  .to_mem()
        self.masks.to_mem() # to_mem ensures that the whole dataset is present in memory
916
917
918
919
920

        # produce data for each MGRS tile in loop
        for GDF_idx, GDF_row in GDF_MGRS_tiles.iterrows():
            tileObj = self.get_subset_obj(mapBounds     = GDF_row.map_bounds_MGRS,
                                          mapBounds_prj = GEOP.EPSG2WKT(GDF_row['MGRStileObj'].EPSG),
921
                                          out_prj       = GEOP.EPSG2WKT(GDF_row['MGRStileObj'].EPSG),
922
923
924
925
926
927
928
929
930
931
932
933
                                          logmsg        = 'Producing MGRS tile %s from scene %s (entity ID %s).'
                                                          %(GDF_row.granuleid, self.scene_ID, self.entity_ID),
                                          v             = v)
            MGRS_tileID       = GDF_row['granuleid']

            # set MGRS info
            tileObj.arr_shape = 'MGRS_tile'
            tileObj.MGRS_info = {'tile_ID': MGRS_tileID, 'grid1mil': MGRS_tileID[:3], 'grid100k': MGRS_tileID[3:]}

            # set logAtThisTile
            tileObj.logAtThisTile = MGRS_tileID == firstTile_ID

934
935
936
937
            # close logger of tileObj and of self in order to avoid logging permission errors
            tileObj.close_GMS_loggers()
            self.close_GMS_loggers()

938
939
940
            yield tileObj

        # set array attributes back to file path if they had been a filePath before
941
942
        if self.arr  .filePath: self.arr  .to_disk()
        if self.masks.filePath: self.masks.to_disk()
943
944
945
946
947


    def to_GMS_file(self, path_gms_file=None):
        self.close_GMS_loggers()

948
949
950
        # make sure meta_odict is present
        self.meta_odict = self.meta_odict

951
        dict2write    = self.attributes2dict(remove_privates=True)
952
        dict2write['arr_shape'], dict2write['arr_pos'] = ['cube', None]
953
        path_gms_file = path_gms_file if path_gms_file else self.pathGen.get_path_gmsfile()
954
955
956
957
958

        for k, v in list(dict2write.items()):
            # if isinstance(v,np.ndarray) or isinstance(v,dict) or hasattr(v,'__dict__'):
            ## so, wenn meta-dicts nicht ins gms-file sollen. ist aber vllt ni schlecht -> erlaubt lesen der metadaten direkt aus gms

959
960
961
            if k=='MetaObj':
                continue # make sure MetaObj getter is not called -> would delete meta_odict
            elif isinstance(v, datetime.datetime):
962
963
                dict2write[k] = v.strftime('%Y-%m-%d %H:%M:%S.%f%z') # FIXME
            elif isinstance(v, DatasetLogger):
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
                if hasattr(v, 'handlers') and v.handlers[:]:
                    warnings.warn('Not properly closed logger at GMS_obj.logger pointing to %s.' % v.path_logfile)
                dict2write[k] = 'not set'
            elif isinstance(v, collections.OrderedDict) or isinstance(v, dict):
                dict2write[k] = dict2write[k].copy()
                if 'logger' in v:
                    if hasattr(dict2write[k]['logger'], 'handlers') and dict2write[k]['logger'].handlers[:]:
                        warnings.warn("Not properly closed logger at %s['logger'] pointing to %s."
                                      % (k, dict2write[k]['logger'].path_logfile))
                    dict2write[k]['logger'] = 'not set'
            elif isinstance(v, np.ndarray):
                # delete every 3D Array larger than 100x100
                if len(v.shape) == 2 and sum(v.shape) <= 200:
                    dict2write[k] = v.tolist()  # numpy arrays are not jsonable
                else:
                    del dict2write[k]
            elif hasattr(v, '__dict__'):
                # löscht Instanzen von Objekten und Arrays, aber keine OrderedDicts
                if hasattr(v, 'logger'):
                    if hasattr(dict2write[k].logger, 'handlers') and dict2write[k].logger.handlers[:]:
                        warnings.warn("Not properly closed logger at %s.logger pointing to %s."
                                      % (k, dict2write[k].logger.path_logfile))
                    dict2write[k].logger = 'not set'
                del dict2write[k]

                # class customJSONEncoder(json.JSONEncoder):
                #     def default(self, obj):
                #         if isinstance(obj, np.ndarray):
                #             return '> numpy array <'
                #         if isinstance(obj, dict): # funktioniert nicht
                #             return '> python dictionary <'
                #         if hasattr(obj,'__dict__'):
                #             return '> python object <'
                #         # Let the base class default method raise the TypeError
                #         return json.JSONEncoder.default(self, obj)
                # json.dump(In, open(path_out_baseN,'w'),skipkeys=True,sort_keys=True,cls=customJSONEncoder,separators=(',', ': '),indent =4)
        with open(path_gms_file, 'w') as outF:
            json.dump(dict2write, outF, skipkeys=True, sort_keys=True, separators=(',', ': '), indent=4)


1004
    def to_ENVI(self, write_masks_as_ENVI_classification=True, is_tempfile=False, compression=False):
1005
        # type: (object, bool, bool) -> None
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
        """Write GMS object to disk. Supports full cubes AND 'block' tiles.

        :param self:                               <object> GMS object, e.g. L1A_P.L1A_object
        :param write_masks_as_ENVI_classification:  <bool> whether to write masks as ENVI classification file
        :param is_tempfile:                         <bool> whether output represents a temporary file
                                                    -> suppresses logging and database updating
                                                    - ATTENTION! This keyword asserts that the actual output file that
                                                     is written later contains the final version of the array. The array
                                                     is not overwritten or written once more later, but only renamed.
        :param compression:                         <bool> enable or disable compression
        """
        envi._write_image = OUT_W.silent_envi_write_image  # monkey patch writer function in order to silence output stream

        assert self.arr_shape in ['cube', 'MGRS_tile', 'block'], \
            "GMS_object.to_ENVI supports only array shapes 'cube', 'MGRS_tile' and 'block'. Got %s." % self.arr_shape

        # set MGRS info in case of MGRS tile
        MGRS_info = None
        if self.arr_shape == 'MGRS_tile':
            assert hasattr(self, 'MGRS_info'), \
                "Tried to write an GMS object in the shape 'MGRS_tile' without without the attribute 'MGRS_info'."
            MGRS_info = self.MGRS_info

        # set self.arr from L1B path to L1A path in order to make to_ENVI copy L1A array (if .arr is not an array)
        if self.proc_level == 'L1B' and not self.arr.is_inmem and os.path.isfile(self.arr.filePath):
1031
            self.arr = PG.path_generator('L1A', self.image_type, self.satellite, self.sensor, self.subsystem,
1032
                                         self.acq_datetime, self.entity_ID, self.logger,
1033
                                         MGRS_info=MGRS_info).get_path_imagedata()  # FIXME this could leed to check the wrong folder in Python exec_mode
1034
1035
1036

        # set dicts
        image_type_dict = {'arr': self.image_type, 'masks': 'MAS', 'mask_clouds': 'MAC'}
1037
1038
        metaAttr_dict   = {'arr': 'meta_odict', 'masks': 'masks_meta', 'mask_clouds': 'masks_meta'}
        out_dtype_dict  = {'arr': 'int16', 'masks': 'uint8', 'mask_clouds': 'uint8'}
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
        print_dict = {'RSD_L1A': 'L1A satellite data', 'MAS_L1A': 'L1A masks', 'MAC_L1A': 'L1A cloud mask',
                      'RSD_L1B': 'L1B satellite data', 'MAS_L1B': 'L1B masks', 'MAC_L1B': 'L1B cloud mask',
                      'RSD_L1C': 'L1C atm. corrected reflectance data', 'MAS_L1C': 'L1C masks',
                      'MAC_L1C': 'L1C cloud mask',
                      'RSD_L2A': 'L2A geometrically homogenized data', 'MAS_L2A': 'L2A masks',
                      'MAC_L2A': 'L2A cloud mask',
                      'RSD_L2B': 'L2B spectrally homogenized data', 'MAS_L2B': 'L2B masks', 'MAC_L2B': 'L2B cloud mask',
                      'RSD_L2C': 'L2C MGRS tiled data', 'MAS_L2C': 'L2C masks', 'MAC_L2C': 'L2C cloud mask'}

        # ensure self.masks exists
        if not hasattr(self, 'masks') or self.masks is None:
            self.build_combined_masks_array()  # creates InObj.masks and InObj.masks_meta

        # loop through all attributes to write and execute writer
        attributes2write = ['arr', 'masks'] + (['mask_clouds'] if write_masks_as_ENVI_classification else [])
1054
1055
        attributes2write = attributes2write if self.proc_level not in ['L1A', 'L1B'] else ['arr', 'masks']

1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
        for arrayname in attributes2write:
            descriptor = '%s_%s' % (image_type_dict[arrayname], self.proc_level)

            if hasattr(self, arrayname) and getattr(self, arrayname) is not None:
                arrayval = getattr(self, arrayname)  # can be a GeoArray (in mem / not in mem) or a numpy.ndarray

                # initial assertions
                assert arrayname in metaAttr_dict, "GMS_object.to_ENVI cannot yet write %s attribute." % arrayname
                assert isinstance(arrayval, (GeoArray, np.ndarray)), "Expected a GeoArray instance or a numpy array " \
                    "for object attribute %s. Got %s." % (arrayname, type(arrayval))

                outpath_hdr = self.pathGen.get_outPath_hdr(arrayname)
                outpath_hdr = os.path.splitext(outpath_hdr)[0] + '__TEMPFILE.hdr' if is_tempfile else outpath_hdr
                if not os.path.exists(os.path.dirname(outpath_hdr)): os.makedirs(os.path.dirname(outpath_hdr))
                out_dtype = out_dtype_dict[arrayname]
                meta_attr = metaAttr_dict[arrayname]

                if not is_tempfile:  self.log_for_fullArr_or_firstTile('Writing %s.' % print_dict[descriptor])

                if isinstance(arrayval, GeoArray) and not arrayval.is_inmem:
                    '''object attribute contains GeoArray in disk mode. This is usually the case if the attribute has
                    been read in Python exec mode from previous processing level and has NOT been modified during processing.'''

                    assert os.path.isfile(arrayval.filePath), "The object attribute '%s' contains a not existing " \
                                                              "file path: %s" % (arrayname, arrayval.filePath)
                    path_to_array = arrayval.filePath

                    if self.arr_shape == 'cube':
                        # image data can just be copied
                        outpath_arr  = os.path.splitext(outpath_hdr)[0] + (os.path.splitext(path_to_array)[1]
                                        if os.path.splitext(path_to_array)[1] else '.%s' % self.outInterleave)
                        hdr2readMeta = os.path.splitext(path_to_array)[0] + '.hdr'
                        meta2write   = INP_R.read_ENVIhdr_to_dict(hdr2readMeta, self.logger) \
                                        if arrayname in ['mask_clouds', ] else getattr(self, meta_attr)
                        meta2write.update({'interleave'    : self.outInterleave,
                                           'byte order'    : 0,
                                           'header offset' : 0,
1093
                                           'data type'     : DEF_D.dtype_lib_Python_IDL[out_dtype],
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
                                           'lines'         : self.shape_fullArr[0],
                                           'samples'       : self.shape_fullArr[1]})
                        meta2write = metaDict_to_metaODict(meta2write, self.logger)

                        if '__TEMPFILE' in path_to_array:
                            os.rename(path_to_array, outpath_arr)
                            envi.write_envi_header(outpath_hdr, meta2write)
                            HLP_F.silentremove(path_to_array)
                            HLP_F.silentremove(os.path.splitext(path_to_array)[0] + '.hdr')
                        else:
                            try:
                                shutil.copy(path_to_array, outpath_arr)  # copies file + permissions
                            except PermissionError:  # prevents permission error if outputfile already exists and is owned by another user
                                HLP_F.silentremove(outpath_arr)
                                shutil.copy(path_to_array, outpath_arr)
                            envi.write_envi_header(outpath_hdr, meta2write)

                        assert OUT_W.check_header_not_empty(outpath_hdr), "HEADER EMPTY: %s" % outpath_hdr
                        setattr(self, arrayname, outpath_arr)  # refresh arr/masks/mask_clouds attributes
                        if arrayname == 'masks':
                            setattr(self, 'mask_nodata', outpath_arr)

                    else:  # 'block' or 'MGRS_tile
                        # data have to be read in subset and then be written
                        if self.arr_pos:
                            (rS, rE), (cS, cE) = self.arr_pos
                            cols, rows         = cE - cS + 1, rE - rS + 1
                        else:
                            cS, rS, cols, rows = [None] * 4

                        if '__TEMPFILE' in path_to_array:
                            raise NotImplementedError

                        if arrayname not in ['mask_clouds', 'mask_nodata']:
                            # read image data in subset
                            tempArr   = gdalnumeric.LoadFile(path_to_array, cS, rS, cols, rows)  # bands, rows, columns OR rows, columns
                            arr2write = tempArr if len(tempArr.shape) == 2 else \
                                        np.swapaxes(np.swapaxes(tempArr, 0, 2), 0, 1)  # rows, columns, (bands)
                        else:
                            # read mask data in subset
1134
1135
                            previous_procL = DEF_D.proc_chain[DEF_D.proc_chain.index(self.proc_level) - 1]
                            PG_obj         = PG.path_generator(self.__dict__, proc_level=previous_procL)
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
                            path_masks     = PG_obj.get_path_maskdata()
                            arr2write      = INP_R.read_mask_subset(path_masks, arrayname, self.logger,
                                                                   (self.arr_shape, self.arr_pos))

                        setattr(self, arrayname, arr2write)


                arrayval = getattr(self, arrayname)  # can be a GeoArray (in mem / not in mem) or a numpy.ndarray

                if isinstance(arrayval, np.ndarray) or isinstance(arrayval, GeoArray) and arrayval.is_inmem:  # must be an if-condition because arrayval can change attribute type from not-inmem-GeoArray to np.ndarray
                    '''object attribute contains array'''

                    # convert array and metadata of mask clouds to envi classification file ready data
                    arr2write, meta2write = OUT_W.mask_to_ENVI_Classification(self, arrayname) \
                                            if arrayname in ['mask_clouds', ] else (arrayval, getattr(self, meta_attr))
                    arr2write = arr2write.arr if isinstance(arr2write, GeoArray) else arr2write
                    assert isinstance(arr2write, np.ndarray), 'Expected a numpy ndarray. Got %s.' % type(arr2write)

                    if self.arr_shape in ['cube', 'MGRS_tile']:
                        # TODO write a function that implements the algorithm from Tiles_Writer for writing cubes -> no need for Spectral Python
                        # write cube-like attributes
                        meta2write = metaDict_to_metaODict(meta2write, self.logger)
                        success    = 1
                        if arrayname not in ['mask_clouds', ]:
                            if compression:
                                success = OUT_W.write_ENVI_compressed(outpath_hdr, arr2write, meta2write)
                                if not success: warnings.warn('Written compressed ENVI file is not GDAL readable! '
                                                              'Writing uncompressed file.')
                            if not compression or not success:
                                envi.save_image(outpath_hdr, arr2write, metadata=meta2write, dtype=out_dtype,
                                                interleave=self.outInterleave, ext=self.outInterleave, force=True)
                        else:
                            if compression:
                                success = OUT_W.write_ENVI_compressed(outpath_hdr, arr2write, meta2write)
                                if not success: warnings.warn('Written compressed ENVI file is not GDAL readable! '
                                                              'Writing uncompressed file.')
                            if not compression or not success:
                                class_names = meta2write['class names']
                                class_colors = meta2write['class lookup']
                                envi.save_classification(outpath_hdr, arr2write, metadata=meta2write, dtype=out_dtype,
                                                         interleave=self.outInterleave, ext=self.outInterleave,
                                                         force=True, class_names=class_names, class_colors=class_colors)
                        if os.path.exists(outpath_hdr):  OUT_W.reorder_ENVI_header(outpath_hdr, OUT_W.enviHdr_keyOrder)

                    else:
                        if compression:  # FIXME
                            warnings.warn(
                                'Compression is not yet supported for GMS object tiles. Writing uncompressed data.')
                        # write block-like attributes
                        bands     = arr2write.shape[2] if len(arr2write.shape) == 3 else 1
                        out_shape = tuple(self.shape_fullArr[:2]) + (bands,)

                        OUT_W.Tiles_Writer(arr2write, outpath_hdr, out_shape, out_dtype, self.outInterleave,
                                     out_meta=meta2write, arr_pos=self.arr_pos, overwrite=False)
                        assert OUT_W.check_header_not_empty(outpath_hdr), "HEADER EMPTY: %s" % outpath_hdr

                    outpath_arr = os.path.splitext(outpath_hdr)[0] + '.%s' % self.outInterleave
                    if CFG.job.exec_mode == 'Python':
                        setattr(self, arrayname, outpath_arr)  # replace array by output path
                        if arrayname == 'masks':
                            setattr(self, 'mask_nodata', outpath_arr)

                            # if compression:
                            #    raise NotImplementedError # FIXME implement working compression
                            #    HLP_F.ENVIfile_to_ENVIcompressed(outpath_hdr)

            else:
                if not is_tempfile:
1204
1205
                    self.logger.warning(
                        "%s can not be written, because there is no corresponding attribute." % print_dict[descriptor])
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237

        # write GMS-file and update database
        # IMPORTANT: DO NOT pass the complete object but only a copy of the dictionary in order to prevent ASCII_writer
        #            and data_DB_updater from modifying the attributes of the object!!
        if self.arr_shape in ['cube', 'MGRS_tile'] or [self.arr_pos[0][0], self.arr_pos[1][0]] == [0,0]:  # cube or 1st tile
            # write GMS file
            self.to_GMS_file()

            # create/update database
            if not is_tempfile:
                DB_T.data_DB_updater(self.attributes2dict())

                # get id of updated record (needed for cross-refs in later db entrys
                if hasattr(self, 'MGRS_info') and self.MGRS_info:
                    res = DB_T.get_info_from_postgreSQLdb(CFG.job.conn_database, 'mgrs_tiles_proc', ['id'],
                                                          {'sceneid'           : self.scene_ID,
                                                           'virtual_sensor_id' : CFG.usecase.virtual_sensor_id,
                                                           'mgrs_code'         : self.MGRS_info['tile_ID']})
                    assert len(res) == 1, 'Found multiple database records for the last updated record. sceneid: %s;' \
                                          'mgrs_code: %s; virtual_sensor_id: %s' \
                                          % (self.scene_ID, self.MGRS_info['tile_ID'], CFG.usecase.virtual_sensor_id)
                    self.mgrs_tiles_proc_ID = res[0][0]
                else:
                    res = DB_T.get_info_from_postgreSQLdb(CFG.job.conn_database, 'scenes_proc', ['id'],
                                                          {'sceneid': self.scene_ID})
                    assert len(res) == 1, \
                        'Found multiple database records for the last updated record. sceneid: %s' % self.scene_ID
                    self.scenes_proc_ID = res[0][0]

        if not is_tempfile:
            self.log_for_fullArr_or_firstTile('%s data successfully saved.' % self.proc_level)

1238
1239
1240
        # close logger
        self.close_GMS_loggers()

1241

1242
    def close_GMS_loggers(self):
1243
        super(GMS_object, self).close_loggers()
1244
1245
1246


    def delete_previous_proc_level_results(self):
1247
        """Deletes results of the previous processing level if the respective flag CFG.job.exec_L**P[2]) is set to True.
1248
1249
           The function is skipped if the results of the current processing level have not yet been written.
        """
Daniel Scheffler's avatar
Daniel Scheffler committed
1250

1251
        tgt_proc_level = DEF_D.proc_chain[DEF_D.proc_chain.index(self.proc_level) - 1]
1252

1253
        if getattr(CFG.job, 'exec_%sP' % tgt_proc_level)[2]:
1254
1255

            pathGenPrev = PG.path_generator(self.__dict__.copy(), proc_level=tgt_proc_level)
1256

1257
1258
1259
1260
            files2delete = [pathGenPrev.get_path_imagedata(),
                            pathGenPrev.get_path_gmsfile(),
                            pathGenPrev.get_path_maskdata(),
                            pathGenPrev.get_path_cloudmaskdata()]
1261
1262

            # ensure that the results of the current processing level have been written
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
            if self.proc_level != 'L2A':
                pathGenCurr = self.pathGen
            else:
                # after geometric homogenization and subsystems merging (L2A) a path generator without subsystem is needed
                dict4pathGen = self.__dict__.copy()
                dict4pathGen['subsystem'] = ''
                pathGenCurr = PG.path_generator(dict4pathGen)

            filesCurrProcL = [pathGenCurr.get_path_imagedata(),
                              pathGenCurr.get_path_gmsfile(),
                              pathGenCurr.get_path_maskdata(),
                              pathGenCurr.get_path_cloudmaskdata()]

            if self.proc_level=='L2A' and self.subsystem:
                # in case subsystems have been merged in L2A -> also delete logfiles of L1C
                files2delete.append(pathGenPrev.get_path_logfile())
                filesCurrProcL.append(pathGenCurr.get_path_logfile())
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291

            for fPath, fPathCP in zip(files2delete, filesCurrProcL):
                hdr = '%s.hdr' % os.path.splitext(fPath)[0]
                if os.path.exists(fPath) and os.path.exists(fPathCP):
                    HLP_F.silentremove(fPath)
                    if os.path.exists(hdr):
                        HLP_F.silentremove(hdr)


    def delete_tempFiles(self):
        """Delete all temporary files that have been written during GMS object processing.
        """
Daniel Scheffler's avatar
Daniel Scheffler committed
1292

Daniel Scheffler's avatar
Daniel Scheffler committed
1293
1294
1295
1296
        # TODO ensure that all GDAL dataset are closed before. Otherwise the system creates a .fuse_hiddenXXX...
        # TODO right in the moment when we delete the data that are opened in GDAL


1297
1298
1299
        self.logger.info('Deleting temporary data...')

        if sys.platform.startswith('linux'):
1300
            # delete temporary extraction folder
Daniel Scheffler's avatar
Daniel Scheffler committed
1301
1302
1303
            if os.path.isdir(self.ExtractedFolder):
                shutil.rmtree(self.ExtractedFolder)

1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
            if os.path.isdir(self.ExtractedFolder):
                self.logger.warning('Could not delete temporary extraction folder: %s' % self.ExtractedFolder)

            # delete empty folders: subsystem > sensor > Rootname
            pardir = None
            for i in range(2):  # FIXME range(3)?
                deldir = self.ExtractedFolder if i == 0 else pardir
                pardir = os.path.abspath(os.path.join(deldir, os.path.pardir))
                if not glob.glob('%s/*' % pardir) and os.path.isdir(pardir):
                    os.rmdir(pardir)
                else:
                    break

1317
            # delete all files containing __TEMPFILE
1318
            path_procdata = self.pathGen.get_path_procdata()
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
            tempfiles = glob.glob(os.path.join(path_procdata, '*__TEMPFILE*'))
            [HLP_F.silentremove(tf) for tf in tempfiles]
        else:
            raise NotImplementedError

        # delete tempfiles created by HLP_F.get_tempfile()
        files2delete = glob.glob(os.path.join(self.ExtractedFolder, 'GeoMultiSens_*'))
        files2delete += glob.glob(os.path.join('/dev/shm/', 'GeoMultiSens_*'))
        for f in files2delete:
            HLP_F.silentremove(f)

1330
        # delete previous proc_level results on demand (according to CFG.job.exec_L**P[2])
1331
1332
1333
        self.delete_previous_proc_level_results()

        self.logger.close()
1334
1335
1336
1337
1338
        self.logger = None



class failed_GMS_object(GMS_object):
1339
1340
1341
    def delete_tempFiles(self):
        pass

1342
    def __init__(self, GMS_object_or_OrdDict, failedMapper, exc_type, exc_val, exc_tb):
1343
        super(failed_GMS_object, self).__init__()
1344
1345
        needed_attr = ['proc_level', 'image_type', 'scene_ID', 'entity_ID', 'satellite', 'sensor', 'subsystem',
                       'arr_shape', 'arr_pos']
1346

1347
1348
        if isinstance(GMS_object_or_OrdDict, collections.OrderedDict):  # in case of unhandled exception within L1A_map
            OrdDic         = GMS_object_or_OrdDict
1349
1350
1351
1352
            [setattr(self, k, OrdDic[k]) for k in needed_attr[:-2]]
            self.arr_shape = 'cube' if 'arr_shape' not in OrdDic else OrdDic['arr_shape']
            self.arr_pos   = None   if 'arr_pos'   not in OrdDic else OrdDic['arr_pos']

1353
1354
1355
1356
1357
1358
        else:  # in case of any other GMS mapper
            [setattr(self, k, getattr(GMS_object_or_OrdDict, k)) for k in needed_attr]

        self.failedMapper = failedMapper
        self.ExceptionType = exc_type.__name__
        self.ExceptionValue = repr(exc_val)
1359
1360
        self.ExceptionTraceback = exc_tb

1361

1362
1363
1364
    @property
    def pandasRecord(self):
        columns = ['scene_ID', 'entity_ID', 'satellite', 'sensor', 'subsystem', 'image_type', 'proc_level',
1365
                   'arr_shape', 'arr_pos', 'failedMapper', 'ExceptionType', 'ExceptionValue', 'ExceptionTraceback']
1366
1367
1368
1369
1370
1371
1372
1373
        return DataFrame([getattr(self, k) for k in columns], columns=columns)