definition_dicts.py 9.55 KB
Newer Older
1
2
# -*- coding: utf-8 -*-

3
4
# gms_preprocessing, spatial and spectral homogenization of satellite remote sensing data
#
5
# Copyright (C) 2020  Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
6
7
8
9
10
11
#
# This software was developed within the context of the GeoMultiSens project funded
# by the German Federal Ministry of Education and Research
# (project grant code: 01 IS 14 010 A-C).
#
# This program is free software: you can redistribute it and/or modify it under
12
13
14
15
16
17
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version. Please note the following exception: `spechomo` depends on tqdm, which
# is distributed under the Mozilla Public Licence (MPL) v2.0 except for the files
# "tqdm/_tqdm.py", "setup.py", "README.rst", "MANIFEST.in" and ".gitignore".
# Details can be found here: https://github.com/tqdm/tqdm/blob/master/LICENCE.
18
19
20
21
22
23
24
25
26
27
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program.  If not, see <http://www.gnu.org/licenses/>.


28
29
30
31
import collections
import re

import numpy as np
32
from typing import TYPE_CHECKING  # noqa F401  # flake8 issue
33

34
from ..options.config import GMS_config as CFG
35

36
37
38
if TYPE_CHECKING:
    from ..model.gms_object import GMS_identifier  # noqa F401  # flake8 issue

39
40
41
42
43
44
45
46
47
48
__author__ = 'Daniel Scheffler'

dtype_lib_Python_IDL = {'bool_': 0, 'uint8': 1, 'int8': 1, 'int_': 1, 'int16': 2, 'uint16': 12, 'int32': 3,
                        'uint32': 13, 'int64': 14, 'uint64': 15, 'float32': 4, 'float64': 5, 'complex_': 6,
                        'complex64': 9}
dtype_lib_IDL_Python = {0: np.bool_, 1: np.uint8, 2: np.int16, 3: np.int32, 4: np.float32, 5: np.float64,
                        6: np.complex64, 9: np.complex128, 12: np.uint16, 13: np.uint32, 14: np.int64, 15: np.uint64}
dtype_lib_GDAL_Python = {"uint8": 1, "int8": 1, "uint16": 2, "int16": 3, "uint32": 4, "int32": 5, "float32": 6,
                         "float64": 7, "complex64": 10, "complex128": 11}
proc_chain = ['L1A', 'L1B', 'L1C', 'L2A', 'L2B', 'L2C']
49
db_jobs_statistics_def = {'pending': 1, 'started': 2, None: 2, 'L1A': 3, 'L1B': 4, 'L1C': 5, 'L2A': 6, 'L2B': 7,
50
                          'L2C': 8, 'FAILED': 9}  # NOTE: OrderedDicts passed to L1A_map have proc_level=None
51
bandslist_all_errors = ['ac_errors', 'mask_clouds_confidence', 'spat_homo_errors', 'spec_homo_errors']
52
53


54
55
def get_GMS_sensorcode(GMS_id):
    # type: (GMS_identifier) -> str
56

57
    Satellite, Sensor, Subsystem = (GMS_id.satellite, GMS_id.sensor, GMS_id.subsystem)
58
    Sensor = Sensor[:-1] if re.match(r'SPOT', Satellite, re.I) and Sensor[-1] not in ['1', '2'] else Sensor
59
    meta_sensorcode = Satellite + '_' + Sensor + ('_' + Subsystem if Subsystem not in ["", None] else "")
60
    sensorcode_dic = {
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
        'ALOS_AVNIR-2': 'AVNIR-2',
        'Landsat-4_TM': 'TM4',  # call from layerstacker
        'Landsat-4_TM_SAM': 'TM4',  # call from metadata object
        'Landsat-5_TM': 'TM5',
        'Landsat-5_TM_SAM': 'TM5',
        'Landsat-7_ETM+': 'TM7',
        'Landsat-7_ETM+_SAM': 'TM7',
        'Landsat-8_OLI': 'LDCM',
        'Landsat-8_OLI_TIRS': 'LDCM',
        'Landsat-8_LDCM': 'LDCM',
        'SPOT-1_HRV1': 'SPOT1a',  # MS
        'SPOT-1_HRV2': 'SPOT1b',
        'SPOT-2_HRV1': 'SPOT2a',
        'SPOT-2_HRV2': 'SPOT2b',
        'SPOT-3_HRV1': 'SPOT3a',
        'SPOT-3_HRV2': 'SPOT3b',
        'SPOT-4_HRVIR1': 'SPOT4a',
        'SPOT-4_HRVIR2': 'SPOT4b',
        'SPOT-5_HRG1': 'SPOT5a',  # PAN HRG2A
        'SPOT-5_HRG2': 'SPOT5b',  # MS HRG2J
        'RapidEye-1_MSI': 'RE1',
        'RapidEye-2_MSI': 'RE2',
        'RapidEye-3_MSI': 'RE3',
        'RapidEye-4_MSI': 'RE4',
        'RapidEye-5_MSI': 'RE5',
        'SRTM_SRTM2': 'SRTM2',
        'Terra_ASTER': 'AST_full',
        'Terra_ASTER_VNIR1': 'AST_V1',
        'Terra_ASTER_VNIR2': 'AST_V2',
        'Terra_ASTER_SWIR': 'AST_S',
        'Terra_ASTER_TIR': 'AST_T',
        'Sentinel-2A_MSI': 'S2A_full',
        'Sentinel-2B_MSI': 'S2B_full',
        'Sentinel-2A_MSI_S2A10': 'S2A10',
        'Sentinel-2A_MSI_S2A20': 'S2A20',
        'Sentinel-2A_MSI_S2A60': 'S2A60',
        'Sentinel-2B_MSI_S2B10': 'S2B10',
        'Sentinel-2B_MSI_S2B20': 'S2B20',
        'Sentinel-2B_MSI_S2B60': 'S2B60'
100
101
102
103
104
    }
    try:
        return sensorcode_dic[meta_sensorcode]
    except KeyError:
        raise KeyError('Sensor %s is not included in sensorcode dictionary and can not be converted into GMS '
105
                       'sensorcode.' % meta_sensorcode)
106
107


108
def get_mask_classdefinition(maskname, satellite):
109
    if maskname == 'mask_nodata':
110
        return {'No data': 0,
111
                'Data': 1}
112
    elif maskname == 'mask_clouds':
113
        legends = {
114
            'FMASK': {
115
                'No Data': 0,
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
                'Clear': 1,
                'Cloud': 2,
                'Shadow': 3,
                'Snow': 4,
                'Water': 5},
            # seems to be outdated:
            # {'Clear Land': 0, 'Clear Water': 1, 'Cloud Shadow': 2, 'Snow': 3, 'Cloud': 4, 'No data': 255}
            'Classical Bayesian': {
                'Clear': 10,
                'Thick Clouds': 20,
                'Thin Clouds': 30,
                'Snow': 40},  # Classical Bayesian py_tools_ah
            'SICOR': {
                'Clear': 10,
                'Water': 20,
131
132
                'Shadow': 30,
                'Cirrus': 40,
133
134
135
                'Cloud': 50,
                'Snow': 60}  # SICOR
        }
136

137
        return legends[CFG.cloud_masking_algorithm[satellite]]
138
    else:
139
        raise ValueError("'%s' is not a supported mask name." % maskname)
140
141


Daniel Scheffler's avatar
Bugfix    
Daniel Scheffler committed
142
143
def get_mask_colormap(maskname):
    if maskname == 'mask_clouds':
144
145
146
        # return collections.OrderedDict(zip(['No data','Clear','Thick Clouds','Thin Clouds','Snow','Unknown Class'],
        #                                     [[0,0,0] ,[0,255,0],[80,80,80], [175,175,175],[255,255,255],[255,0,0]]))
        return collections.OrderedDict((
147
148
149
150
151
152
153
154
155
156
            ('No data', [0, 0, 0]),
            ('Clear', [0, 255, 0]),
            ('Water', [0, 0, 255]),
            ('Shadow', [50, 50, 50]),
            ('Cirrus', [175, 175, 175]),
            ('Cloud', [80, 80, 80]),
            ('Snow', [255, 255, 255]),
            ('Unknown Class', [255, 0, 0]),))
    else:
        return None
Daniel Scheffler's avatar
Bugfix    
Daniel Scheffler committed
157
158


159
160
161
def get_outFillZeroSaturated(dtype):
    """Returns the values for 'fill-', 'zero-' and 'saturated' pixels of an image
    to be written with regard to the target data type.
162

163
    :param dtype: data type of the image to be written"""
164

165
    dtype = str(np.dtype(dtype))
166
167
168
169
170
    assert dtype in ['bool', 'int8', 'uint8', 'int16', 'uint16', 'float32'], \
        "get_outFillZeroSaturated: Unknown dType: '%s'." % dtype
    dict_outFill = {'bool': None, 'int8': -128, 'uint8': 0, 'int16': -9999, 'uint16': 9999, 'float32': -9999.}
    dict_outZero = {'bool': None, 'int8': 0, 'uint8': 1, 'int16': 0, 'uint16': 0, 'float32': 0.}
    dict_outSaturated = {'bool': None, 'int8': 127, 'uint8': 256, 'int16': 32767, 'uint16': 65535, 'float32': 65535.}
171
172
173
    return dict_outFill[dtype], dict_outZero[dtype], dict_outSaturated[dtype]


174
175
def is_dataset_provided_as_fullScene(GMS_id):
    # type: (GMS_identifier) -> bool
176
177
178
    """Returns True if the dataset belonging to the given GMS_identifier is provided as full scene and returns False if
     it is provided as multiple tiles.

179
    :param GMS_id:
180
181
182
    :return:
    """

183
    sensorcode = get_GMS_sensorcode(GMS_id)
184
    dict_fullScene_or_tiles = {
185
        'AVNIR-2': True,
186
        'AST_full': True,
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
        'AST_V1': True,
        'AST_V2': True,
        'AST_S': True,
        'AST_T': True,
        'TM4': True,
        'TM5': True,
        'TM7': True,
        'LDCM': True,
        'SPOT1a': True,
        'SPOT2a': True,
        'SPOT3a': True,
        'SPOT4a': True,
        'SPOT5a': True,
        'SPOT1b': True,
        'SPOT2b': True,
        'SPOT3b': True,
        'SPOT4b': True,
        'SPOT5b': True,
        'RE5': False,
206
        'S2A_full': False,
207
208
209
        'S2A10': False,
        'S2A20': False,
        'S2A60': False,
210
        'S2B_full': False,
211
212
213
        'S2B10': False,
        'S2B20': False,
        'S2B60': False, }
214
    return dict_fullScene_or_tiles[sensorcode]
215
216


217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
def datasetid_to_sat_sen(dsid):
    # type: (int) -> tuple
    conv_dict = {
        8: ('Terra', 'ASTER'),  # ASTER L1B
        104: ('Landsat-8', 'OLI_TIRS'),  # pre-collection-ID
        108: ('Landsat-5', 'TM'),  # pre-collection-ID
        112: ('Landsat-7', 'ETM+'),  # pre-collection-ID SLC-off
        113: ('Landsat-7', 'ETM+'),  # pre-collection-ID SLC-on
        189: ('Terra', 'ASTER'),  # ASTER L1T
        249: ('Sentinel-2A', 'MSI'),  # actually only Sentinel-2
        250: ('Landsat-8', 'OLI_TIRS'),
        251: ('Landsat-7', 'ETM+'),
        252: ('Landsat-5', 'TM'),  # also includes Landsat-4
        }
    try:
        return conv_dict[dsid]
    except KeyError:
        raise ValueError('No satellite / sensor tuple available for dataset ID %s.' % dsid)


def sat_sen_to_datasetid(satellite, sensor):
    # type: (str, str) -> int
    conv_dict = {
        ('Landsat-5', 'TM'): 252,
        ('Landsat-7', 'ETM+'): 251,
        ('Landsat-8', 'OLI_TIRS'): 250,
        ('Sentinel-2A', 'MSI'): 249,
        ('Sentinel-2B', 'MSI'): 249,
        ('Terra', 'ASTER'): 189  # ASTER L1T
    }
    try:
        return conv_dict[(satellite, sensor)]
    except KeyError:
        raise ValueError('No dataset ID available for %s %s.' % (satellite, sensor))