Commit be82838c authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Fixed issue #66 (Number of wavelengths does not match number bands in L2C header file).

-> required to revise metadata storage for layer dependent metadata (they are now stored as dicts instead of lists)
parent 3e0b05f0
...@@ -314,14 +314,9 @@ class L1A_object(GMS_object): ...@@ -314,14 +314,9 @@ class L1A_object(GMS_object):
proc_opt_therm = sorted(list(set(self.dict_LayerOptTherm.values()))) proc_opt_therm = sorted(list(set(self.dict_LayerOptTherm.values())))
assert proc_opt_therm in [['optical', 'thermal'], ['optical'], ['thermal']] assert proc_opt_therm in [['optical', 'thermal'], ['optical'], ['thermal']]
# get input parameters inFill, inZero, inSaturated = \
off, gai, irr, zen, esd, inFill, inZero, inSaturated, k1, k2 = \ self.MetaObj.spec_vals['fill'], self.MetaObj.spec_vals['zero'], self.MetaObj.spec_vals['saturated']
[self.MetaObj.Offsets, self.MetaObj.Gains, self.MetaObj.SolIrradiance, (rS, rE), (cS, cE) = self.arr_pos or ((0, self.shape_fullArr[0]), (0, self.shape_fullArr[1]))
90 - float(self.MetaObj.SunElevation),
self.MetaObj.EarthSunDist, self.MetaObj.spec_vals['fill'], self.MetaObj.spec_vals['zero'],
self.MetaObj.spec_vals['saturated'], self.MetaObj.ThermalConstK1, self.MetaObj.ThermalConstK2]
(rS, rE), (cS, cE) = self.arr_pos if self.arr_pos else ((0, self.shape_fullArr[0]), (0, self.shape_fullArr[1]))
# in_mem = hasattr(self, 'arr') and isinstance(self.arr, np.ndarray) # in_mem = hasattr(self, 'arr') and isinstance(self.arr, np.ndarray)
# if in_mem: # if in_mem:
# (rS, rE), (cS, cE) = # (rS, rE), (cS, cE) =
...@@ -336,7 +331,10 @@ class L1A_object(GMS_object): ...@@ -336,7 +331,10 @@ class L1A_object(GMS_object):
"DN2RadRef-Input data have %s bands although %s bands are specified in self.LayerBandsAssignment." \ "DN2RadRef-Input data have %s bands although %s bands are specified in self.LayerBandsAssignment." \
% (self.arr.bands, len(self.LayerBandsAssignment)) % (self.arr.bands, len(self.LayerBandsAssignment))
# perform conversion if needed ################################
# perform conversion if needed #
################################
data_optical, data_thermal, optical_bandsList, thermal_bandsList = None, None, [], [] data_optical, data_thermal, optical_bandsList, thermal_bandsList = None, None, [], []
for optical_thermal in ['optical', 'thermal']: for optical_thermal in ['optical', 'thermal']:
if optical_thermal not in self.dict_LayerOptTherm.values(): if optical_thermal not in self.dict_LayerOptTherm.values():
...@@ -359,7 +357,21 @@ class L1A_object(GMS_object): ...@@ -359,7 +357,21 @@ class L1A_object(GMS_object):
if arr_desc == 'DN': if arr_desc == 'DN':
self.log_for_fullArr_or_firstTile('Calculating %s...' % conv, subset) self.log_for_fullArr_or_firstTile('Calculating %s...' % conv, subset)
# get input parameters #
########################
off = [self.MetaObj.Offsets[b] for b in self.LayerBandsAssignment]
gai = [self.MetaObj.Gains[b] for b in self.LayerBandsAssignment]
irr = [self.MetaObj.SolIrradiance[b] for b in self.LayerBandsAssignment]
zen, esd = 90 - float(self.MetaObj.SunElevation), self.MetaObj.EarthSunDist
k1 = [self.MetaObj.ThermalConstK1[b] for b in self.LayerBandsAssignment]
k2 = [self.MetaObj.ThermalConstK2[b] for b in self.LayerBandsAssignment]
OFF, GAI, IRR, K1, K2 = [list(np.array(par)[bList]) for par in [off, gai, irr, k1, k2]] OFF, GAI, IRR, K1, K2 = [list(np.array(par)[bList]) for par in [off, gai, irr, k1, k2]]
# perform conversion #
######################
res = \ res = \
GEOP.DN2Rad(inArray, OFF, GAI, inFill, inZero, inSaturated) if conv == 'Rad' else \ GEOP.DN2Rad(inArray, OFF, GAI, inFill, inZero, inSaturated) if conv == 'Rad' else \
GEOP.DN2TOARef(inArray, OFF, GAI, IRR, zen, esd, inFill, inZero, GEOP.DN2TOARef(inArray, OFF, GAI, IRR, zen, esd, inFill, inZero,
...@@ -410,7 +422,10 @@ class L1A_object(GMS_object): ...@@ -410,7 +422,10 @@ class L1A_object(GMS_object):
else: else:
data_thermal, thermal_bandsList = res, bList data_thermal, thermal_bandsList = res, bList
# combine results from optical and thermal data #################################################
# combine results from optical and thermal data #
#################################################
if data_optical is not None and data_thermal is not None: if data_optical is not None and data_thermal is not None:
bands_opt, bands_therm = [1 if len(d.shape) == 2 else d.shape[2] for d in [data_optical, data_thermal]] bands_opt, bands_therm = [1 if len(d.shape) == 2 else d.shape[2] for d in [data_optical, data_thermal]]
r, c, b = data_optical.shape[0], data_optical.shape[1], bands_opt + bands_therm r, c, b = data_optical.shape[0], data_optical.shape[1], bands_opt + bands_therm
......
...@@ -568,9 +568,11 @@ class AtmCorr(object): ...@@ -568,9 +568,11 @@ class AtmCorr(object):
solar_irradiance = {} solar_irradiance = {}
for inObj in self.inObjs: for inObj in self.inObjs:
for bandN, bandIdx in inObj.arr.bandnames.items(): for bandN in inObj.arr.bandnames:
lba_key = bandN[2:] if bandN.startswith('B0') else bandN[1:]
if bandN not in solar_irradiance: if bandN not in solar_irradiance:
solar_irradiance[bandN] = inObj.meta_odict['SolIrradiance'][bandIdx] solar_irradiance[bandN] = inObj.MetaObj.SolIrradiance[lba_key]
return solar_irradiance return solar_irradiance
def _meta_get_viewing_zenith(self): def _meta_get_viewing_zenith(self):
...@@ -938,21 +940,28 @@ class AtmCorr(object): ...@@ -938,21 +940,28 @@ class AtmCorr(object):
ac_bandNs = [bandN for bandN in inObj.arr.bandnames if bandN in self.results.data_ac.keys()] ac_bandNs = [bandN for bandN in inObj.arr.bandnames if bandN in self.results.data_ac.keys()]
out_LBA = [bN.split('B0')[1] if bN.startswith('B0') else bN.split('B')[1] for bN in ac_bandNs] out_LBA = [bN.split('B0')[1] if bN.startswith('B0') else bN.split('B')[1] for bN in ac_bandNs]
# update metadata # update metadata #
###################
inObj.arr_desc = 'BOA_Ref' inObj.arr_desc = 'BOA_Ref'
inObj.MetaObj.bands = len(self.results.data_ac) inObj.MetaObj.bands = len(self.results.data_ac)
inObj.MetaObj.PhysUnit = 'BOA_Reflectance in [0-%d]' % CFG.scale_factor_BOARef inObj.MetaObj.PhysUnit = 'BOA_Reflectance in [0-%d]' % CFG.scale_factor_BOARef
inObj.MetaObj.LayerBandsAssignment = out_LBA inObj.MetaObj.LayerBandsAssignment = out_LBA
inObj.LayerBandsAssignment = out_LBA
inObj.MetaObj.filter_layerdependent_metadata() inObj.MetaObj.filter_layerdependent_metadata()
inObj.meta_odict = inObj.MetaObj.to_odict() # actually auto-updated by getter inObj.meta_odict = inObj.MetaObj.to_odict() # actually auto-updated by getter
# join SURFACE REFLECTANCE as 3D int16 array, scaled to scale factor from config ##################################################################################
# join SURFACE REFLECTANCE as 3D int16 array, scaled to scale factor from config #
##################################################################################
# FIXME AC output nodata values = 0 -> new nodata areas but mask not updated # FIXME AC output nodata values = 0 -> new nodata areas but mask not updated
oF_refl, oZ_refl, oS_refl = get_outFillZeroSaturated(inObj.arr.dtype) oF_refl, oZ_refl, oS_refl = get_outFillZeroSaturated(inObj.arr.dtype)
surf_refl = np.dstack((self.results.data_ac[bandN] for bandN in ac_bandNs)) surf_refl = np.dstack((self.results.data_ac[bandN] for bandN in ac_bandNs))
surf_refl *= CFG.scale_factor_BOARef # scale using scale factor (output is float16) surf_refl *= CFG.scale_factor_BOARef # scale using scale factor (output is float16)
# FIXME really set AC nodata values to GMS outZero? # FIXME really set AC nodata values to GMS outZero?
surf_refl[nodata] = oZ_refl # overwrite AC nodata values with GMS outZero surf_refl[nodata] = oZ_refl # overwrite AC nodata values with GMS outZero
# apply the original nodata mask (indicating background values) # apply the original nodata mask (indicating background values)
surf_refl[np.array(inObj.mask_nodata).astype(np.int8) == 0] = oF_refl surf_refl[np.array(inObj.mask_nodata).astype(np.int8) == 0] = oF_refl
...@@ -962,8 +971,7 @@ class AtmCorr(object): ...@@ -962,8 +971,7 @@ class AtmCorr(object):
surf_refl[ surf_refl[
surf_refl == self.results.bad_data_value] = oF_refl # FIXME meaningful to set AC nans to -9999? surf_refl == self.results.bad_data_value] = oF_refl # FIXME meaningful to set AC nans to -9999?
# overwrite LayerBandsAssignment and use inObj.arr setter to generate a GeoArray # use inObj.arr setter to generate a GeoArray
inObj.LayerBandsAssignment = out_LBA
inObj.arr = surf_refl.astype(inObj.arr.dtype) # -> int16 (also converts NaNs to 0 if needed inObj.arr = surf_refl.astype(inObj.arr.dtype) # -> int16 (also converts NaNs to 0 if needed
else: else:
......
...@@ -62,7 +62,6 @@ class L2B_object(L2A_object): ...@@ -62,7 +62,6 @@ class L2B_object(L2A_object):
src_dsID = sat_sen_to_datasetid(self.satellite, self.sensor) src_dsID = sat_sen_to_datasetid(self.satellite, self.sensor)
src_cwls = self.meta_odict['wavelength'] src_cwls = self.meta_odict['wavelength']
# FIXME exclude or include thermal bands; respect sorted CWLs in context of LayerBandsAssignment # FIXME exclude or include thermal bands; respect sorted CWLs in context of LayerBandsAssignment
tgt_cwls = CFG.target_CWL
tgt_sat, tgt_sen = datasetid_to_sat_sen(CFG.datasetid_spectral_ref) tgt_sat, tgt_sen = datasetid_to_sat_sen(CFG.datasetid_spectral_ref)
# NOTE: get target LBA at L2A, because spectral characteristics of target sensor do not change after AC # NOTE: get target LBA at L2A, because spectral characteristics of target sensor do not change after AC
tgt_LBA = get_LayerBandsAssignment( tgt_LBA = get_LayerBandsAssignment(
...@@ -78,7 +77,7 @@ class L2B_object(L2A_object): ...@@ -78,7 +77,7 @@ class L2B_object(L2A_object):
"the spectral refernce sensor.") "the spectral refernce sensor.")
return return
if src_cwls == tgt_cwls or (self.satellite == tgt_sat and self.sensor == tgt_sen): if src_cwls == CFG.target_CWL or (self.satellite == tgt_sat and self.sensor == tgt_sen):
# FIXME catch the case if LayerBandsAssignments are unequal with np.take # FIXME catch the case if LayerBandsAssignments are unequal with np.take
self.logger.info("Spectral homogenization has been skipped because the current spectral characteristics " self.logger.info("Spectral homogenization has been skipped because the current spectral characteristics "
"are already equal to the target sensor's.") "are already equal to the target sensor's.")
...@@ -93,7 +92,7 @@ class L2B_object(L2A_object): ...@@ -93,7 +92,7 @@ class L2B_object(L2A_object):
if method == 'LI' or CFG.datasetid_spectral_ref is None: if method == 'LI' or CFG.datasetid_spectral_ref is None:
# linear interpolation (if intended by user or in case of custom spectral characteristics of target sensor) # linear interpolation (if intended by user or in case of custom spectral characteristics of target sensor)
# -> no classifier for that case available -> linear interpolation # -> no classifier for that case available -> linear interpolation
im = SpH.interpolate_cube(self.arr, src_cwls, tgt_cwls, kind='linear') im = SpH.interpolate_cube(self.arr, src_cwls, CFG.target_CWL, kind='linear')
if CFG.spechomo_estimate_accuracy: if CFG.spechomo_estimate_accuracy:
self.logger.warning("Unable to compute any error information in case spectral homogenization algorithm " self.logger.warning("Unable to compute any error information in case spectral homogenization algorithm "
...@@ -115,7 +114,7 @@ class L2B_object(L2A_object): ...@@ -115,7 +114,7 @@ class L2B_object(L2A_object):
compute_errors=CFG.spechomo_estimate_accuracy, compute_errors=CFG.spechomo_estimate_accuracy,
bandwise_errors=CFG.spechomo_bandwise_accuracy, bandwise_errors=CFG.spechomo_bandwise_accuracy,
fallback_argskwargs=dict( fallback_argskwargs=dict(
args=dict(source_CWLs=src_cwls, target_CWLs=tgt_cwls,), args=dict(source_CWLs=src_cwls, target_CWLs=CFG.target_CWL,),
kwargs=dict(kind='linear') kwargs=dict(kind='linear')
)) ))
...@@ -124,8 +123,9 @@ class L2B_object(L2A_object): ...@@ -124,8 +123,9 @@ class L2B_object(L2A_object):
################### ###################
self.LayerBandsAssignment = tgt_LBA self.LayerBandsAssignment = tgt_LBA
self.meta_odict['wavelength'] = list(tgt_cwls) self.meta_odict['wavelength'] = list(CFG.target_CWL)
self.meta_odict['bands'] = len(tgt_cwls) self.meta_odict['bandwidths'] = list(CFG.target_FWHM)
self.meta_odict['bands'] = len(CFG.target_CWL)
if 'band names' in self.meta_odict: # FIXME bug workaround if 'band names' in self.meta_odict: # FIXME bug workaround
del self.meta_odict['band names'] # TODO del self.meta_odict['band names'] # TODO
...@@ -139,8 +139,8 @@ class L2B_object(L2A_object): ...@@ -139,8 +139,8 @@ class L2B_object(L2A_object):
if self.ac_errors and self.ac_errors.ndim == 3: if self.ac_errors and self.ac_errors.ndim == 3:
self.logger.info("Performing linear interpolation for 'AC errors' array to match target sensor bands " self.logger.info("Performing linear interpolation for 'AC errors' array to match target sensor bands "
"number..") "number..")
outarr = \ outarr = interp1d(np.array(src_cwls), self.ac_errors,
interp1d(np.array(src_cwls), self.ac_errors, axis=2, kind='linear', fill_value='extrapolate')(tgt_cwls) axis=2, kind='linear', fill_value='extrapolate')(CFG.target_CWL)
self.ac_errors = outarr.astype(np.int16) self.ac_errors = outarr.astype(np.int16)
......
...@@ -193,6 +193,7 @@ def SRF_reader(GMS_identifier, no_thermal=None, no_pan=None, v=False): ...@@ -193,6 +193,7 @@ def SRF_reader(GMS_identifier, no_thermal=None, no_pan=None, v=False):
LayerBandsAssignment = META.get_LayerBandsAssignment(GMS_identifier, no_thermal=no_thermal, no_pan=no_pan) LayerBandsAssignment = META.get_LayerBandsAssignment(GMS_identifier, no_thermal=no_thermal, no_pan=no_pan)
SRF_dict = collections.OrderedDict() SRF_dict = collections.OrderedDict()
SRF_dir = PG.get_path_srf_file(GMS_identifier) SRF_dir = PG.get_path_srf_file(GMS_identifier)
if os.path.isdir(SRF_dir): if os.path.isdir(SRF_dir):
for bandname in LayerBandsAssignment: for bandname in LayerBandsAssignment:
SRF_path = PG.get_path_srf_file(GMS_identifier, bandname=bandname) SRF_path = PG.get_path_srf_file(GMS_identifier, bandname=bandname)
......
...@@ -38,7 +38,7 @@ from sicor.options import get_options as get_ac_options ...@@ -38,7 +38,7 @@ from sicor.options import get_options as get_ac_options
from ..misc.logging import GMS_logger as DatasetLogger from ..misc.logging import GMS_logger as DatasetLogger
from ..model.mgrs_tile import MGRS_tile from ..model.mgrs_tile import MGRS_tile
from ..model.metadata import METADATA, get_dict_LayerOptTherm, metaDict_to_metaODict from ..model.metadata import METADATA, get_dict_LayerOptTherm, metaDict_to_metaODict, layerdependent_metadata
from ..model.dataset import Dataset from ..model.dataset import Dataset
from ..misc import path_generator as PG from ..misc import path_generator as PG
from ..misc import database_tools as DB_T from ..misc import database_tools as DB_T
...@@ -739,9 +739,7 @@ class GMS_object(Dataset): ...@@ -739,9 +739,7 @@ class GMS_object(Dataset):
del self.pathGen # must be refreshed because subsystem is now '' del self.pathGen # must be refreshed because subsystem is now ''
self.close_GMS_loggers() # must also be refreshed because it depends on pathGen self.close_GMS_loggers() # must also be refreshed because it depends on pathGen
for attrN in ['SolIrradiance', 'CWL', 'FWHM', 'Offsets', 'OffsetsRef', 'Gains', 'GainsRef', for attrN in layerdependent_metadata:
'ThermalConstK1', 'ThermalConstK2', 'ViewingAngle_arrProv', 'IncidenceAngle_arrProv']:
# combine values from separate subsystems to a single value # combine values from separate subsystems to a single value
attrDic_fullLBA = {} attrDic_fullLBA = {}
for GMS_obj in list_GMS_objs: for GMS_obj in list_GMS_objs:
......
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment