Commit 6929e4d1 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

PEP-8 editing.


Former-commit-id: a5f41a39
parent b2fef42c
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """Level 1C Processor: Atmospheric correction of TOA-reflectance data."""
Level 1C Processor:
Performed operations:
Atmospheric correction of TOA-reflectance data:
Written by Daniel Scheffler
"""
import warnings import warnings
import re import re
...@@ -16,6 +9,7 @@ import traceback ...@@ -16,6 +9,7 @@ import traceback
from typing import List from typing import List
import numpy as np import numpy as np
try: try:
from osgeo import osr from osgeo import osr
except ImportError: except ImportError:
...@@ -36,6 +30,8 @@ from sicor.sicor_ac import ac_gms ...@@ -36,6 +30,8 @@ from sicor.sicor_ac import ac_gms
from sicor.sensors import RSImage from sicor.sensors import RSImage
from sicor.Mask import S2Mask from sicor.Mask import S2Mask
__author__ = 'Daniel Scheffler'
class L1C_object(L1B_object): class L1C_object(L1B_object):
def __init__(self, L1B_obj=None): def __init__(self, L1B_obj=None):
...@@ -43,7 +39,7 @@ class L1C_object(L1B_object): ...@@ -43,7 +39,7 @@ class L1C_object(L1B_object):
if L1B_obj: if L1B_obj:
# populate attributes # populate attributes
[setattr(self, key, value) for key,value in L1B_obj.__dict__.items()] [setattr(self, key, value) for key, value in L1B_obj.__dict__.items()]
# private attributes # private attributes
self._VZA_arr = None self._VZA_arr = None
...@@ -96,7 +92,7 @@ class L1C_object(L1B_object): ...@@ -96,7 +92,7 @@ class L1C_object(L1B_object):
float(self.meta_odict['ViewingAngle']), float(self.meta_odict['ViewingAngle']),
float(self.meta_odict['FieldOfView']), float(self.meta_odict['FieldOfView']),
self.logger, self.logger,
nodata_mask=None, # dont overwrite areas outside the image with nodata nodata_mask=None, # dont overwrite areas outside image with nodata
outFill=get_outFillZeroSaturated(np.float32)[0], outFill=get_outFillZeroSaturated(np.float32)[0],
meshwidth=10) # for faster processing meshwidth=10) # for faster processing
return self._VZA_arr return self._VZA_arr
...@@ -198,7 +194,11 @@ class L1C_object(L1B_object): ...@@ -198,7 +194,11 @@ class L1C_object(L1B_object):
self.SAA_arr = None # not needed anymore self.SAA_arr = None # not needed anymore
self.RAA_arr = None # not needed anymore self.RAA_arr = None # not needed anymore
self.lonlat_arr = None # not needed anymore self.lonlat_arr = None # not needed anymore
del self.dem # uses deleter # would have to be resampled when writing MGRS tiles -> better to directly warp it to the output dims and projection
# use self.dem deleter
# would have to be resampled when writing MGRS tiles
# -> better to directly warp it to the output dims and projection
del self.dem
class AtmCorr(object): class AtmCorr(object):
...@@ -214,25 +214,25 @@ class AtmCorr(object): ...@@ -214,25 +214,25 @@ class AtmCorr(object):
L1C_objs = L1C_objs if isinstance(L1C_objs, tuple) else (L1C_objs,) L1C_objs = L1C_objs if isinstance(L1C_objs, tuple) else (L1C_objs,)
# hidden attributes # hidden attributes
self._logger = None self._logger = None
self._GSDs = [] self._GSDs = []
self._data = {} self._data = {}
self._metadata = {} self._metadata = {}
self._nodata = {} self._nodata = {}
self._band_spatial_sampling = {} self._band_spatial_sampling = {}
self._options = {} self._options = {}
# assertions # assertions
scene_IDs = [obj.scene_ID for obj in L1C_objs] scene_IDs = [obj.scene_ID for obj in L1C_objs]
assert len(list(set(scene_IDs))) == 1, \ assert len(list(set(scene_IDs))) == 1, \
"Input GMS objects for 'AtmCorr' must all belong to the same scene ID!. Received %s." %scene_IDs "Input GMS objects for 'AtmCorr' must all belong to the same scene ID!. Received %s." % scene_IDs
self.inObjs = L1C_objs # type: List[L1B_object] self.inObjs = L1C_objs # type: List[L1C_object]
self.reporting = reporting self.reporting = reporting
self.ac_input = {} # set by self.run_atmospheric_correction() self.ac_input = {} # set by self.run_atmospheric_correction()
self.results = None # direct output of external atmCorr module (set by run_atmospheric_correction) self.results = None # direct output of external atmCorr module (set by run_atmospheric_correction)
self.proc_info = {} self.proc_info = {}
self.outObjs = [] # atmospherically corrected L1C objects self.outObjs = [] # atmospherically corrected L1C objects
# append AtmCorr object to input L1C objects # append AtmCorr object to input L1C objects
# [setattr(L1C_obj, 'AtmCorr', self) for L1C_obj in self.inObjs] # too big for serialization # [setattr(L1C_obj, 'AtmCorr', self) for L1C_obj in self.inObjs] # too big for serialization
...@@ -257,7 +257,7 @@ class AtmCorr(object): ...@@ -257,7 +257,7 @@ class AtmCorr(object):
# add additional file handlers for the remaining inObj (that belong to the same scene_ID) # add additional file handlers for the remaining inObj (that belong to the same scene_ID)
for inObj in self.inObjs[1:]: for inObj in self.inObjs[1:]:
path_logfile = inObj.pathGen.get_path_logfile() path_logfile = inObj.pathGen.get_path_logfile()
fileHandler = logging.FileHandler(path_logfile, mode='a') fileHandler = logging.FileHandler(path_logfile, mode='a')
fileHandler.setFormatter(logger_atmCorr.formatter_fileH) fileHandler.setFormatter(logger_atmCorr.formatter_fileH)
fileHandler.setLevel(logging.DEBUG) fileHandler.setLevel(logging.DEBUG)
...@@ -294,7 +294,7 @@ class AtmCorr(object): ...@@ -294,7 +294,7 @@ class AtmCorr(object):
for obj in self.inObjs: for obj in self.inObjs:
if obj.arr.xgsd != obj.arr.ygsd: if obj.arr.xgsd != obj.arr.ygsd:
warnings.warn("X/Y GSD is not equal for entity ID %s" % obj.entity_ID + warnings.warn("X/Y GSD is not equal for entity ID %s" % obj.entity_ID +
(' (%s)'%obj.subsystem if obj.subsystem else '') + (' (%s)' % obj.subsystem if obj.subsystem else '') +
'Using X-GSD as key for spatial sampling dictionary.') 'Using X-GSD as key for spatial sampling dictionary.')
self._GSDs.append(obj.arr.xgsd) self._GSDs.append(obj.arr.xgsd)
...@@ -326,19 +326,20 @@ class AtmCorr(object): ...@@ -326,19 +326,20 @@ class AtmCorr(object):
for inObj in self.inObjs: for inObj in self.inObjs:
for bandN, bandIdx in inObj.arr.bandnames.items(): for bandN, bandIdx in inObj.arr.bandnames.items():
if bandN not in data_dict: if bandN not in data_dict:
arr2pass = inObj.arr[:, :, bandIdx].astype(np.float32) # conversion to np.float16 will convert -9999 to -10000 # float32! -> conversion to np.float16 will convert -9999 to -10000
arr2pass[arr2pass==inObj.arr.nodata] = np.nan # set nodata values to np.nan arr2pass = inObj.arr[:, :, bandIdx].astype(np.float32)
data_dict[bandN] = (arr2pass/inObj.meta_odict['ScaleFactor']).astype(np.float16) arr2pass[arr2pass == inObj.arr.nodata] = np.nan # set nodata values to np.nan
data_dict[bandN] = (arr2pass / inObj.meta_odict['ScaleFactor']).astype(np.float16)
else: else:
inObj.logger.warning("Band '%s' cannot be included into atmospheric correction because it " inObj.logger.warning("Band '%s' cannot be included into atmospheric correction because it "
"exists multiple times." % bandN) "exists multiple times." % bandN)
# validate: data must have all bands needed for AC # validate: data must have all bands needed for AC
full_LBA = get_LayerBandsAssignment(self.inObjs[0].GMS_identifier, return_fullLBA=True) full_LBA = get_LayerBandsAssignment(self.inObjs[0].GMS_identifier, return_fullLBA=True)
all_bNs_AC = ['B%s'% i if len(i) == 2 else 'B0%s' % i for i in full_LBA] all_bNs_AC = ['B%s' % i if len(i) == 2 else 'B0%s' % i for i in full_LBA]
if not all([bN in list(data_dict.keys()) for bN in all_bNs_AC]): if not all([bN in list(data_dict.keys()) for bN in all_bNs_AC]):
raise RuntimeError('Atmospheric correction did not receive all the needed bands. \n\tExpected: %s;\n\t' raise RuntimeError('Atmospheric correction did not receive all the needed bands. \n\tExpected: %s;\n\t'
'Received: %s' %(str(all_bNs_AC), str(list(sorted(data_dict.keys()))))) 'Received: %s' % (str(all_bNs_AC), str(list(sorted(data_dict.keys())))))
self._data = data_dict self._data = data_dict
...@@ -436,21 +437,23 @@ class AtmCorr(object): ...@@ -436,21 +437,23 @@ class AtmCorr(object):
______ key:SENSING_TIME, value_type:<class 'datetime.datetime'>, repr: 2016-03-26 10:34:06.538000+00:00 ______ key:SENSING_TIME, value_type:<class 'datetime.datetime'>, repr: 2016-03-26 10:34:06.538000+00:00
""" """
# TODO add SRF object # TODO add SRF object
metadata = {}
if not self._metadata: if not self._metadata:
del self.logger # otherwise each input object would have multiple fileHandlers del self.logger # otherwise each input object would have multiple fileHandlers
metadata['U'] = self.inObjs[0].meta_odict['EarthSunDist'] metadata = dict(
metadata['SENSING_TIME'] = self.inObjs[0].acq_datetime U=self.inObjs[0].meta_odict['EarthSunDist'],
# metadata['SENSING_TIME'] = datetime.strptime('2015-08-12 10:40:21 +0000', '%Y-%m-%d %H:%M:%S %z') SENSING_TIME=self.inObjs[0].acq_datetime,
metadata['viewing_zenith'] = self._meta_get_viewing_zenith() # SENSING_TIME=datetime.strptime('2015-08-12 10:40:21 +0000', '%Y-%m-%d %H:%M:%S %z'),
metadata['viewing_azimuth'] = self._meta_get_viewing_azimuth() viewing_zenith=self._meta_get_viewing_zenith(),
metadata['relative_viewing_azimuth'] = self._meta_get_relative_viewing_azimuth() viewing_azimuth=self._meta_get_viewing_azimuth(),
metadata['sun_mean_azimuth'] = self.inObjs[0].meta_odict['SunAzimuth'] relative_viewing_azimuth=self._meta_get_relative_viewing_azimuth(),
metadata['sun_mean_zenith'] = 90-self.inObjs[0].meta_odict['SunElevation'] sun_mean_azimuth=self.inObjs[0].meta_odict['SunAzimuth'],
metadata['solar_irradiance'] = self._meta_get_solar_irradiance() sun_mean_zenith=90 - self.inObjs[0].meta_odict['SunElevation'],
metadata['aux_data'] = self._meta_get_aux_data() solar_irradiance=self._meta_get_solar_irradiance(),
metadata['spatial_samplings'] = self._meta_get_spatial_samplings() aux_data=self._meta_get_aux_data(),
spatial_samplings=self._meta_get_spatial_samplings()
)
self._metadata = metadata self._metadata = metadata
...@@ -460,6 +463,7 @@ class AtmCorr(object): ...@@ -460,6 +463,7 @@ class AtmCorr(object):
def options(self): def options(self):
"""Returns a dictionary containing AC options. """Returns a dictionary containing AC options.
""" """
# type: -> dict
if self._options: if self._options:
return self._options return self._options
else: else:
...@@ -504,12 +508,12 @@ class AtmCorr(object): ...@@ -504,12 +508,12 @@ class AtmCorr(object):
# set spatial information # set spatial information
spatial_samplings[inObj.arr.xgsd] = dict( spatial_samplings[inObj.arr.xgsd] = dict(
ULX = inObj.arr.box.boxMapYX[0][1], ULX=inObj.arr.box.boxMapYX[0][1],
ULY = inObj.arr.box.boxMapYX[0][0], ULY=inObj.arr.box.boxMapYX[0][0],
XDIM = inObj.arr.xgsd, XDIM=inObj.arr.xgsd,
YDIM = -inObj.arr.ygsd, YDIM=-inObj.arr.ygsd,
NROWS = inObj.arr.rows, NROWS=inObj.arr.rows,
NCOLS = inObj.arr.cols) NCOLS=inObj.arr.cols)
return spatial_samplings return spatial_samplings
...@@ -712,7 +716,7 @@ class AtmCorr(object): ...@@ -712,7 +716,7 @@ class AtmCorr(object):
if inObj.arr.xgsd == cm_geoarray.xgsd: if inObj.arr.xgsd == cm_geoarray.xgsd:
inObj.mask_clouds = cm_geoarray inObj.mask_clouds = cm_geoarray
inObj.build_combined_masks_array() inObj.build_combined_masks_array()
break # appending it to one inObj is enough break # appending it to one inObj is enough
return S2Mask(mask_array=cm_array, return S2Mask(mask_array=cm_array,
mask_legend=cm_legend, mask_legend=cm_legend,
...@@ -731,14 +735,15 @@ class AtmCorr(object): ...@@ -731,14 +735,15 @@ class AtmCorr(object):
self.logger.info('Calculating input data for atmospheric correction...') self.logger.info('Calculating input data for atmospheric correction...')
rs_data = dict( rs_data = dict(
data = self.data, data=self.data,
metadata = self.metadata, metadata=self.metadata,
nodata = self.nodata, nodata=self.nodata,
band_spatial_sampling = self.band_spatial_sampling, band_spatial_sampling=self.band_spatial_sampling,
tile_name = self.tile_name, tile_name=self.tile_name,
dem = self._get_dem(), dem=self._get_dem(),
srf = self._get_srf(), srf=self._get_srf(),
mask_clouds = self._get_mask_clouds() # returns an instance of S2Mask or None if cloud mask is not given by input GMS objects mask_clouds=self._get_mask_clouds()
# returns an instance of S2Mask or None if cloud mask is not given by input GMS objects
) # NOTE: all keys of this dict are later converted to attributes of RSImage ) # NOTE: all keys of this dict are later converted to attributes of RSImage
script = False script = False
...@@ -748,7 +753,7 @@ class AtmCorr(object): ...@@ -748,7 +753,7 @@ class AtmCorr(object):
self.ac_input = dict( self.ac_input = dict(
rs_image=rs_image, rs_image=rs_image,
options=self.options, options=self.options, # type: dict
logger=repr(self.logger), # only a string logger=repr(self.logger), # only a string
script=script script=script
) )
...@@ -789,7 +794,7 @@ class AtmCorr(object): ...@@ -789,7 +794,7 @@ class AtmCorr(object):
self.proc_info = self.ac_input['options']['processing'] # FIXME this is not appended to GMS objects self.proc_info = self.ac_input['options']['processing'] # FIXME this is not appended to GMS objects
# join results # join results
self._join_results_to_inObjs() # sets self.outObjs self._join_results_to_inObjs() # sets self.outObjs
# delete input arrays that are not needed anymore # delete input arrays that are not needed anymore
[inObj.delete_ac_input_arrays() for inObj in self.inObjs] [inObj.delete_ac_input_arrays() for inObj in self.inObjs]
...@@ -802,7 +807,9 @@ class AtmCorr(object): ...@@ -802,7 +807,9 @@ class AtmCorr(object):
""" """
self.logger.info('Joining results of atmospheric correction to input GMS objects.') self.logger.info('Joining results of atmospheric correction to input GMS objects.')
del self.logger # otherwise logging in inObjs would open a second FileHandler to the same file (which is permitted) # delete logger
# -> otherwise logging in inObjs would open a second FileHandler to the same file (which is permitted)
del self.logger
self._join_data_ac() self._join_data_ac()
self._join_data_errors() self._join_data_errors()
...@@ -816,7 +823,8 @@ class AtmCorr(object): ...@@ -816,7 +823,8 @@ class AtmCorr(object):
def _join_data_ac(self): def _join_data_ac(self):
""" """
Join ATMOSPHERICALLY CORRECTED ARRAY as 3D int8 or int16 BOA reflectance array, scaled to scale factor from config. Join ATMOSPHERICALLY CORRECTED ARRAY as 3D int8 or int16 BOA reflectance array, scaled to scale factor from
config.
""" """
if self.results.data_ac is not None: if self.results.data_ac is not None:
...@@ -841,12 +849,14 @@ class AtmCorr(object): ...@@ -841,12 +849,14 @@ class AtmCorr(object):
surf_refl *= CFG.usecase.scale_factor_BOARef # scale using scale factor (output is float16) surf_refl *= CFG.usecase.scale_factor_BOARef # scale using scale factor (output is float16)
# FIXME really set AC nodata values to GMS outZero? # FIXME really set AC nodata values to GMS outZero?
surf_refl[nodata] = oZ_refl # overwrite AC nodata values with GMS outZero surf_refl[nodata] = oZ_refl # overwrite AC nodata values with GMS outZero
surf_refl[np.array(inObj.mask_nodata) == False] = oF_refl # apply the original nodata mask (indicating background values) # apply the original nodata mask (indicating background values)
surf_refl[np.array(inObj.mask_nodata) == False] = oF_refl
if self.results.bad_data_value is np.nan: if self.results.bad_data_value is np.nan:
surf_refl[np.isnan(surf_refl)] = oF_refl surf_refl[np.isnan(surf_refl)] = oF_refl
else: else:
surf_refl[surf_refl == self.results.bad_data_value] = oF_refl # FIXME meaningful to set AC nans to -9999? surf_refl[
surf_refl == self.results.bad_data_value] = oF_refl # FIXME meaningful to set AC nans to -9999?
# overwrite LayerBandsAssignment and use inObj.arr setter to generate a GeoArray # overwrite LayerBandsAssignment and use inObj.arr setter to generate a GeoArray
inObj.LayerBandsAssignment = out_LBA inObj.LayerBandsAssignment = out_LBA
......
...@@ -456,7 +456,7 @@ class DEM_Creator(object): ...@@ -456,7 +456,7 @@ class DEM_Creator(object):
# Build GDAL VRT from pathes and create output DEM # Build GDAL VRT from pathes and create output DEM
with tempFile(dir=CFG.job.path_tempdir, prefix='GeoMultiSens_', suffix='_dem_merged.vrt') as tFm, \ with tempFile(dir=CFG.job.path_tempdir, prefix='GeoMultiSens_', suffix='_dem_merged.vrt') as tFm, \
tempFile(dir=CFG.job.path_tempdir, prefix='GeoMultiSens_', suffix='_dem_out.vrt') as tFo: tempFile(dir=CFG.job.path_tempdir, prefix='GeoMultiSens_', suffix='_dem_out.vrt') as tFo:
try: try:
os.chdir(self.rootpath_DEMs) os.chdir(self.rootpath_DEMs)
......
...@@ -91,7 +91,7 @@ def get_info_from_SQLdb(path_db,tablename,vals2return,cond_dict,records2fetch=0) ...@@ -91,7 +91,7 @@ def get_info_from_SQLdb(path_db,tablename,vals2return,cond_dict,records2fetch=0)
connection = sqlite3.connect(path_db) connection = sqlite3.connect(path_db)
cursor = connection.cursor() cursor = connection.cursor()
condition = "WHERE " + " AND ".join(["%s=?" %(list(cond_dict.keys())[i]) for i in range(len(cond_dict))]) condition = "WHERE " + " AND ".join(["%s=?" %(list(cond_dict.keys())[i]) for i in range(len(cond_dict))])
cursor.execute("SELECT " +','.join(vals2return)+ " FROM " +tablename+ " " + condition, list(cond_dict.values())) cursor.execute("SELECT " +','.join(vals2return)+ " FROM " + tablename + " " + condition, list(cond_dict.values()))
records2return = cursor.fetchall() if records2fetch == 0 else [cursor.fetchone()] if records2fetch == 1 else \ records2return = cursor.fetchall() if records2fetch == 0 else [cursor.fetchone()] if records2fetch == 1 else \
cursor.fetchmany(size = records2fetch) # e.g. [('LE71950282003121EDC00',), ('LE71950282003105ASN00',)] cursor.fetchmany(size = records2fetch) # e.g. [('LE71950282003121EDC00',), ('LE71950282003105ASN00',)]
cursor.close() cursor.close()
......
...@@ -306,7 +306,7 @@ class GMS_object(Dataset): ...@@ -306,7 +306,7 @@ class GMS_object(Dataset):
path_ac_options = PG.get_path_ac_options(self.GMS_identifier) path_ac_options = PG.get_path_ac_options(self.GMS_identifier)
if path_ac_options and os.path.exists(path_ac_options): if path_ac_options and os.path.exists(path_ac_options):
opt_dict = get_ac_options(path_ac_options, validation=False) # don't validate because options contain pathes that do not exist on another server opt_dict = get_ac_options(path_ac_options, validation=False) # don't validate because options contain pathes that do not exist on another server
# update some file paths depending on the current environment # update some file paths depending on the current environment
opt_dict['DEM']['fn'] = CFG.job.path_dem_proc_srtm_90m opt_dict['DEM']['fn'] = CFG.job.path_dem_proc_srtm_90m
...@@ -314,7 +314,7 @@ class GMS_object(Dataset): ...@@ -314,7 +314,7 @@ class GMS_object(Dataset):
for key in opt_dict['RTFO']: for key in opt_dict['RTFO']:
if 'atm_tables_fn' in opt_dict['RTFO'][key]: if 'atm_tables_fn' in opt_dict['RTFO'][key]:
opt_dict['RTFO'][key]['atm_tables_fn'] = PG.get_path_ac_table(key) opt_dict['RTFO'][key]['atm_tables_fn'] = PG.get_path_ac_table(key)
opt_dict['S2Image']['S2_MSI_granule_path'] = None # only a placeholder -> will always be None for GMS usage opt_dict['S2Image']['S2_MSI_granule_path'] = None # only a placeholder -> will always be None for GMS usage
opt_dict['cld_mask']['persistence_file'] = PG.get_path_cloud_class_obj(self.GMS_identifier) opt_dict['cld_mask']['persistence_file'] = PG.get_path_cloud_class_obj(self.GMS_identifier)
opt_dict['cld_mask']['novelty_detector'] = None # FIXME update this after switching to SICOR opt_dict['cld_mask']['novelty_detector'] = None # FIXME update this after switching to SICOR
opt_dict['output'] = [] # outputs are not needed for GMS -> so opt_dict['output'] = [] # outputs are not needed for GMS -> so
...@@ -322,7 +322,7 @@ class GMS_object(Dataset): ...@@ -322,7 +322,7 @@ class GMS_object(Dataset):
if 'uncertainties' in opt_dict: if 'uncertainties' in opt_dict:
opt_dict['uncertainties']['snr_model'] = PG.get_path_snr_model(self.GMS_identifier) opt_dict['uncertainties']['snr_model'] = PG.get_path_snr_model(self.GMS_identifier)
#opt_dict['AC']['n_cores'] = CFG.job.CPUs if CFG.job.allow_subMultiprocessing else 1 # opt_dict['AC']['n_cores'] = CFG.job.CPUs if CFG.job.allow_subMultiprocessing else 1
self._ac_options = opt_dict self._ac_options = opt_dict
else: else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment