Commit 6929e4d1 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

PEP-8 editing.


Former-commit-id: a5f41a39
parent b2fef42c
# -*- coding: utf-8 -*-
"""
Level 1C Processor:
Performed operations:
Atmospheric correction of TOA-reflectance data:
Written by Daniel Scheffler
"""
"""Level 1C Processor: Atmospheric correction of TOA-reflectance data."""
import warnings
import re
......@@ -16,6 +9,7 @@ import traceback
from typing import List
import numpy as np
try:
from osgeo import osr
except ImportError:
......@@ -36,6 +30,8 @@ from sicor.sicor_ac import ac_gms
from sicor.sensors import RSImage
from sicor.Mask import S2Mask
__author__ = 'Daniel Scheffler'
class L1C_object(L1B_object):
def __init__(self, L1B_obj=None):
......@@ -43,7 +39,7 @@ class L1C_object(L1B_object):
if L1B_obj:
# populate attributes
[setattr(self, key, value) for key,value in L1B_obj.__dict__.items()]
[setattr(self, key, value) for key, value in L1B_obj.__dict__.items()]
# private attributes
self._VZA_arr = None
......@@ -96,7 +92,7 @@ class L1C_object(L1B_object):
float(self.meta_odict['ViewingAngle']),
float(self.meta_odict['FieldOfView']),
self.logger,
nodata_mask=None, # dont overwrite areas outside the image with nodata
nodata_mask=None, # dont overwrite areas outside image with nodata
outFill=get_outFillZeroSaturated(np.float32)[0],
meshwidth=10) # for faster processing
return self._VZA_arr
......@@ -198,7 +194,11 @@ class L1C_object(L1B_object):
self.SAA_arr = None # not needed anymore
self.RAA_arr = None # not needed anymore
self.lonlat_arr = None # not needed anymore
del self.dem # uses deleter # would have to be resampled when writing MGRS tiles -> better to directly warp it to the output dims and projection
# use self.dem deleter
# would have to be resampled when writing MGRS tiles
# -> better to directly warp it to the output dims and projection
del self.dem
class AtmCorr(object):
......@@ -214,25 +214,25 @@ class AtmCorr(object):
L1C_objs = L1C_objs if isinstance(L1C_objs, tuple) else (L1C_objs,)
# hidden attributes
self._logger = None
self._GSDs = []
self._data = {}
self._logger = None
self._GSDs = []
self._data = {}
self._metadata = {}
self._nodata = {}
self._nodata = {}
self._band_spatial_sampling = {}
self._options = {}
self._options = {}
# assertions
scene_IDs = [obj.scene_ID for obj in L1C_objs]
assert len(list(set(scene_IDs))) == 1, \
"Input GMS objects for 'AtmCorr' must all belong to the same scene ID!. Received %s." %scene_IDs
"Input GMS objects for 'AtmCorr' must all belong to the same scene ID!. Received %s." % scene_IDs
self.inObjs = L1C_objs # type: List[L1B_object]
self.inObjs = L1C_objs # type: List[L1C_object]
self.reporting = reporting
self.ac_input = {} # set by self.run_atmospheric_correction()
self.results = None # direct output of external atmCorr module (set by run_atmospheric_correction)
self.ac_input = {} # set by self.run_atmospheric_correction()
self.results = None # direct output of external atmCorr module (set by run_atmospheric_correction)
self.proc_info = {}
self.outObjs = [] # atmospherically corrected L1C objects
self.outObjs = [] # atmospherically corrected L1C objects
# append AtmCorr object to input L1C objects
# [setattr(L1C_obj, 'AtmCorr', self) for L1C_obj in self.inObjs] # too big for serialization
......@@ -257,7 +257,7 @@ class AtmCorr(object):
# add additional file handlers for the remaining inObj (that belong to the same scene_ID)
for inObj in self.inObjs[1:]:
path_logfile = inObj.pathGen.get_path_logfile()
fileHandler = logging.FileHandler(path_logfile, mode='a')
fileHandler = logging.FileHandler(path_logfile, mode='a')
fileHandler.setFormatter(logger_atmCorr.formatter_fileH)
fileHandler.setLevel(logging.DEBUG)
......@@ -294,7 +294,7 @@ class AtmCorr(object):
for obj in self.inObjs:
if obj.arr.xgsd != obj.arr.ygsd:
warnings.warn("X/Y GSD is not equal for entity ID %s" % obj.entity_ID +
(' (%s)'%obj.subsystem if obj.subsystem else '') +
(' (%s)' % obj.subsystem if obj.subsystem else '') +
'Using X-GSD as key for spatial sampling dictionary.')
self._GSDs.append(obj.arr.xgsd)
......@@ -326,19 +326,20 @@ class AtmCorr(object):
for inObj in self.inObjs:
for bandN, bandIdx in inObj.arr.bandnames.items():
if bandN not in data_dict:
arr2pass = inObj.arr[:, :, bandIdx].astype(np.float32) # conversion to np.float16 will convert -9999 to -10000
arr2pass[arr2pass==inObj.arr.nodata] = np.nan # set nodata values to np.nan
data_dict[bandN] = (arr2pass/inObj.meta_odict['ScaleFactor']).astype(np.float16)
# float32! -> conversion to np.float16 will convert -9999 to -10000
arr2pass = inObj.arr[:, :, bandIdx].astype(np.float32)
arr2pass[arr2pass == inObj.arr.nodata] = np.nan # set nodata values to np.nan
data_dict[bandN] = (arr2pass / inObj.meta_odict['ScaleFactor']).astype(np.float16)
else:
inObj.logger.warning("Band '%s' cannot be included into atmospheric correction because it "
"exists multiple times." % bandN)
# validate: data must have all bands needed for AC
full_LBA = get_LayerBandsAssignment(self.inObjs[0].GMS_identifier, return_fullLBA=True)
all_bNs_AC = ['B%s'% i if len(i) == 2 else 'B0%s' % i for i in full_LBA]
full_LBA = get_LayerBandsAssignment(self.inObjs[0].GMS_identifier, return_fullLBA=True)
all_bNs_AC = ['B%s' % i if len(i) == 2 else 'B0%s' % i for i in full_LBA]
if not all([bN in list(data_dict.keys()) for bN in all_bNs_AC]):
raise RuntimeError('Atmospheric correction did not receive all the needed bands. \n\tExpected: %s;\n\t'
'Received: %s' %(str(all_bNs_AC), str(list(sorted(data_dict.keys())))))
'Received: %s' % (str(all_bNs_AC), str(list(sorted(data_dict.keys())))))
self._data = data_dict
......@@ -436,21 +437,23 @@ class AtmCorr(object):
______ key:SENSING_TIME, value_type:<class 'datetime.datetime'>, repr: 2016-03-26 10:34:06.538000+00:00
"""
# TODO add SRF object
metadata = {}
if not self._metadata:
del self.logger # otherwise each input object would have multiple fileHandlers
metadata['U'] = self.inObjs[0].meta_odict['EarthSunDist']
metadata['SENSING_TIME'] = self.inObjs[0].acq_datetime
# metadata['SENSING_TIME'] = datetime.strptime('2015-08-12 10:40:21 +0000', '%Y-%m-%d %H:%M:%S %z')
metadata['viewing_zenith'] = self._meta_get_viewing_zenith()
metadata['viewing_azimuth'] = self._meta_get_viewing_azimuth()
metadata['relative_viewing_azimuth'] = self._meta_get_relative_viewing_azimuth()
metadata['sun_mean_azimuth'] = self.inObjs[0].meta_odict['SunAzimuth']
metadata['sun_mean_zenith'] = 90-self.inObjs[0].meta_odict['SunElevation']
metadata['solar_irradiance'] = self._meta_get_solar_irradiance()
metadata['aux_data'] = self._meta_get_aux_data()
metadata['spatial_samplings'] = self._meta_get_spatial_samplings()
metadata = dict(
U=self.inObjs[0].meta_odict['EarthSunDist'],
SENSING_TIME=self.inObjs[0].acq_datetime,
# SENSING_TIME=datetime.strptime('2015-08-12 10:40:21 +0000', '%Y-%m-%d %H:%M:%S %z'),
viewing_zenith=self._meta_get_viewing_zenith(),
viewing_azimuth=self._meta_get_viewing_azimuth(),
relative_viewing_azimuth=self._meta_get_relative_viewing_azimuth(),
sun_mean_azimuth=self.inObjs[0].meta_odict['SunAzimuth'],
sun_mean_zenith=90 - self.inObjs[0].meta_odict['SunElevation'],
solar_irradiance=self._meta_get_solar_irradiance(),
aux_data=self._meta_get_aux_data(),
spatial_samplings=self._meta_get_spatial_samplings()
)
self._metadata = metadata
......@@ -460,6 +463,7 @@ class AtmCorr(object):
def options(self):
"""Returns a dictionary containing AC options.
"""
# type: -> dict
if self._options:
return self._options
else:
......@@ -504,12 +508,12 @@ class AtmCorr(object):
# set spatial information
spatial_samplings[inObj.arr.xgsd] = dict(
ULX = inObj.arr.box.boxMapYX[0][1],
ULY = inObj.arr.box.boxMapYX[0][0],
XDIM = inObj.arr.xgsd,
YDIM = -inObj.arr.ygsd,
NROWS = inObj.arr.rows,
NCOLS = inObj.arr.cols)
ULX=inObj.arr.box.boxMapYX[0][1],
ULY=inObj.arr.box.boxMapYX[0][0],
XDIM=inObj.arr.xgsd,
YDIM=-inObj.arr.ygsd,
NROWS=inObj.arr.rows,
NCOLS=inObj.arr.cols)
return spatial_samplings
......@@ -712,7 +716,7 @@ class AtmCorr(object):
if inObj.arr.xgsd == cm_geoarray.xgsd:
inObj.mask_clouds = cm_geoarray
inObj.build_combined_masks_array()
break # appending it to one inObj is enough
break # appending it to one inObj is enough
return S2Mask(mask_array=cm_array,
mask_legend=cm_legend,
......@@ -731,14 +735,15 @@ class AtmCorr(object):
self.logger.info('Calculating input data for atmospheric correction...')
rs_data = dict(
data = self.data,
metadata = self.metadata,
nodata = self.nodata,
band_spatial_sampling = self.band_spatial_sampling,
tile_name = self.tile_name,
dem = self._get_dem(),
srf = self._get_srf(),
mask_clouds = self._get_mask_clouds() # returns an instance of S2Mask or None if cloud mask is not given by input GMS objects
data=self.data,
metadata=self.metadata,
nodata=self.nodata,
band_spatial_sampling=self.band_spatial_sampling,
tile_name=self.tile_name,
dem=self._get_dem(),
srf=self._get_srf(),
mask_clouds=self._get_mask_clouds()
# returns an instance of S2Mask or None if cloud mask is not given by input GMS objects
) # NOTE: all keys of this dict are later converted to attributes of RSImage
script = False
......@@ -748,7 +753,7 @@ class AtmCorr(object):
self.ac_input = dict(
rs_image=rs_image,
options=self.options,
options=self.options, # type: dict
logger=repr(self.logger), # only a string
script=script
)
......@@ -789,7 +794,7 @@ class AtmCorr(object):
self.proc_info = self.ac_input['options']['processing'] # FIXME this is not appended to GMS objects
# join results
self._join_results_to_inObjs() # sets self.outObjs
self._join_results_to_inObjs() # sets self.outObjs
# delete input arrays that are not needed anymore
[inObj.delete_ac_input_arrays() for inObj in self.inObjs]
......@@ -802,7 +807,9 @@ class AtmCorr(object):
"""
self.logger.info('Joining results of atmospheric correction to input GMS objects.')
del self.logger # otherwise logging in inObjs would open a second FileHandler to the same file (which is permitted)
# delete logger
# -> otherwise logging in inObjs would open a second FileHandler to the same file (which is permitted)
del self.logger
self._join_data_ac()
self._join_data_errors()
......@@ -816,7 +823,8 @@ class AtmCorr(object):
def _join_data_ac(self):
"""
Join ATMOSPHERICALLY CORRECTED ARRAY as 3D int8 or int16 BOA reflectance array, scaled to scale factor from config.
Join ATMOSPHERICALLY CORRECTED ARRAY as 3D int8 or int16 BOA reflectance array, scaled to scale factor from
config.
"""
if self.results.data_ac is not None:
......@@ -841,12 +849,14 @@ class AtmCorr(object):
surf_refl *= CFG.usecase.scale_factor_BOARef # scale using scale factor (output is float16)
# FIXME really set AC nodata values to GMS outZero?
surf_refl[nodata] = oZ_refl # overwrite AC nodata values with GMS outZero
surf_refl[np.array(inObj.mask_nodata) == False] = oF_refl # apply the original nodata mask (indicating background values)
# apply the original nodata mask (indicating background values)
surf_refl[np.array(inObj.mask_nodata) == False] = oF_refl
if self.results.bad_data_value is np.nan:
surf_refl[np.isnan(surf_refl)] = oF_refl
else:
surf_refl[surf_refl == self.results.bad_data_value] = oF_refl # FIXME meaningful to set AC nans to -9999?
surf_refl[
surf_refl == self.results.bad_data_value] = oF_refl # FIXME meaningful to set AC nans to -9999?
# overwrite LayerBandsAssignment and use inObj.arr setter to generate a GeoArray
inObj.LayerBandsAssignment = out_LBA
......
......@@ -456,7 +456,7 @@ class DEM_Creator(object):
# Build GDAL VRT from pathes and create output DEM
with tempFile(dir=CFG.job.path_tempdir, prefix='GeoMultiSens_', suffix='_dem_merged.vrt') as tFm, \
tempFile(dir=CFG.job.path_tempdir, prefix='GeoMultiSens_', suffix='_dem_out.vrt') as tFo:
tempFile(dir=CFG.job.path_tempdir, prefix='GeoMultiSens_', suffix='_dem_out.vrt') as tFo:
try:
os.chdir(self.rootpath_DEMs)
......
......@@ -91,7 +91,7 @@ def get_info_from_SQLdb(path_db,tablename,vals2return,cond_dict,records2fetch=0)
connection = sqlite3.connect(path_db)
cursor = connection.cursor()
condition = "WHERE " + " AND ".join(["%s=?" %(list(cond_dict.keys())[i]) for i in range(len(cond_dict))])
cursor.execute("SELECT " +','.join(vals2return)+ " FROM " +tablename+ " " + condition, list(cond_dict.values()))
cursor.execute("SELECT " +','.join(vals2return)+ " FROM " + tablename + " " + condition, list(cond_dict.values()))
records2return = cursor.fetchall() if records2fetch == 0 else [cursor.fetchone()] if records2fetch == 1 else \
cursor.fetchmany(size = records2fetch) # e.g. [('LE71950282003121EDC00',), ('LE71950282003105ASN00',)]
cursor.close()
......
......@@ -306,7 +306,7 @@ class GMS_object(Dataset):
path_ac_options = PG.get_path_ac_options(self.GMS_identifier)
if path_ac_options and os.path.exists(path_ac_options):
opt_dict = get_ac_options(path_ac_options, validation=False) # don't validate because options contain pathes that do not exist on another server
opt_dict = get_ac_options(path_ac_options, validation=False) # don't validate because options contain pathes that do not exist on another server
# update some file paths depending on the current environment
opt_dict['DEM']['fn'] = CFG.job.path_dem_proc_srtm_90m
......@@ -314,7 +314,7 @@ class GMS_object(Dataset):
for key in opt_dict['RTFO']:
if 'atm_tables_fn' in opt_dict['RTFO'][key]:
opt_dict['RTFO'][key]['atm_tables_fn'] = PG.get_path_ac_table(key)
opt_dict['S2Image']['S2_MSI_granule_path'] = None # only a placeholder -> will always be None for GMS usage
opt_dict['S2Image']['S2_MSI_granule_path'] = None # only a placeholder -> will always be None for GMS usage
opt_dict['cld_mask']['persistence_file'] = PG.get_path_cloud_class_obj(self.GMS_identifier)
opt_dict['cld_mask']['novelty_detector'] = None # FIXME update this after switching to SICOR
opt_dict['output'] = [] # outputs are not needed for GMS -> so
......@@ -322,7 +322,7 @@ class GMS_object(Dataset):
if 'uncertainties' in opt_dict:
opt_dict['uncertainties']['snr_model'] = PG.get_path_snr_model(self.GMS_identifier)
#opt_dict['AC']['n_cores'] = CFG.job.CPUs if CFG.job.allow_subMultiprocessing else 1
# opt_dict['AC']['n_cores'] = CFG.job.CPUs if CFG.job.allow_subMultiprocessing else 1
self._ac_options = opt_dict
else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment