Commit 5107ea13 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Fixed bug regarding matplotlib backend. PEP-8 editing.

parent ba047c71
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
__author__ = """Daniel Scheffler""" import os
__email__ = 'daniel.scheffler@gfz-potsdam.de' if 'MPLBACKEND' not in os.environ:
__version__ = '0.6.2' os.environ['MPLBACKEND'] = 'Agg'
__versionalias__ = '20170906.01'
from . import algorithms from . import algorithms
from . import io from . import io
...@@ -13,10 +11,14 @@ from . import processing ...@@ -13,10 +11,14 @@ from . import processing
from . import config from . import config
from .processing.process_controller import process_controller from .processing.process_controller import process_controller
__all__ = ['algorithms', __author__ = """Daniel Scheffler"""
'io', __email__ = 'daniel.scheffler@gfz-potsdam.de'
'misc', __version__ = '0.6.3'
'processing', __versionalias__ = '20170915.01'
'config', __all__ = ['algorithms',
'process_controller', 'io',
] 'misc',
'processing',
'config',
'process_controller',
]
This diff is collapsed.
This diff is collapsed.
...@@ -39,7 +39,7 @@ from sicor.Mask import S2Mask ...@@ -39,7 +39,7 @@ from sicor.Mask import S2Mask
class L1C_object(L1B_object): class L1C_object(L1B_object):
def __init__(self, L1B_obj=None): def __init__(self, L1B_obj=None):
super().__init__() super(L1C_object, self).__init__()
if L1B_obj: if L1B_obj:
# populate attributes # populate attributes
...@@ -326,7 +326,7 @@ class AtmCorr(object): ...@@ -326,7 +326,7 @@ class AtmCorr(object):
for inObj in self.inObjs: for inObj in self.inObjs:
for bandN, bandIdx in inObj.arr.bandnames.items(): for bandN, bandIdx in inObj.arr.bandnames.items():
if bandN not in data_dict: if bandN not in data_dict:
arr2pass = inObj.arr[:,:,bandIdx].astype(np.float32) # conversion to np.float16 will convert -9999 to -10000 arr2pass = inObj.arr[:, :, bandIdx].astype(np.float32) # conversion to np.float16 will convert -9999 to -10000
arr2pass[arr2pass==inObj.arr.nodata] = np.nan # set nodata values to np.nan arr2pass[arr2pass==inObj.arr.nodata] = np.nan # set nodata values to np.nan
data_dict[bandN] = (arr2pass/inObj.meta_odict['ScaleFactor']).astype(np.float16) data_dict[bandN] = (arr2pass/inObj.meta_odict['ScaleFactor']).astype(np.float16)
else: else:
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
############################################################################### """Level 2A Processor: Spatial homogenization"""
#
# Level 2A Processor:
# Spatial homogenization
#
###############################################################################
__author__='Daniel Scheffler'
import collections import collections
import os import os
import warnings import warnings
import numpy as np import numpy as np
from geoarray import GeoArray from geoarray import GeoArray
...@@ -19,6 +12,8 @@ from py_tools_ds.geo.map_info import mapinfo2geotransform ...@@ -19,6 +12,8 @@ from py_tools_ds.geo.map_info import mapinfo2geotransform
from ..config import GMS_config as CFG from ..config import GMS_config as CFG
from .L1C_P import L1C_object from .L1C_P import L1C_object
__author__ = 'Daniel Scheffler'
def get_DESHIFTER_configs(dicts_GMS_obj, attrnames2deshift, proc_bandwise=False, paramsFromUsecase=True, **kwargs): def get_DESHIFTER_configs(dicts_GMS_obj, attrnames2deshift, proc_bandwise=False, paramsFromUsecase=True, **kwargs):
""" """
...@@ -48,41 +43,42 @@ def get_DESHIFTER_configs(dicts_GMS_obj, attrnames2deshift, proc_bandwise=False, ...@@ -48,41 +43,42 @@ def get_DESHIFTER_configs(dicts_GMS_obj, attrnames2deshift, proc_bandwise=False,
""" """
# FIXME diese Methode muss target grid festlegen, auch wenn keine Referenz verfügbar ist! # FIXME diese Methode muss target grid festlegen, auch wenn keine Referenz verfügbar ist!
illegal_kw = [i for i in kwargs if i not in ['align_grids','out_gsd','match_gsd','no_resamp','cliptoextent']] illegal_kw = [i for i in kwargs if i not in ['align_grids', 'out_gsd', 'match_gsd', 'no_resamp', 'cliptoextent']]
assert illegal_kw == [], "'%s' is not a legal keyword argument for L1B_P.get_DESHIFTER_configs()" %illegal_kw[0] assert illegal_kw == [], "'%s' is not a legal keyword argument for L1B_P.get_DESHIFTER_configs()" % illegal_kw[0]
dicts_GMS_obj = [dicts_GMS_obj] if not isinstance(dicts_GMS_obj,list) else dicts_GMS_obj dicts_GMS_obj = [dicts_GMS_obj] if not isinstance(dicts_GMS_obj, list) else dicts_GMS_obj
attrnames2deshift = [attrnames2deshift] if not isinstance(attrnames2deshift,list) else attrnames2deshift attrnames2deshift = [attrnames2deshift] if not isinstance(attrnames2deshift, list) else attrnames2deshift
# get general kwargs # get general kwargs
gen_kwargs = collections.OrderedDict() gen_kwargs = collections.OrderedDict()
if paramsFromUsecase: if paramsFromUsecase:
gen_kwargs.update({'align_grids':CFG.usecase.align_coord_grids}) gen_kwargs.update({'align_grids': CFG.usecase.align_coord_grids})
gen_kwargs.update({'out_gsd' :CFG.usecase.target_gsd}) gen_kwargs.update({'out_gsd': CFG.usecase.target_gsd})
gen_kwargs.update({'match_gsd' :CFG.usecase.match_gsd}) gen_kwargs.update({'match_gsd': CFG.usecase.match_gsd})
else: else:
[gen_kwargs.update({kw:kwargs.get(kw)}) for kw in ['align_grids','out_gsd','match_gsd'] if kw in kwargs] [gen_kwargs.update({kw: kwargs.get(kw)}) for kw in ['align_grids', 'out_gsd', 'match_gsd'] if kw in kwargs]
[gen_kwargs.update({kw:kwargs.get(kw)}) for kw in ['no_resamp','cliptoextent'] if kw in kwargs] [gen_kwargs.update({kw: kwargs.get(kw)}) for kw in ['no_resamp', 'cliptoextent'] if kw in kwargs]
config_dicts = [] config_dicts = []
for obj in dicts_GMS_obj: for obj in dicts_GMS_obj:
# FIXME workaround für fehlende refererence geotransform -> eigentlich müsste nicht gt, sondern target grid berechnet werden # FIXME workaround für fehlende refererence geotransform
assert isinstance(obj,dict) # FIXME -> eigentlich müsste nicht gt, sondern target grid berechnet werden
assert isinstance(obj, dict)
if not obj['coreg_info']['reference geotransform']: if not obj['coreg_info']['reference geotransform']:
obj['coreg_info']['reference geotransform'] = mapinfo2geotransform( obj['coreg_info']['reference geotransform'] = mapinfo2geotransform(
obj['coreg_info']['original map info']) obj['coreg_info']['original map info'])
obj['coreg_info']['reference geotransform'][1] = CFG.usecase.target_gsd[0] obj['coreg_info']['reference geotransform'][1] = CFG.usecase.target_gsd[0]
obj['coreg_info']['reference geotransform'][5] = -abs(CFG.usecase.target_gsd[1]) obj['coreg_info']['reference geotransform'][5] = -abs(CFG.usecase.target_gsd[1])
item2add = [obj] item2add = [obj]
for attrname in attrnames2deshift: for attrname in attrnames2deshift:
attrVal = obj[attrname] attrVal = obj[attrname]
attritem2add = item2add+[attrname] attritem2add = item2add + [attrname]
if isinstance(attrVal,np.ndarray) or isinstance(attrVal,GeoArray) and attrVal.is_inmem: if isinstance(attrVal, np.ndarray) or isinstance(attrVal, GeoArray) and attrVal.is_inmem:
bands = attrVal.shape[2] if attrVal.ndim==3 else None bands = attrVal.shape[2] if attrVal.ndim == 3 else None
elif isinstance(attrVal,GeoArray) and not attrVal.is_inmem: elif isinstance(attrVal, GeoArray) and not attrVal.is_inmem:
if os.path.exists(attrVal): if os.path.exists(attrVal):
bands = attrVal.bands bands = attrVal.bands
else: else:
...@@ -91,18 +87,18 @@ def get_DESHIFTER_configs(dicts_GMS_obj, attrnames2deshift, proc_bandwise=False, ...@@ -91,18 +87,18 @@ def get_DESHIFTER_configs(dicts_GMS_obj, attrnames2deshift, proc_bandwise=False,
elif attrVal is None: elif attrVal is None:
continue continue
else: else:
raise Exception('Unexpected attribute type %s in attribute %s.' %(type(attrVal),attrname)) raise Exception('Unexpected attribute type %s in attribute %s.' % (type(attrVal), attrname))
if proc_bandwise and bands is not None and 'band2process' not in kwargs: if proc_bandwise and bands is not None and 'band2process' not in kwargs:
for bI in range(bands): for bI in range(bands):
kwargs2add = collections.OrderedDict() kwargs2add = collections.OrderedDict()
kwargs2add.update({'band2process':bI+1}) kwargs2add.update({'band2process': bI + 1})
kwargs2add.update(gen_kwargs) kwargs2add.update(gen_kwargs)
banditem2add = attritem2add+[kwargs2add] banditem2add = attritem2add + [kwargs2add]
config_dicts.append(banditem2add) config_dicts.append(banditem2add)
elif 'band2process' in kwargs and kwargs.get('band2process') is not None: elif 'band2process' in kwargs and kwargs.get('band2process') is not None:
assert isinstance(kwargs.get('band2process'),int), "'band2process' must contain an integer." assert isinstance(kwargs.get('band2process'), int), "'band2process' must contain an integer."
kwargs2add = collections.OrderedDict({'band2process':kwargs.get('band2process')}) kwargs2add = collections.OrderedDict({'band2process': kwargs.get('band2process')})
kwargs2add.update(gen_kwargs) kwargs2add.update(gen_kwargs)
attritem2add.append(kwargs2add) attritem2add.append(kwargs2add)
config_dicts.append(attritem2add) config_dicts.append(attritem2add)
...@@ -115,10 +111,10 @@ def get_DESHIFTER_configs(dicts_GMS_obj, attrnames2deshift, proc_bandwise=False, ...@@ -115,10 +111,10 @@ def get_DESHIFTER_configs(dicts_GMS_obj, attrnames2deshift, proc_bandwise=False,
class L2A_object(L1C_object): class L2A_object(L1C_object):
def __init__(self, L1C_obj=None): def __init__(self, L1C_obj=None):
super().__init__() super(L2A_object, self).__init__()
if L1C_obj: if L1C_obj:
# populate attributes # populate attributes
[setattr(self, key, value) for key,value in L1C_obj.__dict__.items()] [setattr(self, key, value) for key, value in L1C_obj.__dict__.items()]
self.proc_level = 'L2A' self.proc_level = 'L2A'
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
############################################################################### """Level 2C Processor: Quality layers"""
#
# Level 2C Processor: from .L2B_P import L2B_object
# Accurracy layers
# __author__ = 'Daniel Scheffler'
###############################################################################
__author__='Daniel Scheffler'
shared = {} shared = {}
res = {} res = {}
from .L2B_P import L2B_object
class L2C_object(L2B_object): class L2C_object(L2B_object):
def __init__(self, L2B_obj=None): def __init__(self, L2B_obj=None):
super(L2C_object, self).__init__()
super().__init__()
if L2B_obj: if L2B_obj:
# populate attributes # populate attributes
...@@ -27,4 +23,4 @@ class L2C_object(L2B_object): ...@@ -27,4 +23,4 @@ class L2C_object(L2B_object):
pass pass
def calc_spectral_accurracy(self): def calc_spectral_accurracy(self):
pass pass
\ No newline at end of file
...@@ -16,13 +16,13 @@ from . import L2A_P ...@@ -16,13 +16,13 @@ from . import L2A_P
from . import L2B_P from . import L2B_P
from . import L2C_P from . import L2C_P
__all__=['GEOPROCESSING', __all__ = ['GEOPROCESSING',
'gms_cloud_classifier', 'gms_cloud_classifier',
'L1A_P', 'L1A_P',
'L1B_P', 'L1B_P',
'L1C_P', 'L1C_P',
'L2A_P', 'L2A_P',
'L2B_P', 'L2B_P',
'L2C_P'] 'L2C_P']
__author__='Daniel Scheffler' __author__ = 'Daniel Scheffler'
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
__author__ = 'Daniel Scheffler'
import os import os
import fnmatch import fnmatch
...@@ -26,6 +24,8 @@ from ..misc.database_tools import get_info_from_postgreSQLdb ...@@ -26,6 +24,8 @@ from ..misc.database_tools import get_info_from_postgreSQLdb
from ..misc.exceptions import FmaskError, FmaskWarning from ..misc.exceptions import FmaskError, FmaskWarning
from geoarray import GeoArray from geoarray import GeoArray
__author__ = 'Daniel Scheffler'
class _FMASK_Runner(object): class _FMASK_Runner(object):
"""The FMASK runner base class (not to be called directly).""" """The FMASK runner base class (not to be called directly)."""
...@@ -56,18 +56,17 @@ class _FMASK_Runner(object): ...@@ -56,18 +56,17 @@ class _FMASK_Runner(object):
tempdir_rootPath = '/dev/shm/GeoMultiSens' tempdir_rootPath = '/dev/shm/GeoMultiSens'
if not os.path.isdir(tempdir_rootPath): if not os.path.isdir(tempdir_rootPath):
os.makedirs(tempdir_rootPath) os.makedirs(tempdir_rootPath)
self.tempdir = tempfile.mkdtemp(dir=tempdir_rootPath, prefix='FMASK__%s__' %os.path.basename(self.path_archive)) self.tempdir = tempfile.mkdtemp(dir=tempdir_rootPath,
prefix='FMASK__%s__' % os.path.basename(self.path_archive))
# create subdirectory for FMASK internal intermediate files # create subdirectory for FMASK internal intermediate files
os.makedirs(os.path.join(self.tempdir, 'FMASK_intermediates')) os.makedirs(os.path.join(self.tempdir, 'FMASK_intermediates'))
def validate_inputs(self): def validate_inputs(self):
if not os.path.exists(self.path_archive): if not os.path.exists(self.path_archive):
raise FileNotFoundError(self.path_archive) raise FileNotFoundError(self.path_archive)
if not self.satellite in ['Landsat-4', 'Landsat-5', 'Landsat-7', 'Landsat-8', 'Sentinel-2A', 'Sentinel-2B']: if self.satellite not in ['Landsat-4', 'Landsat-5', 'Landsat-7', 'Landsat-8', 'Sentinel-2A', 'Sentinel-2B']:
raise ValueError('%s is not a supported satellite for cloud mask calculation via FMASK.' %self.satellite) raise ValueError('%s is not a supported satellite for cloud mask calculation via FMASK.' % self.satellite)
@property @property
def is_GMSConfig_available(self): def is_GMSConfig_available(self):
...@@ -78,14 +77,12 @@ class _FMASK_Runner(object): ...@@ -78,14 +77,12 @@ class _FMASK_Runner(object):
except (EnvironmentError, OSError): except (EnvironmentError, OSError):
return False return False
@property @property
def gdal_path_archive(self): def gdal_path_archive(self):
if not self._gdal_path_archive: if not self._gdal_path_archive:
self._gdal_path_archive = convert_absPathArchive_to_GDALvsiPath(self.path_archive) self._gdal_path_archive = convert_absPathArchive_to_GDALvsiPath(self.path_archive)
return self._gdal_path_archive return self._gdal_path_archive
@property @property
def files_in_archive(self): def files_in_archive(self):
if not self._files_in_archive: if not self._files_in_archive:
...@@ -93,7 +90,6 @@ class _FMASK_Runner(object): ...@@ -93,7 +90,6 @@ class _FMASK_Runner(object):
self._files_in_archive = gdal.ReadDirRecursive(self.gdal_path_archive) self._files_in_archive = gdal.ReadDirRecursive(self.gdal_path_archive)
return self._files_in_archive return self._files_in_archive
@staticmethod @staticmethod
def run_cmd(cmd): def run_cmd(cmd):
output, exitcode, err = subcall_with_output(cmd) output, exitcode, err = subcall_with_output(cmd)
...@@ -102,21 +98,18 @@ class _FMASK_Runner(object): ...@@ -102,21 +98,18 @@ class _FMASK_Runner(object):
if output: if output:
return output.decode('UTF-8') return output.decode('UTF-8')
def extract_tar_archive(self): def extract_tar_archive(self):
with tarfile.open(self.path_archive) as tarF: with tarfile.open(self.path_archive) as tarF:
tarF.extractall(self.tempdir) tarF.extractall(self.tempdir)
self.is_extracted = True self.is_extracted = True
def extract_zip_archive(self): def extract_zip_archive(self):
with zipfile.ZipFile(self.path_archive, "r") as z: with zipfile.ZipFile(self.path_archive, "r") as z:
z.extractall(self.tempdir) z.extractall(self.tempdir)
self.is_extracted = True self.is_extracted = True
def to_saved_rasterFile(self, value, attrname): def to_saved_rasterFile(self, value, attrname):
pathFile = os.path.join(self.tempdir, "%s.bsq" %attrname) pathFile = os.path.join(self.tempdir, "%s.bsq" % attrname)
if isinstance(value, str) and os.path.exists(value): if isinstance(value, str) and os.path.exists(value):
pathFile = value pathFile = value
elif isinstance(value, GeoArray): elif isinstance(value, GeoArray):
...@@ -124,13 +117,12 @@ class _FMASK_Runner(object): ...@@ -124,13 +117,12 @@ class _FMASK_Runner(object):
value.save(pathFile) value.save(pathFile)
else: else:
raise TypeError("The attribute '%s' can only be set by an existing path or an instance of GeoArray. " raise TypeError("The attribute '%s' can only be set by an existing path or an instance of GeoArray. "
"Received %s" %(attrname, type(value))) "Received %s" % (attrname, type(value)))
assert isinstance(pathFile, str) and os.path.exists(pathFile) assert isinstance(pathFile, str) and os.path.exists(pathFile)
return pathFile return pathFile
def calc_cloudMask(self, path_out=None, fmt=None): def calc_cloudMask(self, path_out=None, fmt=None):
if path_out: if path_out:
gdal.Translate(path_out, gdal.Open(self.cloud_mask), format=fmt) gdal.Translate(path_out, gdal.Open(self.cloud_mask), format=fmt)
...@@ -144,14 +136,13 @@ class _FMASK_Runner(object): ...@@ -144,14 +136,13 @@ class _FMASK_Runner(object):
if self.is_GMSConfig_available: if self.is_GMSConfig_available:
self.cloud_mask.legend = \ self.cloud_mask.legend = \
get_mask_classdefinition('mask_clouds', self.satellite) get_mask_classdefinition('mask_clouds', self.satellite)
else: # use default FMASK legend else: # use default FMASK legend
warnings.warn('GMS configuration not available. Using default cloud mask legend.', FmaskWarning) warnings.warn('GMS configuration not available. Using default cloud mask legend.', FmaskWarning)
self.cloud_mask.legend = \ self.cloud_mask.legend = \
{'No Data': 0, 'Clear': 1, 'Cloud': 2, 'Shadow': 3, 'Snow': 4, 'Water': 5} {'No Data': 0, 'Clear': 1, 'Cloud': 2, 'Shadow': 3, 'Snow': 4, 'Water': 5}
return self.cloud_mask return self.cloud_mask
def clean(self): def clean(self):
shutil.rmtree(self.tempdir) shutil.rmtree(self.tempdir)
self.is_extracted = False self.is_extracted = False
...@@ -161,11 +152,7 @@ class _FMASK_Runner(object): ...@@ -161,11 +152,7 @@ class _FMASK_Runner(object):
self._TOARef = None self._TOARef = None
class FMASK_Runner_Landsat(_FMASK_Runner): class FMASK_Runner_Landsat(_FMASK_Runner):
def __init__(self, path_providerArchive, satellite, TOARef=None, opticalDNs=None, thermalDNs=None): def __init__(self, path_providerArchive, satellite, TOARef=None, opticalDNs=None, thermalDNs=None):
"""FMASK wrapper class for Landsat 4-8. """FMASK wrapper class for Landsat 4-8.
...@@ -189,7 +176,7 @@ class FMASK_Runner_Landsat(_FMASK_Runner): ...@@ -189,7 +176,7 @@ class FMASK_Runner_Landsat(_FMASK_Runner):
'Landsat-8': dict(optical='L*_B[1-7,9].TIF', thermal='L*_B1[0,1].TIF', meta='*_MTL.txt') 'Landsat-8': dict(optical='L*_B[1-7,9].TIF', thermal='L*_B1[0,1].TIF', meta='*_MTL.txt')
}[satellite] }[satellite]
super(FMASK_Runner_Landsat,self).__init__(path_providerArchive, satellite, extract_archive=False) super(FMASK_Runner_Landsat, self).__init__(path_providerArchive, satellite, extract_archive=False)
# populate optional attributes # populate optional attributes
if TOARef is not None: if TOARef is not None:
...@@ -199,7 +186,6 @@ class FMASK_Runner_Landsat(_FMASK_Runner): ...@@ -199,7 +186,6 @@ class FMASK_Runner_Landsat(_FMASK_Runner):
if thermalDNs is not None: if thermalDNs is not None:
self.thermal_stack = thermalDNs self.thermal_stack = thermalDNs
@property @property
def optical_stack(self): def optical_stack(self):
if self._optical_stack is None: if self._optical_stack is None:
...@@ -214,12 +200,10 @@ class FMASK_Runner_Landsat(_FMASK_Runner): ...@@ -214,12 +200,10 @@ class FMASK_Runner_Landsat(_FMASK_Runner):
return self._optical_stack return self._optical_stack
@optical_stack.setter @optical_stack.setter
def optical_stack(self, value): def optical_stack(self, value):
self._optical_stack = super(FMASK_Runner_Landsat, self).to_saved_rasterFile(value, 'optical_stack') self._optical_stack = super(FMASK_Runner_Landsat, self).to_saved_rasterFile(value, 'optical_stack')
@property @property
def thermal_stack(self): def thermal_stack(self):
if self._thermal_stack is None: if self._thermal_stack is None:
...@@ -234,43 +218,38 @@ class FMASK_Runner_Landsat(_FMASK_Runner): ...@@ -234,43 +218,38 @@ class FMASK_Runner_Landsat(_FMASK_Runner):
return self._thermal_stack return self._thermal_stack
@thermal_stack.setter @thermal_stack.setter
def thermal_stack(self, value): def thermal_stack(self, value):
self._thermal_stack = super(FMASK_Runner_Landsat, self).to_saved_rasterFile(value, 'thermal_stack') self._thermal_stack = super(FMASK_Runner_Landsat, self).to_saved_rasterFile(value, 'thermal_stack')
@property @property
def metaFile(self): def metaFile(self):
if not self._metaFile: if not self._metaFile:
if not self.is_extracted: if not self.is_extracted:
self.extract_tar_archive() self.extract_tar_archive()
self._metaFile = os.path.join(self.tempdir,fnmatch.filter(os.listdir(self.tempdir), '*_MTL.txt')[0]) self._metaFile = os.path.join(self.tempdir, fnmatch.filter(os.listdir(self.tempdir), '*_MTL.txt')[0])
return self._metaFile return self._metaFile
@property @property
def angles_stack(self): def angles_stack(self):
if self._angles_stack is None: if self._angles_stack is None:
self._angles_stack = os.path.join(self.tempdir, 'angles.vrt') self._angles_stack = os.path.join(self.tempdir, 'angles.vrt')
self.run_cmd('fmask_usgsLandsatMakeAnglesImage.py -m %s -t %s -o %s' self.run_cmd('fmask_usgsLandsatMakeAnglesImage.py -m %s -t %s -o %s'
% (self.metaFile, self.optical_stack, self._angles_stack)) % (self.metaFile, self.optical_stack, self._angles_stack