Commit b0333947 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Created a commonly usable base class "Dataset" from which GMS_object is subclassed.

- added submodule 'model' including the module 'dataset'
- moved all GMS specific attributes and functions of GMS_object to new class /models/dataset/Dataset

updated __version__
parent 9b284fe8
......@@ -15,7 +15,7 @@ from . import config
from .processing.process_controller import process_controller
__version__ = '20170428.01'
__version__ = '20170523.01'
__author__ = 'Daniel Scheffler'
__all__ = ['algorithms',
'io',
......
......@@ -115,7 +115,7 @@ class METADATA(object):
self.map_info = []
self.projection = ""
self.wvlUnit = ""
self.spec_vals = {}
self.spec_vals = {'fill': None, 'zero': None, 'saturated':None}
"""****OBJECT METHODS******************************************************"""
......
......@@ -39,7 +39,7 @@ if __name__ == "__main__":
import sys
from datetime import datetime
sys.path.append("/home/danscheff/GeoMultiSens_dev/") # FIXME
sys.path.append("/home/danscheff/GeoMultiSens/") # FIXME
fn_l1a = glob("./clfs/ETM+*.pkl")[0] # gms l1a object
with open(fn_l1a, "rb") as fl:
......
......@@ -25,12 +25,12 @@ except ImportError:
import gdalnumeric
from geoarray import GeoArray, NoDataMask, CloudMask
from geoarray import GeoArray
from py_tools_ds.ptds.geo.coord_grid import is_coord_grid_equal
from py_tools_ds.ptds.geo.projection import WKT2EPSG
from py_tools_ds.ptds.geo.coord_calc import calc_FullDataset_corner_positions
from py_tools_ds.ptds.geo.coord_trafo import pixelToLatLon, pixelToMapYX, imXY2mapXY
from py_tools_ds.ptds.geo.map_info import geotransform2mapinfo, mapinfo2geotransform
from py_tools_ds.ptds.geo.coord_calc import calc_FullDataset_corner_positions
from py_tools_ds.ptds.geo.coord_trafo import pixelToLatLon, pixelToMapYX, imXY2mapXY
from ..misc.logging import GMS_logger
from ..misc.mgrs_tile import MGRS_tile
......@@ -44,77 +44,48 @@ from ..io import Output_writer as OUT_W
from ..misc import helper_functions as HLP_F
from ..misc import definition_dicts as DEF_D
from ..model.dataset import Dataset
from ..misc.logging import GMS_logger as DatasetLogger
from S2SCAPEM.options import get_options as get_ac_options
class GMS_object(object):
def __init__(self):
# protected attributes
self._logger = None
self._loggers_disabled = False
self._log = ''
self._GMS_identifier = None
self._MetaObj = None
self._meta_odict = None
self._LayerBandsAssignment = None
self._dict_LayerOptTherm = None
self._coreg_needed = None
self._resamp_needed = None
self._arr = None
self._mask_nodata = None
self._mask_clouds = None
self._mask_clouds_confidence= None
self._masks = None
self._dem = None
self._pathGen = None
self._ac_options = {}
self._ac_errors = None
# defaults
self.proc_level = 'init'
self.job_ID = CFG.job.ID
self.image_type = ''
self.satellite = ''
self.sensor = ''
self.subsystem = ''
self.sensormode = ''
self.acq_datetime = None # also includes time, set by from_disk() and within L1A_P
self.entity_ID = ''
self.scene_ID = -9999
self.filename = ''
self.dataset_ID = -9999
#self.dataset_ID = int(DB_T.get_info_from_postgreSQLdb(CFG.job.conn_database, 'scenes', ['datasetid'],
class GMS_object(Dataset):
def __init__(self, pathImage=''):
# get all attributes of base class "Dataset"
super(GMS_object, self).__init__()
# add EnMAP specific attributes
self._dict_LayerOptTherm = None
self.job_ID = CFG.job.ID
#self.dataset_ID = int(DB_T.get_info_from_postgreSQLdb(CFG.job.conn_database, 'scenes', ['datasetid'],
# {'id': self.scene_ID})[0][0]) if self.scene_ID !=-9999 else -9999 # FIXME not needed anymore?
self.outInterleave = 'bsq'
self.VAA_mean = None # set by self.calc_mean_VAA()
self.corner_lonlat = None
self.trueDataCornerPos = [] # set by self.calc_corner_positions()
self.trueDataCornerLonLat = [] # set by self.calc_corner_positions()
self.fullSceneCornerPos = [] # set by self.calc_corner_positions()
self.fullSceneCornerLonLat = [] # set by self.calc_corner_positions()
self.shape_fullArr = [None, None, None] # rows,cols,bands of the full scene (not of the subset as possibly represented by self.arr.shape)
self.arr_shape = 'cube'
self.arr_desc = '' # description of data units for self.arr
self.arr_pos = None # <tuple> in the form ((row_start,row_end),(col_start,col_end))
self.tile_pos = None # <list>, filled by self.get_tilepos()
self.GeoTransProj_ok = True # set by self.validate_GeoTransProj_GeoAlign()
self.GeoAlign_ok = True # set by self.validate_GeoTransProj_GeoAlign()
self.masks_meta = {} # set by self.build_L1A_masks()
self.scenes_proc_ID = None # set by Output writer after creation/update of db record in table scenes_proc
self.mgrs_tiles_proc_ID = None # set by Output writer after creation/update of db record in table mgrs_tiles_proc
self.MGRS_info = None
# self.CLD_obj = CLD_P.GmsCloudClassifier(classifier=self.path_cloud_class_obj)
self.scenes_proc_ID = None # set by Output writer after creation/update of db record in table scenes_proc
self.mgrs_tiles_proc_ID = None # set by Output writer after creation/update of db record in table mgrs_tiles_proc
self.MGRS_info = None
# set pathes
self.path_archive = ''
self.path_procdata = ''
self.ExtractedFolder = ''
self.path_cloud_class_obj = ''
self.baseN = ''
self.path_logfile = ''
self.path_archive_valid = False
self.path_InFilePreprocessor= None
self.path_MetaPreprocessor = None
self.path_cloud_class_obj = ''
# handle initialization arguments
if pathImage:
self.arr = pathImage # run the setter for 'arr' of the base class 'Dataset' which creates an Instance of GeoArray
def __getstate__(self):
"""Defines how the attributes of GMS object are pickled."""
super(GMS_object, self).__getstate__()
# delete arrays if their in-mem size is to big to be pickled
# => (avoids MaybeEncodingError: Error sending result: '[<GeoMultiSens.algorithms.L2C_P.L2C_object
# object at 0x7fc44f6399e8>]'. Reason: 'error("'i' format requires -2147483648 <= number <= 2147483647",)')
if self.proc_level=='L2C' and CFG.job.exec_mode=='Flink':
if self.mask_nodata is not None and self.masks.bands>1 and self.mask_clouds is not None: # FIXME check by bandname
del self.masks
def set_pathes(self):
......@@ -156,42 +127,6 @@ class GMS_object(object):
'Invalid path for temporary files. Directory %s does not exist.' % self.ExtractedFolder
def __getstate__(self):
"""Defines how the attributes of GMS object are pickled."""
self.close_GMS_loggers()
del self.pathGen # path generator can only be used for the current processing level
# delete arrays if their in-mem size is to big to be pickled
# => (avoids MaybeEncodingError: Error sending result: '[<GeoMultiSens_dev.algorithms.L2C_P.L2C_object
# object at 0x7fc44f6399e8>]'. Reason: 'error("'i' format requires -2147483648 <= number <= 2147483647",)')
if self.proc_level=='L2C' and CFG.job.exec_mode=='Flink':
if self.mask_nodata is not None and self.masks.bands>1 and self.mask_clouds is not None: # FIXME check by bandname
del self.masks
return self.__dict__
def __setstate__(self, ObjDict):
"""Defines how the attributes of GMS object are unpickled."""
self.__dict__ = ObjDict
# TODO unpickle meta to MetaObj
def __deepcopy__(self, memodict={}):
"""Returns a deepcopy of the object excluding loggers because loggers are not serializable."""
cls = self.__class__
result = cls.__new__(cls)
self.close_GMS_loggers()
del self.pathGen # has a logger
memodict[id(self)] = result
for k, v in self.__dict__.items():
setattr(result, k, copy.deepcopy(v, memodict))
return result
@property
def logger(self):
if self._loggers_disabled:
......@@ -199,8 +134,8 @@ class GMS_object(object):
if self._logger and self._logger.handlers[:]:
return self._logger
else:
self._logger = GMS_logger('log__' + self.baseN, fmt_suffix=self.scene_ID, path_logfile=self.path_logfile,
log_level=CFG.job.log_level, append=True)
self._logger = DatasetLogger('log__' + self.baseN, fmt_suffix=self.scene_ID, path_logfile=self.path_logfile,
log_level=CFG.job.log_level, append=True)
return self._logger
......@@ -215,17 +150,6 @@ class GMS_object(object):
self._logger = logger
@property # FIXME does not work yet
def log(self):
"""Returns a string of all logged messages until now."""
return self._log
@log.setter
def log(self, string):
assert isinstance(string, str), "'log' can only be set to a string. Got %s." %type(string)
self._log = string
@property
def GMS_identifier(self):
......@@ -291,31 +215,12 @@ class GMS_object(object):
self._meta_odict = None
@property
def LayerBandsAssignment(self):
# FIXME merge that with self.MetaObj.LayerBandsAssignment -> otherwise a change of LBA in MetaObj is not recognized here
if self._LayerBandsAssignment:
return self._LayerBandsAssignment
elif self.image_type=='RSD':
self._LayerBandsAssignment = get_LayerBandsAssignment(self.GMS_identifier) \
if self.sensormode != 'P' else get_LayerBandsAssignment(self.GMS_identifier, nBands=1)
return self._LayerBandsAssignment
else:
return ''
@LayerBandsAssignment.setter
def LayerBandsAssignment(self, LBA_list):
self._LayerBandsAssignment = LBA_list
self.MetaObj.LayerBandsAssignment = LBA_list
@property
def dict_LayerOptTherm(self):
if self._dict_LayerOptTherm:
return self._dict_LayerOptTherm
elif self.LayerBandsAssignment:
self._dict_LayerOptTherm = get_dict_LayerOptTherm(self.GMS_identifier,self.LayerBandsAssignment)
self._dict_LayerOptTherm = get_dict_LayerOptTherm(self.identifier,self.LayerBandsAssignment)
return self._dict_LayerOptTherm
else:
return None
......@@ -352,151 +257,6 @@ class GMS_object(object):
self._resamp_needed = value
@property
def arr(self):
# TODO this must return a subset if self.subset is not None
return self._arr
@arr.setter
def arr(self, *geoArr_initArgs):
# TODO this must be able to handle subset inputs in tiled processing
self._arr = GeoArray(*geoArr_initArgs)
# set nodata value and geoinfos
# NOTE: MetaObj is NOT gettable before import_metadata has been executed!
if hasattr(self,'MetaObj') and self.MetaObj:
self._arr.nodata = self.MetaObj.spec_vals['fill']
self._arr.gt = mapinfo2geotransform(self.MetaObj.map_info) if self.MetaObj.map_info else [0,1,0,0,0,-1]
self._arr.prj = self.MetaObj.projection
else:
self._arr.nodata = DEF_D.get_outFillZeroSaturated(self._arr.dtype)[0]
if hasattr(self,'meta_odict') and self.meta_odict:
self._arr.gt = mapinfo2geotransform(self.meta_odict['map info'])
self._arr.prj = self.meta_odict['coordinate system string']
# set bandnames like this: [B01, .., B8A,]
if self.LayerBandsAssignment:
if len(self.LayerBandsAssignment) == self._arr.bands:
self._arr.bandnames = self.LBA2bandnames(self.LayerBandsAssignment)
else:
warnings.warn("Cannot set 'bandnames' attribute of GMS_object.arr because LayerBandsAssignment has %s "
"bands and GMS_object.arr has %s bands." %(len(self.LayerBandsAssignment), self._arr.bands))
@arr.deleter
def arr(self):
self._arr = None
@property
def mask_nodata(self):
if self._mask_nodata is not None:
if not self._mask_nodata.is_inmem and self._mask_nodata.bands > 1:
# NoDataMask object self._mask_nodata points to multi-band image file (bands mask_nodata/mask_clouds)
# -> read processes of not needed bands need to be avoided
self._mask_nodata = NoDataMask(self._mask_nodata[:,:,0],
geotransform = self._mask_nodata.gt,
projection = self._mask_nodata.prj)
return self._mask_nodata
elif self._masks:
# return nodata mask from self.masks
self._mask_nodata = NoDataMask(self._masks[:,:,0], # TODO use band names
geotransform = self._masks.gt,
projection = self._masks.prj)
return self._mask_nodata
elif isinstance(self.arr, GeoArray):
self.logger.info('Calculating nodata mask...')
self._mask_nodata = self.arr.mask_nodata # calculates mask nodata if not already present
return self._mask_nodata
else:
return None
@mask_nodata.setter
def mask_nodata(self, *geoArr_initArgs):
if geoArr_initArgs[0] is not None:
self._mask_nodata = NoDataMask(*geoArr_initArgs)
self._mask_nodata.nodata = False
self._mask_nodata.gt = self.arr.gt
self._mask_nodata.prj = self.arr.prj
else:
del self.mask_nodata
@mask_nodata.deleter
def mask_nodata(self):
self._mask_nodata = None
@property
def mask_clouds(self):
if self._mask_clouds is not None:
if not self._mask_clouds.is_inmem and self._mask_clouds.bands > 1:
# CloudMask object self._mask_clouds points to multi-band image file on disk (bands mask_nodata/mask_clouds)
# -> read processes of not needed bands need to be avoided
self._mask_clouds = CloudMask(self._mask_clouds[:,:,1],
geotransform = self._mask_clouds.gt,
projection = self._mask_clouds.prj) # TODO add legend
return self._mask_clouds
elif self._masks and self._masks.bands>1: # FIXME this will not be secure if there are more than 2 bands
# return cloud mask from self.masks
self._mask_clouds = CloudMask(self._masks[:, :, 1], # TODO use band names
geotransform = self._masks.gt,
projection = self._masks.prj)
return self._mask_clouds
else:
return None # TODO don't calculate cloud mask?
@mask_clouds.setter
def mask_clouds(self, *geoArr_initArgs):
if geoArr_initArgs[0] is not None: # FIXME shape validation?
self._mask_clouds = CloudMask(*geoArr_initArgs)
self._mask_clouds.nodata = 0
self._mask_clouds.gt = self.arr.gt
self._mask_clouds.prj = self.arr.prj
else:
del self.mask_clouds
@mask_clouds.deleter
def mask_clouds(self):
self._mask_clouds = None
@property
def mask_clouds_confidence(self):
return self._mask_clouds_confidence
@mask_clouds_confidence.setter
def mask_clouds_confidence(self, *geoArr_initArgs):
if geoArr_initArgs[0] is not None:
cnfArr = GeoArray(*geoArr_initArgs)
assert cnfArr.shape == self.arr.shape[:2],\
"The 'mask_clouds_confidence' GeoArray can only be instanced with an array of the same dimensions " \
"like GMS_obj.arr. Got %s." %str(cnfArr.shape)
if cnfArr._nodata is None:
cnfArr.nodata = DEF_D.get_outFillZeroSaturated(cnfArr.dtype)[0]
cnfArr.gt = self.arr.gt
cnfArr.prj = self.arr.prj
self._mask_clouds_confidence = cnfArr
else:
del self._mask_clouds_confidence
@mask_clouds_confidence.deleter
def mask_clouds_confidence(self):
self._mask_clouds_confidence = None
@property
def masks(self):
return self._masks
......@@ -522,63 +282,6 @@ class GMS_object(object):
self._masks = None
@property
def dem(self):
"""
Returns an SRTM DEM in the exact dimension an pixel grid of self.arr as an instance of GeoArray.
"""
if self._dem is None:
self.logger.info('Generating DEM...')
self._dem = INP_R.get_dem_by_extent(
[(YX[1],YX[0]) for YX in self.arr.box.boxMapYX], self.arr.prj, self.arr.xgsd, self.arr.ygsd)
self._dem.nodata = DEF_D.get_outFillZeroSaturated(self._dem.dtype)[0]
return self._dem
@dem.setter
def dem(self, *geoArr_initArgs):
if geoArr_initArgs[0] is not None:
geoArr = GeoArray(*geoArr_initArgs)
assert self._dem.shape[:2]==self.arr.shape[:2]
self._dem = geoArr
self._dem.nodata = DEF_D.get_outFillZeroSaturated(self._dem.dtype)[0]
self._dem.gt = self.arr.gt
self._dem.prj = self.arr.prj
else:
del self.dem
@dem.deleter
def dem(self):
self._dem = None
@property
def pathGen(self):
"""
Returns the path generator object for generating file pathes belonging to the GMS object.
"""
if self._pathGen and self._pathGen.proc_level==self.proc_level:
return self._pathGen
else:
self._pathGen = PG.path_generator(self.__dict__.copy(), MGRS_info=self.MGRS_info)
return self._pathGen
@pathGen.setter
def pathGen(self, pathGen):
assert isinstance(pathGen, PG.path_generator), 'GMS_object.pathGen can only be set to an instance of ' \
'path_generator. Got %s.' %type(pathGen)
self._pathGen = pathGen
@pathGen.deleter
def pathGen(self):
self._pathGen = None
@property
def ac_options(self):
"""
......@@ -614,50 +317,6 @@ class GMS_object(object):
return self._ac_options
@property
def ac_errors(self):
"""Returns an instance of GeoArray containing error information calculated by the atmospheric correction.
:return:
"""
return self._ac_errors # FIXME should give a warning if None
@ac_errors.setter
def ac_errors(self, *geoArr_initArgs):
if geoArr_initArgs[0] is not None:
errArr = GeoArray(*geoArr_initArgs)
assert errArr.shape == self.arr.shape, "The 'ac_errors' GeoArray can only be instanced with an array of " \
"the same dimensions like GMS_obj.arr. Got %s." %str(errArr.shape)
if errArr._nodata is None:
errArr.nodata = DEF_D.get_outFillZeroSaturated(errArr.dtype)[0]
errArr.gt = self.arr.gt
errArr.prj = self.arr.prj
errArr.bandnames = self.LBA2bandnames(self.LayerBandsAssignment)
self._ac_errors = errArr
else:
del self.ac_errors
@ac_errors.deleter
def ac_errors(self):
self._ac_errors = None
@property
def subset(self):
return [self.arr_shape, self.arr_pos]
@staticmethod
def LBA2bandnames(LayerBandsAssignment):
"""Convert LayerbandsAssignment from format ['1','2',...] to bandnames like this: [B01, .., B8A,]."""
return ['B%s' % i if len(i) == 2 else 'B0%s' % i for i in LayerBandsAssignment]
def get_copied_dict_and_props(self, remove_privates=False):
# type: (bool) -> dict
"""Returns a copy of the current object dictionary including the current values of all object properties."""
......@@ -742,67 +401,6 @@ class GMS_object(object):
return copy.copy(self)
def from_tiles(self, list_GMS_tiles):
# type: (list) -> GMS_object
"""Merge separate GMS objects with different spatial coverage but belonging to the same scene-ID to ONE GMS object.
:param list_GMS_tiles: <list> of GMS objects that have been created by cut_GMS_obj_into_blocks()
"""
if 'IMapUnorderedIterator' in str(type(list_GMS_tiles)):
list_GMS_tiles = list(list_GMS_tiles)
# copy all attributes except of array attributes
tile1 = list_GMS_tiles[0]
[setattr(self, i, getattr(tile1, i)) for i in tile1.__dict__ \
if not callable(getattr(tile1, i)) and not isinstance(getattr(tile1, i), (np.ndarray, GeoArray))]
# MERGE ARRAY-ATTRIBUTES
list_arraynames = [i for i in tile1.__dict__ if not callable(getattr(tile1, i)) and \
isinstance(getattr(tile1, i), (np.ndarray, GeoArray))]
list_arraynames = ['_arr'] + [i for i in list_arraynames if i !='_arr'] # list must start with _arr, otherwise setters will not work
for arrname in list_arraynames:
samplearray = getattr(tile1, arrname)
assert isinstance(samplearray, (np.ndarray, GeoArray)), \
'Received a %s object for attribute %s. Expected a numpy array or an instance of GeoArray.'\
% (type(samplearray), arrname)
is_3d = samplearray.ndim == 3
bands = (samplearray.shape[2],) if is_3d else () # dynamic -> works for arr, cld_arr,...
target_shape = tuple(self.shape_fullArr[:2]) + bands
target_dtype = samplearray.dtype
merged_array = self._numba_array_merger(list_GMS_tiles, arrname, target_shape, target_dtype)
setattr(self, arrname if not arrname.startswith('_') else arrname[1:], merged_array) # use setters if possible
# NOTE: this asserts that each attribute starting with '_' has also a property with a setter!
# UPDATE ARRAY-DEPENDENT ATTRIBUTES
self.arr_shape = 'cube'
self.arr_pos = None
# update MetaObj attributes
self.meta_odict.update({
'samples':self.arr.cols, 'lines':self.arr.rows, 'bands':self.arr.bands,
'map info':geotransform2mapinfo(self.arr.gt, self.arr.prj), 'coordinate system string':self.arr.prj,})
# calculate data_corners_imXY (mask_nodata is always an array here because get_mapPos always returns an array)
corners_imYX = calc_FullDataset_corner_positions(
self.mask_nodata, assert_four_corners=False, algorithm='shapely')
self.trueDataCornerPos = [(YX[1], YX[0]) for YX in corners_imYX] # [UL, UR, LL, LR]
# calculate trueDataCornerLonLat
data_corners_LatLon = pixelToLatLon(self.trueDataCornerPos, geotransform=self.arr.gt, projection=self.arr.prj)
self.trueDataCornerLonLat = [(YX[1], YX[0]) for YX in data_corners_LatLon]
# calculate trueDataCornerUTM
data_corners_utmYX = pixelToMapYX(self.trueDataCornerPos, geotransform = self.arr.gt, projection = self.arr.prj) # FIXME asserts gt in UTM coordinates
self.trueDataCornerUTM = [(YX[1], YX[0]) for YX in data_corners_utmYX]
return copy.copy(self)
def from_sensor_subsystems(self, list_GMS_objs):
# type: (list) -> GMS_object
"""Merge separate GMS objects belonging to the same scene-ID into ONE GMS object.
......@@ -996,6 +594,67 @@ class GMS_object(object):
return copy.copy(self)
def from_tiles(self, list_GMS_tiles):
# type: (list) -> self
"""Merge separate GMS objects with different spatial coverage but belonging to the same scene-ID to ONE GMS object.
:param list_GMS_tiles: <list> of GMS objects that have been created by cut_GMS_obj_into_blocks()
"""