Commit d17c62f6 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

- Fixed missing cloud classifier exceptions

- added exceptions module
- FMASK-warnings are now muted during nosetests
- added attribute 'cloud_masking_algorithm' to GMS_object
- removed deprecated attribute 'path_ac_options' from GMS_config.job
- cleaned deprecated entries in .gitignore
- added environment checks for not pip-installable dependencies
parent 03b528cb
Pipeline #558 passed with stages
in 6 minutes and 28 seconds
......@@ -67,36 +67,23 @@ target/
.idea/
BAK/
OLD/
/geomultisens/database/cloud_classifier/
/tests/data/output_mgrs_tiles/
/tests/data/output_scenes/
/tests/data/sampledata/
/geomultisens/database/metadata/
/geomultisens/database/tiling_system/
/geomultisens/database/processed_data/
/geomultisens/database/earth_sun_distance/horizon_web_interface.cgi_files/
/geomultisens/database/earth_sun_distance/horizon_web_interface.cgi.html
/geomultisens/database/earth_sun_distance/README
/geomultisens/database/solar_irradiance/Solar_irradiance_Thuillier_2002.xls
/geomultisens/database/solar_irradiance/Thuillier_2003_solar_irradiance_400_2400.pdf
/geomultisens/database/solar_irradiance/SUNp1fontenla.asc
/geomultisens/database/solar_irradiance/SOLAR.txt
/geomultisens/logs/
geomultisens/database/cloud_classifier/
tests/data/output_mgrs_tiles/
tests/data/output_scenes/
tests/data/sampledata/
geomultisens/database/metadata/
geomultisens/database/tiling_system/
geomultisens/database/processed_data/
geomultisens/database/earth_sun_distance/horizon_web_interface.cgi_files/
geomultisens/database/earth_sun_distance/horizon_web_interface.cgi.html
geomultisens/database/earth_sun_distance/README
geomultisens/database/solar_irradiance/Solar_irradiance_Thuillier_2002.xls
geomultisens/database/solar_irradiance/Thuillier_2003_solar_irradiance_400_2400.pdf
geomultisens/database/solar_irradiance/SUNp1fontenla.asc
geomultisens/database/solar_irradiance/SOLAR.txt
geomultisens/logs/
.ipynb_checkpoints/
io/envi_io.py
sandbox/ENVIcompressed.bsq
sandbox/ENVIcompressed.hdr
sandbox/ENVIcompressed.worked.bsq
sandbox/ENVIcompressed.worked.hdr
sandbox/ENVIselfcompressed.bsq
sandbox/Landsat-8__OLI_TIRS__LC81930242016103LGN00__33UVT_L2C_selfcompressed.bsq
sandbox/Landsat-8__OLI_TIRS__LC81930242016103LGN00__33UVT_L2C_selfcompressed.hdr
sandbox/Landsat-8__OLI_TIRS__LC81930242016103LGN00__33UVT_L2C_selfcompressed.hdr.BAK
sandbox/Landsat-8__OLI_TIRS__LC81930242016119LGN00__32UPB_masks_L2C.bsq
sandbox/Landsat-8__OLI_TIRS__LC81930242016119LGN00__32UPB_masks_L2C.hdr
sandbox/Landsat-8__OLI_TIRS__LC81940242015091LGN00__32UNC_masks_L2C__compressed.bsq
sandbox/job_logs/
sandbox/meta_validation/
......@@ -23,13 +23,10 @@ import fmask
from ..misc.helper_functions import convert_absPathArchive_to_GDALvsiPath, subcall_with_output
from ..misc.definition_dicts import get_mask_classdefinition
from ..misc.database_tools import get_info_from_postgreSQLdb
from ..misc.exceptions import FmaskError, FmaskWarning
from geoarray import GeoArray
class FmaskError(RuntimeError):
"""An error within the Fmask wrapper of GeoMultiSens."""
class _FMASK_Runner(object):
"""The FMASK runner base class (not to be called directly)."""
......@@ -148,7 +145,7 @@ class _FMASK_Runner(object):
self.cloud_mask.legend = \
get_mask_classdefinition('mask_clouds', self.satellite)
else: # use default FMASK legend
warnings.warn('GMS configuration not available. Using default cloud mask legend.')
warnings.warn('GMS configuration not available. Using default cloud mask legend.', FmaskWarning)
self.cloud_mask.legend = \
{'No Data': 0, 'Clear': 1, 'Cloud': 2, 'Shadow': 3, 'Snow': 4, 'Water': 5}
......
......@@ -201,7 +201,6 @@ class Job:
self.path_SRFs = absP(query_cfg('path_SRFs'))
self.path_cloud_classif = query_cfg('path_cloud_classif')
self.path_solar_irr = absP(query_cfg('path_solar_irr'))
self.path_ac_options = absP(query_cfg('path_ac_options'))
self.path_ac_tables = query_cfg('path_ac_tables')
self.path_SNR_models = absP(query_cfg('path_SNR_models'))
self.path_dem_proc_srtm_90m = query_cfg('path_dem_proc_srtm_90m')
......
# -*- coding: utf-8 -*-
__author__='Daniel Scheffler'
import os
try:
from osgeo import gdal
except ImportError:
import gdal
from ..config import GMS_config as CFG
from .SpatialIndexMediator import SpatialIndexMediatorServer
from .exceptions import GMSEnvironmentError, MissingNonPipLibraryWarning
def check_ports():
......@@ -29,12 +37,46 @@ def check_dependencies(logger=None):
_log_or_print('Spatial Index Mediator Server started successfully.', logger.info)
if not SpatIdxSrv.is_running:
_log_or_print('Attempt to start Spatial Index Mediator Server failed.', logger.info)
msg = 'Attempt to start Spatial Index Mediator Server failed.'
_log_or_print(repr(GMSEnvironmentError(msg)), logger.error)
# check for not pip-installable packages
# fmask # conda install -c conda-forge python-fmask
try:
import fmask
except ImportError:
if 'FMASK' in list(CFG.job.cloud_masking_algorithm.values()):
msg = "FMASK library is not installed because it is not pip-installable and must be installed " \
"manually, e.g. for Anaconda by running 'conda install -c conda-forge python-fmask'!"
_log_or_print(repr(MissingNonPipLibraryWarning(msg)), logger.warning)
# 'pyhdf', # conda install --yes -c conda-forge pyhdf
try:
import pyhdf
except ImportError:
if gdal.GetDriverByName('HDF4') is None:
msg = "The library 'pyhdf' is missing and the HDF4 driver of GDAL is not available. ASTER data cannot be " \
"proceessed! For Anaconda, run 'conda install --yes -c conda-forge pyhdf' to fix that!"
_log_or_print(repr(MissingNonPipLibraryWarning(msg)), logger.warning)
# 'sicor', # pip install git+https://gitext.gfz-potsdam.de/hollstei/sicor.git
try:
import sicor
except ImportError:
msg = "The library 'sicor' has not been installed automatically because installation requires login " \
"credentials. See installation instrucions here: https://gitext.gfz-potsdam.de/hollstei/sicor"
_log_or_print(repr(MissingNonPipLibraryWarning(msg)), logger.warning)
def check_paths():
def check_paths(): # TODO to be completed
# check existance of database paths, etc.
# check existance of cloud classifier dill objects from PG.get_path_cloud_class_obj()
pass
if not os.path.exists(CFG.job.path_archive):
raise GMSEnvironmentError('The provider archive path does not exist at %s!' %CFG.job.path_archive)
def _log_or_print(msg, loggerLvl = None):
......
......@@ -150,3 +150,23 @@ def log_uncaught_exceptions(GMS_mapper):
return wrapped_GMS_mapper
def ignore_warning(warning_type):
"""A decorator to ignore a specific warning when executing a function.
:param warning_type: the type of the warning to ignore
"""
def _ignore_warning(func):
@functools.wraps(func)
def __ignore_warning(*args, **kwargs):
with warnings.catch_warnings(record=True) as ws:
# Catch all warnings of this type
warnings.simplefilter('always', warning_type)
# Execute the function
result = func(*args, **kwargs)
return result
return __ignore_warning
return _ignore_warning
# -*- coding: utf-8 -*-
__author__='Daniel Scheffler'
## ENVIRONMENT EXCEPTIONS
class GMSEnvironmentError(EnvironmentError):
"""Missing package, that has not been automatically installed because it is not pip-installable."""
class MissingNonPipLibraryWarning(UserWarning):
"""Missing package, that has not been automatically installed because it is not pip-installable."""
## CLOUD MASKING EXCEPTIONS
class FmaskWarning(UserWarning):
"""A warning within the Fmask wrapper of GeoMultiSens."""
class FmaskError(RuntimeError):
"""An error within the Fmask wrapper of GeoMultiSens."""
......@@ -51,8 +51,10 @@ class GMS_object(Dataset):
# get all attributes of base class "Dataset"
super(GMS_object, self).__init__()
# add EnMAP specific attributes
# add private attributes
self._dict_LayerOptTherm = None
self._cloud_masking_algorithm = None
self._meta_odict = None
self.job_ID = CFG.job.ID
#self.dataset_ID = int(DB_T.get_info_from_postgreSQLdb(CFG.job.conn_database, 'scenes', ['datasetid'],
......@@ -92,7 +94,8 @@ class GMS_object(Dataset):
self.path_logfile = self.pathGen.get_path_logfile()
self.pathGen = PG.path_generator(self.__dict__) # passes a logger in addition to previous attributes
self.path_archive = self.pathGen.get_local_archive_path_baseN()
self.path_cloud_class_obj = PG.get_path_cloud_class_obj(self.GMS_identifier,
self.path_cloud_class_obj = None if self.cloud_masking_algorithm in ['FMASK', 'SICOR'] else \
PG.get_path_cloud_class_obj(self.GMS_identifier,
get_all=True if CFG.job.bench_CLD_class else False)
if CFG.job.exec_mode=='Python':
self.path_InFilePreprocessor = os.path.join(self.ExtractedFolder, '%s%s_DN.bsq'
......@@ -286,6 +289,13 @@ class GMS_object(Dataset):
self._masks = None
@property
def cloud_masking_algorithm(self):
if not self._cloud_masking_algorithm:
self._cloud_masking_algorithm = CFG.job.cloud_masking_algorithm[self.satellite]
return self._cloud_masking_algorithm
@property
def ac_options(self):
"""
......
......@@ -21,3 +21,4 @@ iso8601
pyinstrument
geoalchemy2
sqlalchemy
# fmask # not pip installable
......@@ -14,6 +14,7 @@ requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill',
'pandas', 'numba', 'spectral', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
'py_tools_ds', 'geoarray', 'arosics'
# fmask # conda install -c conda-forge python-fmask
#'pyhdf', # conda install --yes -c conda-forge pyhdf
#'sicor', # pip install git+https://gitext.gfz-potsdam.de/hollstei/sicor.git
]
......
......@@ -13,7 +13,8 @@ import os
from geoarray import GeoArray
from geomultisens import __file__
from geomultisens.algorithms.cloud_masking import FMASK_Runner_Landsat, FMASK_Runner_Sentinel2
from geomultisens.algorithms.cloud_masking import FMASK_Runner_Landsat, FMASK_Runner_Sentinel2, FmaskWarning
from geomultisens.misc.exception_handler import ignore_warning
......@@ -42,31 +43,37 @@ testdata = dict(
class Test_FMASK_Runner_Landsat(unittest.TestCase):
@ignore_warning(FmaskWarning)
def test_Landsat5_collections_data(self):
#os.environ['RIOS_DFLT_DRIVER'] = 'VRT'
FMR = FMASK_Runner_Landsat(testdata['Landsat5_collections_data'], 'Landsat-5')
self.assertIsInstance(FMR.calc_cloudMask(), GeoArray)
@ignore_warning(FmaskWarning)
def test_Landsat5_precollections_data(self):
#os.environ['RIOS_DFLT_DRIVER'] = 'VRT'
FMR = FMASK_Runner_Landsat(testdata['Landsat5_precollections_data'], 'Landsat-5')
self.assertIsInstance(FMR.calc_cloudMask(), GeoArray)
@ignore_warning(FmaskWarning)
def test_Landsat7_collections_data(self):
#os.environ['RIOS_DFLT_DRIVER'] = 'VRT'
FMR = FMASK_Runner_Landsat(testdata['Landsat7_collections_data'], 'Landsat-7')
self.assertIsInstance(FMR.calc_cloudMask(), GeoArray)
@ignore_warning(FmaskWarning)
def test_Landsat7_precollections_data(self):
#os.environ['RIOS_DFLT_DRIVER'] = 'VRT'
FMR = FMASK_Runner_Landsat(testdata['Landsat7_precollections_data'], 'Landsat-7')
self.assertIsInstance(FMR.calc_cloudMask(), GeoArray)
@ignore_warning(FmaskWarning)
def test_Landsat8_collections_data(self):
#os.environ['RIOS_DFLT_DRIVER'] = 'VRT'
FMR = FMASK_Runner_Landsat(testdata['Landsat8_collections_data'], 'Landsat-8')
self.assertIsInstance(FMR.calc_cloudMask(), GeoArray)
@ignore_warning(FmaskWarning)
def test_Landsat8_precollections_data(self):
#os.environ['RIOS_DFLT_DRIVER'] = 'VRT'
FMR = FMASK_Runner_Landsat(testdata['Landsat8_precollections_data'], 'Landsat-8')
......@@ -77,12 +84,14 @@ class Test_FMASK_Runner_Landsat(unittest.TestCase):
class Test_FMASK_Runner_Sentinel2(unittest.TestCase):
@ignore_warning(FmaskWarning)
def test_Sentinel2A_old_style(self):
# FIXME scene ID or granule ID must be provided in case of multiple files within source archive
#os.environ['RIOS_DFLT_DRIVER'] = 'VRT'
FMR = FMASK_Runner_Sentinel2(testdata['Sentinel2A_old_style_data'], 'Sentinel-2A', scene_ID=None)
self.assertIsInstance(FMR.calc_cloudMask(), GeoArray)
@ignore_warning(FmaskWarning)
def test_Sentinel2A_new_style(self):
#os.environ['RIOS_DFLT_DRIVER'] = 'VRT'
FMR = FMASK_Runner_Sentinel2(testdata['Sentinel2A_new_style_data'], 'Sentinel-2A')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment