Commit 657a3323 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Added tests for exception handler. Unified L1A_object inputs.

Former-commit-id: 6de517bc
Former-commit-id: f1f24e11
parent 4aa141dd
...@@ -34,7 +34,7 @@ class L1A_object(GMS_object): ...@@ -34,7 +34,7 @@ class L1A_object(GMS_object):
"""Features input reader and raster-/metadata homogenization.""" """Features input reader and raster-/metadata homogenization."""
def __init__(self, image_type='', satellite='', sensor='', subsystem='', sensormode='', acq_datetime=None, def __init__(self, image_type='', satellite='', sensor='', subsystem='', sensormode='', acq_datetime=None,
entity_ID='', scene_ID=-9999, filename='', dataset_ID=-9999): entity_ID='', scene_ID=-9999, filename='', dataset_ID=-9999, **kwargs):
""":param : instance of gms_object.GMS_object or None """:param : instance of gms_object.GMS_object or None
""" """
# TODO docstring # TODO docstring
......
...@@ -499,7 +499,7 @@ class Usecase: ...@@ -499,7 +499,7 @@ class Usecase:
scenes.acquisitiondate, scenes.acquisitiondate,
scenes.entityid, scenes.entityid,
scenes.filename, scenes.filename,
COALESCE(scenes_proc.proc_level::text, 'L0A') AS proc_level, COALESCE(scenes_proc.proc_level::text, 'L1A') AS proc_level,
datasets.image_type, datasets.image_type,
satellites.name AS satellite, satellites.name AS satellite,
sensors.name AS sensor, sensors.name AS sensor,
...@@ -519,7 +519,7 @@ class Usecase: ...@@ -519,7 +519,7 @@ class Usecase:
ds = OrderedDict() ds = OrderedDict()
ds["proc_level"] = row["proc_level"] ds["proc_level"] = row["proc_level"]
ds["scene_ID"] = row["sceneid"] ds["scene_ID"] = row["sceneid"]
ds["datasetid"] = row["datasetid"] ds["dataset_ID"] = row["datasetid"]
ds["image_type"] = row["image_type"] ds["image_type"] = row["image_type"]
ds["satellite"] = row["satellite"] ds["satellite"] = row["satellite"]
ds["sensor"] = row["sensor"] ds["sensor"] = row["sensor"]
......
...@@ -189,11 +189,10 @@ class ExceptionHandler(object): ...@@ -189,11 +189,10 @@ class ExceptionHandler(object):
@staticmethod @staticmethod
def update_progress_failed(failed_Obj): def update_progress_failed(failed_Obj):
"""Update statistics column in jobs table of postgreSQL database.""" """Update statistics column in jobs table of postgreSQL database."""
if not failed_Obj.subsystem or failed_Obj.subsystem in ['VNIR1', 'S2A10']: DB_T.increment_decrement_arrayCol_in_postgreSQLdb(
DB_T.increment_decrement_arrayCol_in_postgreSQLdb( CFG.job.conn_database, 'jobs', 'statistics', cond_dict={'id': CFG.job.ID},
CFG.job.conn_database, 'jobs', 'statistics', cond_dict={'id': CFG.job.ID}, idx_val2decrement=db_jobs_statistics_def[failed_Obj.proc_level],
idx_val2decrement=db_jobs_statistics_def[failed_Obj.proc_level], idx_val2increment=db_jobs_statistics_def['FAILED'])
idx_val2increment=db_jobs_statistics_def['FAILED'])
def handle_failed(self): def handle_failed(self):
_, exc_val, exc_tb = self.exc_details _, exc_val, exc_tb = self.exc_details
...@@ -225,8 +224,8 @@ class ExceptionHandler(object): ...@@ -225,8 +224,8 @@ class ExceptionHandler(object):
return failed_Obj return failed_Obj
def log_uncaught_exceptions(GMS_mapper): def log_uncaught_exceptions(GMS_mapper, logger=None):
exc_handler = ExceptionHandler() exc_handler = ExceptionHandler(logger=logger)
return exc_handler.log_uncaught_exceptions(GMS_mapper) return exc_handler.log_uncaught_exceptions(GMS_mapper)
......
...@@ -19,16 +19,7 @@ __author__ = 'Daniel Scheffler' ...@@ -19,16 +19,7 @@ __author__ = 'Daniel Scheffler'
def L1A_map(dataset_dict): # map (scene-wise parallelization) def L1A_map(dataset_dict): # map (scene-wise parallelization)
# type: (dict) -> L1A_P.L1A_object # type: (dict) -> L1A_P.L1A_object
L1A_obj = L1A_P.L1A_object(image_type='RSD', L1A_obj = L1A_P.L1A_object(**dataset_dict)
satellite=dataset_dict['satellite'],
sensor=dataset_dict['sensor'],
subsystem=dataset_dict['subsystem'],
sensormode=dataset_dict['sensormode'],
acq_datetime=dataset_dict['acq_datetime'],
entity_ID=dataset_dict['entity_ID'],
scene_ID=dataset_dict['scene_ID'],
filename=dataset_dict['filename'],
dataset_ID=dataset_dict['datasetid'])
L1A_obj.import_rasterdata() L1A_obj.import_rasterdata()
L1A_obj.import_metadata(v=False) L1A_obj.import_metadata(v=False)
L1A_obj.validate_GeoTransProj_GeoAlign() # sets self.GeoTransProj_ok and self.GeoAlign_ok L1A_obj.validate_GeoTransProj_GeoAlign() # sets self.GeoTransProj_ok and self.GeoAlign_ok
...@@ -53,16 +44,7 @@ def L1A_map(dataset_dict): # map (scene-wise parallelization) ...@@ -53,16 +44,7 @@ def L1A_map(dataset_dict): # map (scene-wise parallelization)
def L1A_map_1(dataset_dict, block_size=None): # map (scene-wise parallelization) def L1A_map_1(dataset_dict, block_size=None): # map (scene-wise parallelization)
# type: (dict) -> List[L1A_P.L1A_object] # type: (dict) -> List[L1A_P.L1A_object]
L1A_obj = L1A_P.L1A_object(image_type='RSD', L1A_obj = L1A_P.L1A_object(**dataset_dict)
satellite=dataset_dict['satellite'],
sensor=dataset_dict['sensor'],
subsystem=dataset_dict['subsystem'],
sensormode=dataset_dict['sensormode'],
acq_datetime=dataset_dict['acq_datetime'],
entity_ID=dataset_dict['entity_ID'],
scene_ID=dataset_dict['scene_ID'],
filename=dataset_dict['filename'],
dataset_ID=dataset_dict['datasetid'])
L1A_obj.import_rasterdata() L1A_obj.import_rasterdata()
L1A_obj.import_metadata(v=False) L1A_obj.import_metadata(v=False)
L1A_obj.validate_GeoTransProj_GeoAlign() # sets self.GeoTransProj_ok and self.GeoAlign_ok L1A_obj.validate_GeoTransProj_GeoAlign() # sets self.GeoTransProj_ok and self.GeoAlign_ok
......
...@@ -210,7 +210,7 @@ class process_controller(object): ...@@ -210,7 +210,7 @@ class process_controller(object):
Sensor=dataset['sensor'], Sensor=dataset['sensor'],
Subsystem=dataset['subsystem'], Subsystem=dataset['subsystem'],
proc_level=ProcL, # must be respected because LBA changes after atm. Corr. proc_level=ProcL, # must be respected because LBA changes after atm. Corr.
dataset_ID=dataset['datasetid'], dataset_ID=dataset['dataset_ID'],
logger=None), nBands=(1 if dataset['sensormode'] == 'P' else None)) logger=None), nBands=(1 if dataset['sensormode'] == 'P' else None))
# check if the LayerBandsAssignment of the written dataset on disk equals the # check if the LayerBandsAssignment of the written dataset on disk equals the
......
...@@ -5,28 +5,133 @@ ...@@ -5,28 +5,133 @@
test_exception_handler test_exception_handler
---------------------- ----------------------
Tests for gms_preprocessing.misc.exception_handler Tests for gms_preprocessing.misc.exception_handler.ExceptionHandler
""" """
import unittest import unittest
from gms_preprocessing import process_controller from gms_preprocessing import process_controller
from gms_preprocessing.misc.exception_handler import ExceptionHandler, log_uncaught_exceptions from gms_preprocessing.misc.exception_handler import log_uncaught_exceptions
from gms_preprocessing.algorithms.L1A_P import L1A_object
from gms_preprocessing.misc.database_tools import get_info_from_postgreSQLdb
class Test_ExceptionHandler(unittest.TestCase): class BaseTest_ExceptionHandler:
"""Tests class for gms_preprocessing.misc.exception_handler.ExceptionHandler.""" """Test class for gms_preprocessing.misc.exception_handler.ExceptionHandler."""
class Test_ExceptionHandler(unittest.TestCase):
PC = None # default
@classmethod def get_process_controller(self, jobID):
def setUpClass(cls): self.PC = process_controller(jobID, parallelization_level='scenes', db_host='localhost',
cls.PC = process_controller(jobID, parallelization_level='scenes', db_host='geoms', delete_old_output=True, job_config_kwargs=dict(is_test=True, log_level='DEBUG'))
job_config_kwargs=config)
# update attributes of DB_job_record and related DB entry
self.PC.DB_job_record.reset_job_progress()
@log_uncaught_exceptions [ds.update({'proc_level': None}) for ds in self.PC.usecase.data_list]
def dummy_gms_mapper_success(self, dummy_GMSobj):
return dummy_GMSobj
@log_uncaught_exceptions self.PC.job.status = 'running'
def dummy_gms_mapper_fail(self): self.PC.update_DB_job_record() # TODO implement that into job.status.setter
raise RuntimeError()
def tearDown(self):
self.PC.DB_job_record.reset_job_progress()
@staticmethod
@log_uncaught_exceptions
def dummy_L1A_mapper_success(dummy_GMSobj):
return L1A_object(**dummy_GMSobj)
@staticmethod
@log_uncaught_exceptions
def dummy_gms_mapper_fail(dummy_GMSobj):
raise RuntimeError('TestException raised by dummy_gms_mapper_fail()')
def get_current_progress_stats(self):
return get_info_from_postgreSQLdb(self.PC.job.conn_database, 'jobs', 'statistics',
cond_dict=dict(id=self.PC.job.ID))[0][0]
def is_sceneid_in_failedIDs(self, sceneid):
return sceneid in get_info_from_postgreSQLdb(self.PC.job.conn_database, 'jobs', ['failed_sceneids'],
{'id': self.PC.job.ID})[0][0]
class Test_ExceptionHandler_NoSubsystems(BaseTest_ExceptionHandler.Test_ExceptionHandler):
def setUp(self):
super().get_process_controller(26186261) # Landsat-8 Coll. Data
def test_L1A_mapper_success(self):
"""Check correctness of progress stats if scene succeeds."""
self.dummy_L1A_mapper_success(self.PC.usecase.data_list[0])
self.assertEqual(self.get_current_progress_stats(), [0, 0, 1, 0, 0, 0, 0, 0, 0])
def test_gms_mapper_fail(self):
"""Check correctness of progress stats if scene fails."""
outObj = self.dummy_gms_mapper_fail(self.PC.usecase.data_list[0])
self.assertTrue(self.is_sceneid_in_failedIDs(outObj.scene_ID))
self.assertEqual(self.get_current_progress_stats(), [0, 0, 0, 0, 0, 0, 0, 0, 1])
def test_gms_mapper_fail_excH_off(self):
"""Check if exception handler raises exceptions of CFG.Job.disable_exception_handler = True."""
self.PC.job.disable_exception_handler = True
self.assertRaises(RuntimeError, self.dummy_gms_mapper_fail, self.PC.usecase.data_list[0])
class Test_ExceptionHandler_Subsystems(BaseTest_ExceptionHandler.Test_ExceptionHandler):
""""""
def setUp(self):
super().get_process_controller(26186268) # Sentinel-2
def test_L1A_mapper_success(self):
"""Check correctness of progress stats if all subsystems succeed."""
for subObj in self.PC.usecase.data_list:
self.dummy_L1A_mapper_success(subObj)
# validate that stats are only updated by first subsystem and keep the same value
self.assertEqual(self.get_current_progress_stats(), [0, 0, 1, 0, 0, 0, 0, 0, 0])
# validate that stats only show ONE scene instead of the number of subsystems
self.assertEqual(self.get_current_progress_stats(), [0, 0, 1, 0, 0, 0, 0, 0, 0])
def test_gms_mapper_fail_firstSS(self):
"""Check correctness of progress stats if the first subsystem first fails and another one succeeds."""
for i, subObj in enumerate(self.PC.usecase.data_list):
if subObj['subsystem'] == 'S2A10':
outObj = self.dummy_gms_mapper_fail(subObj)
# check that the scene ID of the failed subsystem has been added to failed_sceneids db column
self.assertTrue(self.is_sceneid_in_failedIDs(outObj.scene_ID))
# check that the scene has been marked as failed in progress stats
self.assertEqual(self.get_current_progress_stats(), [0, 0, 0, 0, 0, 0, 0, 0, 1])
else:
self.dummy_L1A_mapper_success(subObj)
# check that the scene keeps marked as failed
self.assertEqual(self.get_current_progress_stats(), [0, 0, 0, 0, 0, 0, 0, 0, 1])
@unittest.SkipTest
def test_gms_mapper_fail_secondSS(self):
"""Check correctness of progress stats if a first subsystem succeeds and then another one fails.
NOTE: This happens quite rarely because if a mapper fails for a subsystem, it usualy fails for the first
subsystem it receives."""
# TODO: This test currently fails because earlier subsystem updates stats from
# TODO: [0, 1, 0, 0, 0, 0, 0, 0, 0] to [0, 0, 1, 0, 0, 0, 0, 0, 0] and later one from
# TODO: [0, 0, 1, 0, 0, 0, 0, 0, 0] to [0, -1, 1, 0, 0, 0, 0, 0, 1]
# TODO: This is known bug.
for i, subObj in enumerate(self.PC.usecase.data_list):
if subObj['subsystem'] == 'S2A10':
self.dummy_L1A_mapper_success(subObj)
# progress stats must be incremented
self.assertEqual(self.get_current_progress_stats(), [0, 0, 1, 0, 0, 0, 0, 0, 0])
else:
outObj = self.dummy_gms_mapper_fail(subObj)
# scene must be added to failed_scenes column
self.assertTrue(self.is_sceneid_in_failedIDs(outObj.scene_ID))
# progress of whole scene must be marked as failed
self.assertEqual(self.get_current_progress_stats(), [0, 0, 0, 0, 0, 0, 0, 0, 1])
self.assertEqual(self.get_current_progress_stats(), [0, 0, 0, 0, 0, 0, 0, 0, 1])
...@@ -154,7 +154,7 @@ class BaseTestCases: ...@@ -154,7 +154,7 @@ class BaseTestCases:
class Test_Landsat5_PreCollectionData(BaseTestCases.TestAll): class Test_Landsat5_PreCollectionData(BaseTestCases.TestAll):
""" """
Parametrized testclass. Tests the level-processes on a Landsat-5 TM scene (pre-collection data). Parametrized testclass. Tests the level-processes on a Landsat-5 TM scene (pre-collection data).
More information on the dataset will be outputted after the tests-classes are executed. More information on the dataset will be output after the tests-classes are executed.
""" """
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
...@@ -173,7 +173,7 @@ class Test_Landsat5_PreCollectionData(BaseTestCases.TestAll): ...@@ -173,7 +173,7 @@ class Test_Landsat5_PreCollectionData(BaseTestCases.TestAll):
class Test_Landsat7_SLC_on_PreCollectionData(BaseTestCases.TestAll): class Test_Landsat7_SLC_on_PreCollectionData(BaseTestCases.TestAll):
""" """
Parametrized testclass. Tests the level-processes on a Landsat-7 ETM+_SLC_ON scene (pre-collection data). Parametrized testclass. Tests the level-processes on a Landsat-7 ETM+_SLC_ON scene (pre-collection data).
More information on the dataset will be outputted after after the tests-classes are executed. More information on the dataset will be output after after the tests-classes are executed.
""" """
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
...@@ -183,7 +183,7 @@ class Test_Landsat7_SLC_on_PreCollectionData(BaseTestCases.TestAll): ...@@ -183,7 +183,7 @@ class Test_Landsat7_SLC_on_PreCollectionData(BaseTestCases.TestAll):
class Test_Landsat7_SLC_off_PreCollectionData(BaseTestCases.TestAll): class Test_Landsat7_SLC_off_PreCollectionData(BaseTestCases.TestAll):
""" """
Parametrized testclass. Tests the level-processes on a Landsat-7 ETM+_SLC_OFF scene (pre-collection data). Parametrized testclass. Tests the level-processes on a Landsat-7 ETM+_SLC_OFF scene (pre-collection data).
More information on the dataset will be outputted after the tests-classes are executed. More information on the dataset will be output after the tests-classes are executed.
""" """
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
...@@ -203,7 +203,7 @@ class Test_Landsat7_SLC_off_PreCollectionData(BaseTestCases.TestAll): ...@@ -203,7 +203,7 @@ class Test_Landsat7_SLC_off_PreCollectionData(BaseTestCases.TestAll):
class Test_Landsat8_PreCollectionData(BaseTestCases.TestAll): class Test_Landsat8_PreCollectionData(BaseTestCases.TestAll):
""" """
Parametrized testclass. Tests the level-processes on a Landsat-8 OLI_TIRS scene (pre-collection data). Parametrized testclass. Tests the level-processes on a Landsat-8 OLI_TIRS scene (pre-collection data).
More information on the dataset will be outputted after the tests-classes are executed. More information on the dataset will be output after the tests-classes are executed.
""" """
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
...@@ -213,27 +213,27 @@ class Test_Landsat8_PreCollectionData(BaseTestCases.TestAll): ...@@ -213,27 +213,27 @@ class Test_Landsat8_PreCollectionData(BaseTestCases.TestAll):
class Test_Landsat8_CollectionData(BaseTestCases.TestAll): class Test_Landsat8_CollectionData(BaseTestCases.TestAll):
""" """
Parametrized testclass. Tests the level-processes on a Landsat-8 OLI_TIRS scene (collection data). Parametrized testclass. Tests the level-processes on a Landsat-8 OLI_TIRS scene (collection data).
More information on the dataset will be outputted after the tests-classes are executed. More information on the dataset will be output after the tests-classes are executed.
""" """
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.create_job(26186261, job_config_kwargs) cls.create_job(26186261, job_config_kwargs)
class Test_Sentinel2A_CollectionData(BaseTestCases.TestAll): class Test_Sentinel2A_SingleGranule(BaseTestCases.TestAll):
""" """
Parametrized testclass. Tests the level-processes on a Sentinel-2A MSI scene (pre-collection data). Parametrized testclass. Tests the level-processes on a Sentinel-2A MSI scene (1 granule in archive: > 2017).
More information on the dataset will be outputted after the tests-classes are executed. More information on the dataset will be output after the tests-classes are executed.
""" """
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.create_job(26186268, job_config_kwargs) cls.create_job(26186268, job_config_kwargs)
class Test_Sentinel2A_PreCollectionData(BaseTestCases.TestAll): class Test_Sentinel2A_MultiGranule(BaseTestCases.TestAll):
""" """
Parametrized testclass. Tests the level-processes on a Sentinel-2A MSI scene (collection data). Parametrized testclass. Tests the level-processes on a Sentinel-2A MSI scene (multiple granules in archive: < 2017).
More information on the dataset will be outputted after the tests-classes are executed. More information on the dataset will be output after the tests-classes are executed.
""" """
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment