Commit 405fe519 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Fixed exceptions during parsing of most recent Sentinel-2A metadata XMLs.

parent 98c1894b
Pipeline #1673 passed with stage
in 10 minutes and 22 seconds
......@@ -197,7 +197,7 @@ def get_gms_argparser():
'downloaded satellite image archives!',
help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
"image archives! (Sub-Parser).",
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_constraints = subparsers.add_parser(
'constraints', parents=[general_opts_parser],
......
......@@ -1218,41 +1218,54 @@ class METADATA(object):
def Read_Sentinel2A_xmls(self):
"""Read metadata from Sentinel-2A generic xml and granule xml"""
# self.default_attr()
assert self.SceneID is not None and self.SceneID != -9999, "Read_Sentinel2A_xmls(): Missing scene ID. "
# query database to get entityid
assert self.SceneID and self.SceneID != -9999, "Read_Sentinel2A_xmls(): Missing scene ID. "
res = DB_T.get_info_from_postgreSQLdb(CFG.conn_database, 'scenes', ['entityid'], {'id': self.SceneID})
assert len(res) != 0, \
"Invalid SceneID given - no corresponding scene with the ID=%s found in database.\n" % self.SceneID
assert len(res) == 1, "Error in database. The sceneid %s exists more than once. \n" % self.SceneID
S2AgranuleID = res[0][0]
#################
# get XML roots #
#################
if os.path.isdir(self.FolderOrArchive):
# metadata has to be read from folder
#####################################
# get xml_Scene_root (contains scene metadata)
glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'S2A*.xml'))
if not glob_res:
# new stype packaging
# new style packaging
glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'MTD*.xml'))
assert len(glob_res) > 0, 'No S2A*.xml or MTD*.xml file can be found in %s/!' % self.FolderOrArchive
self.Metafile = glob_res[0]
xml_Scene_root = ET.parse(glob_res[0]).getroot() # xml_parser from file
# get xml_GR_root (contains granule metadata)
glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2AgranuleID + '/S2A*.xml'))
if not glob_res:
# new stype packaging
# new style packaging
glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2AgranuleID + '/MTD*.xml'))
assert len(glob_res) > 0, \
'No /GRANULE/<S2AgranuleID>/S2A*.xml or MTD*.xml file can be found in %s/!' % self.FolderOrArchive
self.Metafile = self.Metafile + ", " + glob_res[0]
xml_GR_root = ET.parse(glob_res).getroot() # xml_parser from file
else: # archive
else:
# metadata has to be read from within archive
#############################################
# get xml_Scene_root and xml_GR_root (contain scene and granule metadata)
try:
# old style packaging
xml_SC_str_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/S2A*.xml')
xml_GR_str_, Metafile_ = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
S2AgranuleID + '/S2A*.xml')
except AssertionError:
# new stype packaging
# new style packaging
xml_SC_str_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/MTD*.xml')
xml_GR_str_, Metafile_ = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
S2AgranuleID + '/MTD*.xml')
......@@ -1261,13 +1274,23 @@ class METADATA(object):
xml_GR_root = ET.fromstring(xml_GR_str_)
self.Metafile = self.Metafile + ", " + Metafile_
###################################
# EXTRACT METADATA FROM XML ROOTS #
###################################
# define Sentinel 2A metadata (hard coded)
self.Sensor = "MSI"
# extract metadata from xml_Scene_root
namespace = "https://psd-13.sentinel2.eo.esa.int/PSD/User_Product_Level-1C.xsd"
self.EntityID = xml_Scene_root.find(".//Datatake").attrib[
'datatakeIdentifier'] # FIXME tileID (Granule) oder scene ID???
# extract metadata from xml_Scene_root #
########################################
# get the namespace within the xml_Scene_root
m = re.match('{(.*)\}', xml_Scene_root.tag) # extract namespace from "{https://....}Level-1C_Tile_ID"
assert m, 'XML Namespace could not be identified within metadata XML file.'
namespace = m.group(1)
self.EntityID = \
xml_Scene_root.find(".//Datatake").attrib['datatakeIdentifier'] # FIXME tileID (Granule) oder scene ID???
self.Satellite = xml_Scene_root.find(".//SPACECRAFT_NAME").text
# AcqDate, AcqTime
......@@ -1292,8 +1315,13 @@ class METADATA(object):
self.ProcLCode = xml_Scene_root.find(".//PROCESSING_LEVEL").text
# extract metadata from xml_GR_root
namespace = "https://psd-12.sentinel2.eo.esa.int/PSD/S2_PDI_Level-1C_Tile_Metadata.xsd"
# extract metadata from xml_GR_root #
#####################################
# get the namespace within the xml_GR_root
m = re.match('{(.*)\}', xml_GR_root.tag) # extract namespace from "{https://....}Level-1C_Tile_ID"
assert m, 'XML Namespace could not be identified within metadata XML file.'
namespace = m.group(1) # e.g., "https://psd-12.sentinel2.eo.esa.int/PSD/S2_PDI_Level-1C_Tile_Metadata.xsd"
# set self.AcqDateTime as well as self.AcqDate and self.AcqTime
self.AcqDateTime = iso8601.parse_date(xml_GR_root.find(".//SENSING_TIME").text)
......@@ -1385,14 +1413,23 @@ class METADATA(object):
# Quality flags # FIXME does not work (at least with old data)
Quality_temp = (xml_Scene_root.find(".//Technical_Quality_Assessment"))
Quality_temp.find("./DEGRADED_ANC_DATA_PERCENTAGE").text
self.Quality.append(["DEGRADED_ANC_DATA_PERCENTAGE", Quality_temp.find("./DEGRADED_ANC_DATA_PERCENTAGE").text])
self.Quality.append(["DEGRADED_MSI_DATA_PERCENTAGE", Quality_temp.find("./DEGRADED_MSI_DATA_PERCENTAGE").text])
Quality_temp2 = xml_Scene_root.find(".//Quality_Inspections")
quality_flags = ["SENSOR_QUALITY_FLAG", "GEOMETRIC_QUALITY_FLAG", "GENERAL_QUALITY_FLAG",
"FORMAT_CORRECTNESS_FLAG", "RADIOMETRIC_QUALITY_FLAG"]
for ql in quality_flags:
self.Quality.append([ql, Quality_temp2.find("./" + ql).text])
try:
# layout example: <SENSOR_QUALITY_FLAG>PASSED</SENSOR_QUALITY_FLAG>
for ql in quality_flags:
self.Quality.append([ql, Quality_temp2.find("./" + ql).text])
except AttributeError:
# since ~11/2017 the quality checks layout in the XML has changed:
# layout example: <quality_check checkType="SENSOR_QUALITY">PASSED</quality_check>
elements = Quality_temp2.findall('quality_check')
checkTypeValDict = {ele.attrib['checkType']: ele.text for ele in elements}
for ql in quality_flags:
self.Quality.append([ql, checkTypeValDict[ql.split('_FLAG')[0]]])
self.FOV = get_FieldOfView(self.GMS_identifier)
self.orbitParams = get_orbit_params(self.GMS_identifier)
......
......@@ -286,10 +286,11 @@ summary_testResults, summary_errors, summary_failures, summary_skipped, jobstatu
class Test_in_normal_mode(unittest.TestCase):
def setUp(self):
# self.job_id = 26186740 # Testjob Landsat-8
self.job_id = 26186906 # Bug Input Validator
# self.job_id = 26186906 # Bug Input Validator
self.job_id = 26186925 # 1 Sentinel-2A, Bug NoneType' object has no attribute 'find'
self.PC = process_controller(self.job_id, **dict(is_test=False, parallelization_level='scenes', db_host='geoms',
delete_old_output=True,))
delete_old_output=True, disable_exception_handler=True))
def test(self):
self.PC.run_all_processors()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment