Commit 0497218d authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

added support for new-style Sentinel-2 data provided as single granules

 algorithms.L1A_PL1A_object:
 - archive_to_rasObj(): revised matching expression to match S2A files within archive

 algorithms.METADATA.METADATA:
 - Read_Sentinel2A_xmls(): revised matching expressions to find correct xml data

 io.Input_reader:
 - open_specific_file_within_archive(): added assertions, docstring and revised function style

- updated __version__
parent 28d3de70
......@@ -15,7 +15,7 @@ from . import config
from .processing.process_controller import process_controller
__version__ = '20170318.01'
__version__ = '20170320.01'
__author__ = 'Daniel Scheffler'
__all__ = ['algorithms',
'io',
......
......@@ -334,13 +334,13 @@ class L1A_object(GMS_object):
re.search("[\S]*_B[1-9][0-9]?[\S]*.TIF", File) if re.search('Landsat', self.satellite) else \
re.search("[0-9]*.tif", File) if re.search('RapidEye', self.satellite) else \
re.search("imagery.tif",File) if re.search('SPOT', self.satellite) else \
re.search("[\S]*.SAFE/GRANULE/%s/IMG_DATA/S2A_[\S]*_B[0-9][\S]*.jp2" %self.entity_ID,File)\
re.search("[\S]*.SAFE/GRANULE/%s/IMG_DATA/[\S]*_B[0-9][\S]*.jp2" %self.entity_ID,File)\
if re.search('Sentinel-2',self.satellite) else None
if search_res:
if re.search('Sentinel-2', self.satellite):
# add only those files that are corresponding to subsystem (e.g. S2A10: fullLBA = ['2','3','4','8'])
if 1 in [1 if re.search("S2A_[\S]*_B[0]?%s.jp2" %LBAn, os.path.basename(File)) else 0
if 1 in [1 if re.search("[\S]*_B[0]?%s.jp2" %LBAn, os.path.basename(File)) else 0
for LBAn in full_LayerBandsAssignment]:
image_files.append(File)
else:
......
......@@ -1184,27 +1184,43 @@ class METADATA(object):
if os.path.isdir(self.FolderOrArchive):
glob_res = glob.glob(os.path.join(self.FolderOrArchive,'S2A*.xml'))
assert len(glob_res)>0, 'No S2A*.xml file can be found in %s/!' %self.FolderOrArchive
if not glob_res:
# new stype packaging
glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'MTD*.xml'))
assert len(glob_res)>0, 'No S2A*.xml or MTD*.xml file can be found in %s/!' %self.FolderOrArchive
self.Metafile = glob_res[0]
xml_Scene_root = ET.parse(glob_res[0]).getroot() # xml_parser from file
glob_res = glob.glob(os.path.join(self.FolderOrArchive,'GRANULE/'+S2AgranuleID+'/S2A*.xml'))
assert len(glob_res)>0, 'No /GRANULE/<S2AgranuleID>/S2A*.xml file can be found in %s/!' %self.FolderOrArchive
if not glob_res:
# new stype packaging
glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2AgranuleID + '/MTD*.xml'))
assert len(glob_res)>0, \
'No /GRANULE/<S2AgranuleID>/S2A*.xml or MTD*.xml file can be found in %s/!' %self.FolderOrArchive
self.Metafile = self.Metafile + ", " + glob_res[0]
xml_GR_root = ET.parse(glob_res).getroot() # xml_parser from file
else: # archive
xml_str_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/S2A*.xml')
xml_Scene_root = ET.fromstring(xml_str_)
xml_str_, Metafile_ = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
S2AgranuleID +'/S2A*.xml')
xml_GR_root = ET.fromstring(xml_str_)
try:
# old style packaging
xml_SC_str_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/S2A*.xml')
xml_GR_str_, Metafile_ = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
S2AgranuleID +'/S2A*.xml')
except AssertionError:
# new stype packaging
xml_SC_str_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/MTD*.xml')
xml_GR_str_, Metafile_ = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
S2AgranuleID +'/MTD*.xml')
xml_Scene_root = ET.fromstring(xml_SC_str_)
xml_GR_root = ET.fromstring(xml_GR_str_)
self.Metafile = self.Metafile + ", " + Metafile_
# define Sentinel 2A metadata (hard coded)
self.Sensor = "MSI"
# extract metadata from xml_Scene_root # FIXME scene root is possibly not available in new S2 data provided as granules instead of scenes
# extract metadata from xml_Scene_root
namespace = "https://psd-13.sentinel2.eo.esa.int/PSD/User_Product_Level-1C.xsd"
self.EntityID = xml_Scene_root.find(".//Datatake").attrib['datatakeIdentifier'] #FIXME tileID (Granule) oder scene ID???
self.Satellite = xml_Scene_root.find(".//SPACECRAFT_NAME").text
......
......@@ -75,7 +75,7 @@ class Job:
self.CPUs = multiprocessing.cpu_count()
#self.CPUs = 1
self.allow_subMultiprocessing = True # allows multiprocessing within workers
self.disable_exception_handler = False # disables automatic handling of unexpected exceptions
self.disable_exception_handler = True # disables automatic handling of unexpected exceptions
self.profiling = False
# TODO add log level
......
......@@ -317,17 +317,30 @@ def Solar_Irradiance_reader(resol_nm = None, wvl_min_nm = None, wvl_max_nm = Non
return sol_irr
def open_specific_file_within_archive(path_archive,matching_expression, read_mode='r'):
def open_specific_file_within_archive(path_archive, matching_expression, read_mode='r'):
# type: (str, str, str) -> (str, str)
"""Finds a specific file within an archive using a given matching expression and returns its content as string.
:param path_archive: the file path of the archive
:param matching_expression: the matching expession to find the file within the archive
:param read_mode: the read mode used to open the archive (default: 'r')
:return:
"""
file_suffix = os.path.splitext(path_archive)[1][1:]
file_suffix = 'tar.gz' if path_archive.endswith('tar.gz') else file_suffix
assert file_suffix in ['zip','tar','gz','tgz', 'tar.gz'], '*.%s files are not supported.' %file_suffix
if file_suffix == 'zip':
archive = zipfile.ZipFile(path_archive,'r')
#[print(i) for i in archive.namelist()]
matching_files = fnmatch.filter(archive.namelist(), matching_expression)
count_matching_files = len(matching_files)
assert count_matching_files > 0, 'Matching expression matches no file. Please revise your expression!'
assert count_matching_files == 1, 'Matching expression matches more than 1 file. Please revise your expression!'
content_file = archive.read(matching_files[0])
filename_file = os.path.join(path_archive,matching_files[0])
else: #'tar','gz','tgz', 'tar.gz'
archive = tarfile.open(path_archive, 'r|gz') # open in stream mode is much faster than normal mode
count_matching_files = 0
......@@ -339,9 +352,11 @@ def open_specific_file_within_archive(path_archive,matching_expression, read_mod
count_matching_files += 1
assert count_matching_files > 0, 'Matching expression matches no file. Please revise your expression!'
assert count_matching_files == 1, 'Matching expression matches more than 1 file. Please revise your expression!'
archive.close()
content_file = content_file.decode('latin-1') if isinstance(content_file,bytes) and read_mode == 'r' \
else content_file # Python3
content_file = \
content_file.decode('latin-1') if isinstance(content_file,bytes) and read_mode == 'r' else content_file # Python3
return content_file, filename_file
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment