Commit 26034202 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Converted all regular expression strings to raw strings. Revised code style in metadata.py.


Signed-off-by: Daniel Scheffler's avatarDaniel Scheffler <danschef@gfz-potsdam.de>
parent f210e3f3
Pipeline #8003 passed with stage
in 17 minutes and 48 seconds
......@@ -565,6 +565,8 @@ Fixes and improvements:
* Added .coveragerc
* Modified coverage section in Makefile
* Removed pyhdf from automatically installed setup requirements
* Converted all regular expression strings to raw strings.
* Revised code style in metadata.py.
0.4.0 (2017-06-26)
......
......@@ -75,13 +75,13 @@ class L1A_object(GMS_object):
self.proc_status = proc_status or 'initialized' # if proc_status = 'running' is given by L1A_map
def import_rasterdata(self):
if re.search("ALOS", self.satellite, re.I):
if re.search(r"ALOS", self.satellite, re.I):
'''First 22 lines are nodata: = maybe due to an issue of the GDAL CEOS driver.
But: UL of metadata refers to [row =0, col=21]! So the imported GeoTransform is correct when
the first 21 columns are deleted.'''
self.archive_to_rasObj(self.path_archive, self.path_InFilePreprocessor,
subset=['block', [[None, None], [21, None]]])
elif re.search("Terra", self.satellite, re.I):
elif re.search(r"Terra", self.satellite, re.I):
self.ASTER_HDF_to_rasObj(self.path_archive, path_output=self.path_InFilePreprocessor)
else:
self.archive_to_rasObj(self.path_archive, path_output=self.path_InFilePreprocessor)
......@@ -113,23 +113,23 @@ class L1A_object(GMS_object):
image_files = []
is_ALOS_Landsat_S2 = \
re.search('ALOS', self.satellite) or re.search('Landsat', self.satellite) or \
re.search('Sentinel-2', self.satellite)
re.search(r'ALOS', self.satellite) or re.search(r'Landsat', self.satellite) or \
re.search(r'Sentinel-2', self.satellite)
n_files2search = len(full_LayerBandsAssignment) if is_ALOS_Landsat_S2 else 1
for File in HLP_F.sorted_nicely(files_in_archive):
search_res = \
re.search("IMG-0[0-9]-[\s\S]*", File) if re.search('ALOS', self.satellite) else \
re.search("[\S]*_B[1-9][0-9]?[\S]*.TIF", File) if re.search('Landsat', self.satellite) else \
re.search("[0-9]*.tif", File) if re.search('RapidEye', self.satellite) else \
re.search("imagery.tif", File) if re.search('SPOT', self.satellite) else \
re.search("[\S]*.SAFE/GRANULE/%s/IMG_DATA/[\S]*_B[0-9][\S]*.jp2"
% self.entity_ID, File) if re.search('Sentinel-2', self.satellite) else None
re.search(r"IMG-0[0-9]-[\s\S]*", File) if re.search(r'ALOS', self.satellite) else \
re.search(r"[\S]*_B[1-9][0-9]?[\S]*.TIF", File) if re.search(r'Landsat', self.satellite) else \
re.search(r"[0-9]*.tif", File) if re.search(r'RapidEye', self.satellite) else \
re.search(r"imagery.tif", File) if re.search(r'SPOT', self.satellite) else \
re.search(r"[\S]*.SAFE/GRANULE/%s/IMG_DATA/[\S]*_B[0-9][\S]*.jp2"
% self.entity_ID, File) if re.search(r'Sentinel-2', self.satellite) else None
if search_res:
if re.search('Sentinel-2', self.satellite):
if re.search(r'Sentinel-2', self.satellite):
# add only those files that are corresponding to subsystem (e.g. S2A10: fullLBA = ['2','3','4','8'])
if 1 in [1 if re.search("[\S]*_B[0]?%s.jp2" % LBAn, os.path.basename(File)) else 0
if 1 in [1 if re.search(r"[\S]*_B[0]?%s.jp2" % LBAn, os.path.basename(File)) else 0
for LBAn in full_LayerBandsAssignment]:
image_files.append(File)
else:
......@@ -141,7 +141,7 @@ class L1A_object(GMS_object):
# validate number of expected files #
#####################################
if re.search('ETM+', self.sensor) and self.acq_datetime > datetime.datetime(year=2003, month=5, day=31):
if re.search(r'ETM+', self.sensor) and self.acq_datetime > datetime.datetime(year=2003, month=5, day=31):
expected_files_count = 2 * len(full_LayerBandsAssignment)
else:
expected_files_count = len(full_LayerBandsAssignment)
......@@ -299,9 +299,9 @@ class L1A_object(GMS_object):
self.arr_desc = \
'DN' if self.MetaObj.PhysUnit == 'DN' else \
'Rad' if self.MetaObj.PhysUnit == "W * m-2 * sr-1 * micrometer-1" else \
'TOA_Ref' if re.search('TOA_Reflectance', self.MetaObj.PhysUnit, re.I) else \
'BOA_Ref' if re.search('BOA_Reflectance', self.MetaObj.PhysUnit, re.I) else \
'Temp' if re.search('Degrees Celsius', self.MetaObj.PhysUnit, re.I) else None
'TOA_Ref' if re.search(r'TOA_Reflectance', self.MetaObj.PhysUnit, re.I) else \
'BOA_Ref' if re.search(r'BOA_Reflectance', self.MetaObj.PhysUnit, re.I) else \
'Temp' if re.search(r'Degrees Celsius', self.MetaObj.PhysUnit, re.I) else None
assert self.arr_desc, 'GMS_obj contains an unexpected physical unit: %s' % self.MetaObj.PhysUnit
......@@ -391,7 +391,7 @@ class L1A_object(GMS_object):
(PI * U__earth_sun_distance_correction_factor);
L = (U__earth_sun_distance_correction_factor * rToa * e0__SOLAR_IRRADIANCE_For_band * cos(
Z__Sun_Angles_Grid_Zenith_Values)) / PI;"""
if re.search('Sentinel-2', self.satellite, re.I):
if re.search(r'Sentinel-2', self.satellite, re.I):
warnings.warn('Physical gain values unclear for Sentinel-2! This may cause errors when '
'calculating radiance from TOA Reflectance. ESA provides only 12 gain values for '
'13 bands and it not clear for which bands the gains are provided.')
......@@ -456,7 +456,7 @@ class L1A_object(GMS_object):
os.chdir(os.path.dirname(self.path_archive))
rasObj = GEOP.GEOPROCESSING(self.MetaObj.Dataname, self.logger)
if rasObj.geotransform == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0) and rasObj.projection == '':
if re.search('ALOS', self.satellite) and self.MetaObj.ProcLCode == '1B2':
if re.search(r'ALOS', self.satellite) and self.MetaObj.ProcLCode == '1B2':
self.GeoTransProj_ok, self.GeoAlign_ok = False, True
else:
self.GeoTransProj_ok, self.GeoAlign_ok = False, False
......@@ -547,7 +547,7 @@ class L1A_object(GMS_object):
'mask_nodata') and self.mask_nodata is not None, "The L1A object needs to have a nodata mask."
self.logger.info('Calculating true data corner positions (image and world coordinates)...')
# if re.search('ETM+', self.sensor) and self.acq_datetime > datetime.datetime(year=2003, month=5, day=31,
# if re.search(r'ETM+', self.sensor) and self.acq_datetime > datetime.datetime(year=2003, month=5, day=31,
# tzinfo=datetime.timezone.utc):
if is_dataset_provided_as_fullScene(self.GMS_identifier):
self.trueDataCornerPos = calc_FullDataset_corner_positions(self.mask_nodata, algorithm='numpy',
......@@ -577,7 +577,7 @@ class L1A_object(GMS_object):
else:
if re.search('AVNIR', self.sensor):
if re.search(r'AVNIR', self.sensor):
self.fullSceneCornerPos = calc_FullDataset_corner_positions(self.mask_nodata, algorithm='numpy',
assert_four_corners=False)
# set true data corner positions (lon/lat coordinates)
......@@ -588,7 +588,7 @@ class L1A_object(GMS_object):
else:
# RapidEye or Sentinel-2 data
if re.search('Sentinel-2', self.satellite):
if re.search(r'Sentinel-2', self.satellite):
# get fullScene corner coordinates by database query
# -> calculate footprints for all granules of the same S2 datatake
# -> merge them and calculate overall corner positions
......
......@@ -269,7 +269,7 @@ class AtmCorr(object):
# append AtmCorr object to input L1C objects
# [setattr(L1C_obj, 'AtmCorr', self) for L1C_obj in self.inObjs] # too big for serialization
if not re.search('Sentinel-2', self.inObjs[0].satellite, re.I):
if not re.search(r'Sentinel-2', self.inObjs[0].satellite, re.I):
self.logger.debug('Calculation of acquisition geometry arrays is currently only validated for Sentinel-2!')
# validation possible by comparing S2 angles provided by ESA with own angles # TODO
......@@ -649,7 +649,7 @@ class AtmCorr(object):
:return: <np.ndarray> 2D array (with 20m resolution in case of Sentinel-2)
"""
# determine which input GMS object is used to generate DEM
if re.search('Sentinel-2', self.inObjs[0].satellite):
if re.search(r'Sentinel-2', self.inObjs[0].satellite):
# in case of Sentinel-2 the 20m DEM must be passed
inObj4dem = [obj for obj in self.inObjs if obj.arr.xgsd == 20]
if not inObj4dem:
......
......@@ -481,7 +481,7 @@ class Cloud_Mask_Creator(object):
self.GMS_obj.logger.info("Calculating cloud mask based on '%s' algorithm..." % self.algorithm)
if self.algorithm == 'FMASK':
if re.search('Landsat', self.GMS_obj.satellite, re.I):
if re.search(r'Landsat', self.GMS_obj.satellite, re.I):
FMR = FMASK_Runner_Landsat(self.GMS_obj.path_archive, self.GMS_obj.satellite)
else:
......
......@@ -90,7 +90,7 @@ class GEOPROCESSING(object):
# '/vsitar' '.gz': '/vsigzip'}
p1 = [geodata.split(i)[0] + i for i in ['.zip', '.tar', '.tar.gz', '.gz', '.tgz']
if len(geodata.split(i)) > 1 and geodata.split(i)[1].startswith('/')][0]
path2check = os.path.abspath('.' + re.search('/vsi[\s\S]*(/[\s\S,.]*)', p1, re.I).group(1))
path2check = os.path.abspath('.' + re.search(r'/vsi[\s\S]*(/[\s\S,.]*)', p1, re.I).group(1))
assert os.path.exists(path2check), "ERROR: data %s does not exist!" % path2check
assert self.inDs is not None, "ERROR: Could not open %s!" % self.filename
elif isinstance(geodata, gdal.Dataset):
......@@ -682,7 +682,7 @@ class GEOPROCESSING(object):
with open(os.path.splitext(path_output)[0] + '.hdr', 'r') as inF:
lines = inF.readlines()
outContent = ''.join([i for i in lines if not re.search('map info', i, re.I)])
outContent = ''.join([i for i in lines if not re.search(r'map info', i, re.I)])
with open(os.path.splitext(path_output)[0] + '.hdr', 'w') as outF:
outF.write(outContent)
......@@ -696,7 +696,7 @@ class GEOPROCESSING(object):
#
# with open(os.path.splitext(path_output)[0] + '.hdr', 'r') as inF:
# lines = inF.readlines()
# outContent = ''.join([line if not re.search('coordinate system string', line, re.I) else
# outContent = ''.join([line if not re.search(r'coordinate system string', line, re.I) else
# 'coordinate system string = %s' % self.projection for line in lines])
#
# with open(os.path.splitext(path_output)[0] + '.hdr', 'w') as outF:
......
......@@ -581,9 +581,9 @@ def get_filename_by_entityID(conn_DB, entityid, satellite):
:param satellite: <str> satellite name to which the entity ID is belonging
"""
if re.search('Landsat', satellite, re.I):
if re.search(r'Landsat', satellite, re.I):
filename = '%s.tar.gz' % entityid
elif re.search('Sentinel-2', satellite, re.I):
elif re.search(r'Sentinel-2', satellite, re.I):
filename = get_info_from_postgreSQLdb(conn_DB, 'scenes', ['filename'],
{'entityid': entityid}, records2fetch=1)[0][0]
else:
......@@ -771,12 +771,18 @@ class GMS_JOB(object):
if isinstance(datadict['filenames'], str) and datadict['filenames'].endswith('.csv'):
datadict['filenames'] = None # TODO implement csv reader here
raise NotImplementedError
else:
temp_gdf = GeoDataFrame(datadict, columns=['satellite', 'sensor', 'filenames'])
if re.search('Landsat-7', datadict['satellite'], re.I) and re.search('ETM+', datadict['sensor'], re.I):
if re.search(r'Landsat-7', datadict['satellite'], re.I) and \
re.search(r'ETM+', datadict['sensor'], re.I):
from .helper_functions import Landsat_entityID_decrypter as LED
def get_L7_sensor(fN): return LED(fN.split('.tar.gz')[0]).sensorIncSLC
def get_L7_sensor(fN):
return LED(fN.split('.tar.gz')[0]).sensorIncSLC
temp_gdf['sensor'] = list(temp_gdf['filenames'].map(get_L7_sensor))
all_gdfs.append(temp_gdf)
......@@ -1405,7 +1411,7 @@ def archive_exists_on_fileserver(conn_DB, entityID):
archive_fold = os.path.join(CFG.path_archive, satellite, sensor)
assert os.path.exists(archive_fold), 'Archive folder not found: %s.' % archive_fold
if re.search('Landsat', satellite, re.I):
if re.search(r'Landsat', satellite, re.I):
exists = os.path.exists(os.path.join(archive_fold, entityID + '.tar.gz'))
else:
raise NotImplementedError
......
......@@ -30,7 +30,7 @@ def get_GMS_sensorcode(GMS_id):
# type: (GMS_identifier) -> str
Satellite, Sensor, Subsystem = (GMS_id.satellite, GMS_id.sensor, GMS_id.subsystem)
Sensor = Sensor[:-1] if re.match('SPOT', Satellite, re.I) and Sensor[-1] not in ['1', '2'] else Sensor
Sensor = Sensor[:-1] if re.match(r'SPOT', Satellite, re.I) and Sensor[-1] not in ['1', '2'] else Sensor
meta_sensorcode = Satellite + '_' + Sensor + ('_' + Subsystem if Subsystem not in ["", None] else "")
sensorcode_dic = {
'ALOS_AVNIR-2': 'AVNIR-2',
......
......@@ -288,8 +288,8 @@ def get_path_srf_file(GMS_id, bandname=''):
"""
satellite, sensor = GMS_id.satellite, GMS_id.sensor
satellite = 'RapidEye' if re.match('RapidEye', satellite, re.I) else satellite
sensor = sensor[:-1] if re.match('SPOT', satellite, re.I) and sensor[-1] not in ['1', '2'] else sensor
satellite = 'RapidEye' if re.match(r'RapidEye', satellite, re.I) else satellite
sensor = sensor[:-1] if re.match(r'SPOT', satellite, re.I) and sensor[-1] not in ['1', '2'] else sensor
filename = 'band_%s' % bandname if bandname else ''
return os.path.join(CFG.path_SRFs, satellite, sensor, filename)
......@@ -302,8 +302,8 @@ def get_path_snr_model(GMS_id):
"""
satellite, sensor = (GMS_id.satellite, GMS_id.sensor)
satellite = 'RapidEye' if re.match('RapidEye', satellite, re.I) else satellite
sensor = sensor[:-1] if re.match('SPOT', satellite, re.I) and sensor[-1] not in ['1', '2'] else sensor
satellite = 'RapidEye' if re.match(r'RapidEye', satellite, re.I) else satellite
sensor = sensor[:-1] if re.match(r'SPOT', satellite, re.I) and sensor[-1] not in ['1', '2'] else sensor
return os.path.join(CFG.path_SNR_models, satellite, sensor, 'SNR_model.csv')
......
......@@ -62,7 +62,7 @@ class SpatialIndexMediatorServer:
running = 'is running' in outputStr
# get PID
_process_id = re.search('with pid ([\d]*)', outputStr)
_process_id = re.search(r'with pid ([\d]*)', outputStr)
if _process_id and _process_id.group(1):
process_id = int(_process_id.group(1))
else:
......@@ -87,7 +87,7 @@ class SpatialIndexMediatorServer:
def stop(self):
outputStr = self._communicate('stop')
if outputStr == 'success' or re.search('index-mediator-server stopped', outputStr, re.I):
if outputStr == 'success' or re.search(r'index-mediator-server stopped', outputStr, re.I):
return 'stopped'
else:
warnings.warn("\nStopping Spatial Index Mediator Server failed with message '%s'!"
......
......@@ -354,7 +354,7 @@ class GMS_object(object):
def georef(self):
"""Returns True if the current dataset can serve as spatial reference."""
return True if self.image_type == 'RSD' and re.search('OLI', self.sensor, re.I) else False
return True if self.image_type == 'RSD' and re.search(r'OLI', self.sensor, re.I) else False
@property
def coreg_needed(self):
......@@ -1138,7 +1138,7 @@ class GMS_object(object):
# apply cloud mask to image data and all products derived from image data
# (only if image data represents BOA-Ref and cloud areas are not to be filled with TOA-Ref)
if re.search('BOA_Reflectance', GMS_obj_merged.MetaObj.PhysUnit, re.I) and not CFG.ac_fillnonclear_areas:
if re.search(r'BOA_Reflectance', GMS_obj_merged.MetaObj.PhysUnit, re.I) and not CFG.ac_fillnonclear_areas:
# fill non-clear areas with no data values (for all bands)
for pixVal in nonclear_pixVals:
mask_nonclear = GMS_obj_merged.mask_clouds[:] == pixVal
......@@ -1189,7 +1189,7 @@ class GMS_object(object):
@classmethod
def from_tiles(cls, list_GMS_tiles):
# type: (list) -> GMS_object
"""Merge separate GMS objects with different spatial coverage but belonging to the same scene-ID to ONE GMS object.
"""Merge separate GMS objects with different spatial coverage but belonging to one scene-ID to ONE GMS object.
:param list_GMS_tiles: <list> of GMS objects that have been created by cut_GMS_obj_into_blocks()
"""
......
......@@ -130,17 +130,17 @@ class METADATA(object):
self.FolderOrArchive = data_folderOrArchive
self.LayerBandsAssignment = LayerBandsAssignment if LayerBandsAssignment else []
if re.search("SPOT", self.Satellite, re.I):
if re.search(r"SPOT", self.Satellite, re.I):
self.Read_SPOT_dimap2()
elif re.search("Terra", self.Satellite, re.I):
elif re.search(r"Terra", self.Satellite, re.I):
self.Read_ASTER_hdffile(self.Subsystem)
elif re.search("Sentinel-2A", self.Satellite, re.I) or re.search("Sentinel-2B", self.Satellite, re.I):
elif re.search(r"Sentinel-2A", self.Satellite, re.I) or re.search(r"Sentinel-2B", self.Satellite, re.I):
self.Read_Sentinel2_xmls()
elif re.search("LANDSAT", self.Satellite, re.I):
elif re.search(r"LANDSAT", self.Satellite, re.I):
self.Read_LANDSAT_mtltxt(self.LayerBandsAssignment)
elif re.search("RapidEye", self.Satellite, re.I):
elif re.search(r"RapidEye", self.Satellite, re.I):
self.Read_RE_metaxml()
elif re.search("ALOS", self.Satellite, re.I):
elif re.search(r"ALOS", self.Satellite, re.I):
self.Read_ALOS_summary()
self.Read_ALOS_LEADER() # for gains and offsets
else:
......@@ -179,12 +179,10 @@ class METADATA(object):
def overview(self):
attr2include = \
['Dataname', 'Metafile', 'EntityID', 'Satellite', 'Sensor', 'Subsystem', 'gResolution', 'AcqDate',
'AcqTime',
'CWL', 'FWHM', 'Offsets', 'Gains', 'ProcLCode', 'ProcLStr', 'SunElevation', 'SunAzimuth',
'AcqTime', 'CWL', 'FWHM', 'Offsets', 'Gains', 'ProcLCode', 'ProcLStr', 'SunElevation', 'SunAzimuth',
'ViewingAngle', 'IncidenceAngle', 'FOV', 'SolIrradiance', 'ThermalConstK1', 'ThermalConstK2',
'EarthSunDist', 'Quality', 'PhysUnit', 'additional', 'GainsRef', 'OffsetsRef', 'CornerTieP_LonLat',
'CS_EPSG',
'CS_TYPE', 'CS_DATUM', 'CS_UTM_ZONE', 'LayerBandsAssignment']
'CS_EPSG', 'CS_TYPE', 'CS_DATUM', 'CS_UTM_ZONE', 'LayerBandsAssignment']
return collections.OrderedDict((attr, getattr(self, attr)) for attr in attr2include)
@property
......@@ -215,25 +213,25 @@ class METADATA(object):
dim_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*/scene01/metadata.dim')
# special values
h1 = re.findall("<SPECIAL_VALUE_INDEX>([a-zA-Z0-9]*)</SPECIAL_VALUE_INDEX>", dim_, re.I)
h2 = re.findall("<SPECIAL_VALUE_TEXT>([a-zA-Z0-9]*)</SPECIAL_VALUE_TEXT>", dim_, re.I)
h1 = re.findall(r"<SPECIAL_VALUE_INDEX>([a-zA-Z0-9]*)</SPECIAL_VALUE_INDEX>", dim_, re.I)
h2 = re.findall(r"<SPECIAL_VALUE_TEXT>([a-zA-Z0-9]*)</SPECIAL_VALUE_TEXT>", dim_, re.I)
self.additional.append(["SpecialValues:"])
for ii, ind in enumerate(h1):
self.additional[0].append(["%s:%s" % (ind, h2[ii])])
# EntityID
h3 = re.search("<SOURCE_ID>([a-zA-Z0-9]*)</SOURCE_ID>", dim_, re.I)
h3 = re.search(r"<SOURCE_ID>([a-zA-Z0-9]*)</SOURCE_ID>", dim_, re.I)
self.EntityID = h3.group(1)
# AcqDate
h4 = re.search("<IMAGING_DATE>([0-9-]*)</IMAGING_DATE>", dim_, re.I)
h4 = re.search(r"<IMAGING_DATE>([0-9-]*)</IMAGING_DATE>", dim_, re.I)
AcqDate = h4.group(1)
# Earth sun distance
self.EarthSunDist = self.get_EarthSunDistance(AcqDate)
# AcqTime
h5 = re.search("<IMAGING_TIME>([0-9:]*)</IMAGING_TIME>", dim_, re.I)
h5 = re.search(r"<IMAGING_TIME>([0-9:]*)</IMAGING_TIME>", dim_, re.I)
AcqTime = h5.group(1)
# set self.AcqDateTime as well as self.AcqDate and self.AcqTime
......@@ -241,21 +239,25 @@ class METADATA(object):
'%s %s%s' % (AcqDate, AcqTime, '.000000+0000'), '%Y-%m-%d %H:%M:%S.%f%z')
# Satellite + Sensor
h6 = re.search("<MISSION>([a-zA-Z]*)</MISSION>[a-zA-Z0-9\s]*<MISSION_INDEX>([0-9]*)</MISSION_INDEX>"
"[a-zA-Z0-9\s]*<INSTRUMENT>([a-zA-Z]*)</INSTRUMENT>[a-zA-Z0-9\s]*<INSTRUMENT_INDEX>([0-9]*)"
"</INSTRUMENT_INDEX>[a-zA-Z0-9\s]*<SENSOR_CODE>([a-zA-Z0-9]*)</SENSOR_CODE>", dim_, re.I)
h6 = re.search(r"<MISSION>([a-zA-Z]*)</MISSION>[a-zA-Z0-9\s]*"
r"<MISSION_INDEX>([0-9]*)</MISSION_INDEX>[a-zA-Z0-9\s]*"
r"<INSTRUMENT>([a-zA-Z]*)</INSTRUMENT>[a-zA-Z0-9\s]*"
r"<INSTRUMENT_INDEX>([0-9]*)</INSTRUMENT_INDEX>[a-zA-Z0-9\s]*"
r"<SENSOR_CODE>([a-zA-Z0-9]*)</SENSOR_CODE>",
dim_, re.I)
self.Satellite = "-".join([h6.group(1), h6.group(2)])
self.Sensor = "".join([h6.group(3), h6.group(4), h6.group(5)])
# Angles: incidence angle, sunAzimuth, sunElevation
h7 = re.search("<INCIDENCE_ANGLE>(.*)</INCIDENCE_ANGLE>[\s\S]*<SUN_AZIMUTH>(.*)</SUN_AZIMUTH>[\s\S]"
"*<SUN_ELEVATION>(.*)</SUN_ELEVATION>", dim_, re.I)
h7 = re.search(r"<INCIDENCE_ANGLE>(.*)</INCIDENCE_ANGLE>[\s\S]*"
r"<SUN_AZIMUTH>(.*)</SUN_AZIMUTH>[\s\S]"
r"*<SUN_ELEVATION>(.*)</SUN_ELEVATION>", dim_, re.I)
self.IncidenceAngle = float(h7.group(1))
self.SunAzimuth = float(h7.group(2))
self.SunElevation = float(h7.group(3))
# Viewing Angle: not always included in the metadata.dim file
h8 = re.search("<VIEWING_ANGLE>(.*)</VIEWING_ANGLE>", dim_, re.I)
h8 = re.search(r"<VIEWING_ANGLE>(.*)</VIEWING_ANGLE>", dim_, re.I)
if type(h8).__name__ == 'NoneType':
self.ViewingAngle = 0
else:
......@@ -269,26 +271,29 @@ class METADATA(object):
self.PhysUnit = "DN"
# ProcLevel
h11 = re.search("<PROCESSING_LEVEL>([a-zA-Z0-9]*)</PROCESSING_LEVEL>", dim_, re.I)
h11 = re.search(r"<PROCESSING_LEVEL>([a-zA-Z0-9]*)</PROCESSING_LEVEL>", dim_, re.I)
self.ProcLCode = h11.group(1)
# Quality
h12 = re.findall("<Bad_Pixel>[\s]*<PIXEL_INDEX>([0-9]*)</PIXEL_INDEX>[\s]*<BAD_PIXEL_STATUS>([^<]*)"
"</BAD_PIXEL_STATUS></Bad_Pixel>", dim_, re.I)
h12 = re.findall(r"<Bad_Pixel>[\s]*"
r"<PIXEL_INDEX>([0-9]*)</PIXEL_INDEX>[\s]*"
r"<BAD_PIXEL_STATUS>([^<]*)</BAD_PIXEL_STATUS>"
r"</Bad_Pixel>", dim_,
re.I)
self.Quality = h12
# Coordinate Reference System
re_CS_EPSG = re.search('<HORIZONTAL_CS_CODE>epsg:([0-9]*)</HORIZONTAL_CS_CODE>', dim_, re.I)
re_CS_TYPE = re.search('<HORIZONTAL_CS_TYPE>([a-zA-Z0-9]*)</HORIZONTAL_CS_TYPE>', dim_, re.I)
re_CS_DATUM = re.search('<HORIZONTAL_CS_NAME>([\w+\s]*)</HORIZONTAL_CS_NAME>', dim_, re.I)
re_CS_EPSG = re.search(r'<HORIZONTAL_CS_CODE>epsg:([0-9]*)</HORIZONTAL_CS_CODE>', dim_, re.I)
re_CS_TYPE = re.search(r'<HORIZONTAL_CS_TYPE>([a-zA-Z0-9]*)</HORIZONTAL_CS_TYPE>', dim_, re.I)
re_CS_DATUM = re.search(r'<HORIZONTAL_CS_NAME>([\w+\s]*)</HORIZONTAL_CS_NAME>', dim_, re.I)
self.CS_EPSG = int(re_CS_EPSG.group(1)) if re_CS_EPSG is not None else self.CS_EPSG
self.CS_TYPE = 'LonLat' if re_CS_TYPE is not None and re_CS_TYPE.group(1) == 'GEOGRAPHIC' else 'UTM' \
if re_CS_TYPE is not None and re_CS_TYPE.group(1) == 'UTM' else self.CS_TYPE
self.CS_DATUM = 'WGS84' if re_CS_DATUM is not None and re_CS_DATUM.group(1) == 'WGS 84' else self.CS_DATUM
# Corner Coordinates
h121 = re.findall("<TIE_POINT_CRS_X>(.*)</TIE_POINT_CRS_X>", dim_, re.I)
h122 = re.findall("<TIE_POINT_CRS_Y>(.*)</TIE_POINT_CRS_Y>", dim_, re.I)
h121 = re.findall(r"<TIE_POINT_CRS_X>(.*)</TIE_POINT_CRS_X>", dim_, re.I)
h122 = re.findall(r"<TIE_POINT_CRS_Y>(.*)</TIE_POINT_CRS_Y>", dim_, re.I)
if len(h121) == 4 and self.CS_TYPE == 'LonLat':
# Set Corner Tie Point Coordinates (order = UL,UR,LL,LR)
self.CornerTieP_LonLat.append(tuple([float(h121[0]), float(h122[0])])) # UL
......@@ -305,11 +310,11 @@ class METADATA(object):
LBA_full_sorted = HLP_F.sorted_nicely(self.LayerBandsAssignment_full)
# Gains and Offsets
h9 = re.search("<Image_Interpretation>[\s\S]*</Image_Interpretation>", dim_, re.I)
h9 = re.search(r"<Image_Interpretation>[\s\S]*</Image_Interpretation>", dim_, re.I)
h9_ = h9.group(0)
h91 = re.findall("<PHYSICAL_UNIT>([^<]*)</PHYSICAL_UNIT>", h9_, re.I)
h92 = re.findall("<PHYSICAL_BIAS>([^<]*)</PHYSICAL_BIAS>", h9_, re.I)
h93 = re.findall("<PHYSICAL_GAIN>([^<]*)</PHYSICAL_GAIN>", h9_, re.I)
h91 = re.findall(r"<PHYSICAL_UNIT>([^<]*)</PHYSICAL_UNIT>", h9_, re.I)
h92 = re.findall(r"<PHYSICAL_BIAS>([^<]*)</PHYSICAL_BIAS>", h9_, re.I)
h93 = re.findall(r"<PHYSICAL_GAIN>([^<]*)</PHYSICAL_GAIN>", h9_, re.I)
self.additional.append(["Physical Units per band:"])
for ii, ind in enumerate(h91): # FIXME does not respect sort_bands_by_cwl
self.additional[-1].append(ind)
......@@ -338,9 +343,11 @@ class METADATA(object):
self.SolIrradiance, self.CWL, self.FWHM = self.calc_solar_irradiance_CWL_FWHM_per_band()
# Provider values
if not self.SolIrradiance:
h10 = re.search("<Solar_Irradiance>[\s\S]*</Solar_Irradiance>", dim_, re.I)
h10 = re.search(r"<Solar_Irradiance>[\s\S]*"
r"</Solar_Irradiance>", dim_, re.I)
h10_ = h10.group(0)
h101 = re.findall("<SOLAR_IRRADIANCE_VALUE>([^<]*)</SOLAR_IRRADIANCE_VALUE>", h10_, re.I)
h101 = re.findall(r"<SOLAR_IRRADIANCE_VALUE>([^<]*)"
r"</SOLAR_IRRADIANCE_VALUE>", h10_, re.I)
if h101:
self.SolIrradiance = dict(zip(LBA_full_sorted, h101))
# self.additional.append(["Solar Irradiance per band:"])
......@@ -392,36 +399,42 @@ class METADATA(object):
mtl_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*MTL.txt')
# Processing Level
h1 = re.search('PRODUCT_TYPE = "([a-zA-Z0-9]*)"', mtl_, re.I)
h1 = re.search(r'PRODUCT_TYPE = "([a-zA-Z0-9]*)"', mtl_, re.I)
if h1 is None:
h1 = re.search('DATA_TYPE = "([a-zA-Z0-9]*)"', mtl_, re.I)
h1 = re.search(r'DATA_TYPE = "([a-zA-Z0-9]*)"', mtl_, re.I)
self.ProcLCode = h1.group(1)
# Satellite + Sensor + Sensor Mode
h2 = re.search('SPACECRAFT_ID = "([a-zA-Z0-9_]*)"[\s]*SENSOR_ID = "([a-zA-Z0-9+]*)"[\s]*'
'SENSOR_MODE = "([\S]*)"', mtl_, re.I)
h2 = re.search(r'SPACECRAFT_ID = "([a-zA-Z0-9_]*)"[\s]*'
r'SENSOR_ID = "([a-zA-Z0-9+]*)"[\s]*'
r'SENSOR_MODE = "([\S]*)"',
mtl_, re.I)
if h2:
self.Satellite = 'Landsat-%s' % re.search('LANDSAT[\D]*([0-9])', h2.group(1), re.I).group(1)
self.Satellite = 'Landsat-%s' % re.search(r'LANDSAT[\D]*([0-9])', h2.group(1), re.I).group(1)
self.Sensor = h2.group(2)
self.Sensormode = h2.group(3)
else:
h2a = re.search('SPACECRAFT_ID = "([a-zA-Z0-9_]*)"', mtl_, re.I)
h2b = re.search('SENSOR_ID = "([a-zA-Z0-9_+]*)"', mtl_, re.I)
h2c = re.search('SENSOR_MODE = "([a-zA-Z0-9_+]*)"', mtl_, re.I)
self.Satellite = 'Landsat-%s' % re.search('LANDSAT[\D]*([0-9])', h2a.group(1), re.I).group(1)
h2a = re.search(r'SPACECRAFT_ID = "([a-zA-Z0-9_]*)"', mtl_, re.I)
h2b = re.search(r'SENSOR_ID = "([a-zA-Z0-9_+]*)"', mtl_, re.I)
h2c = re.search(r'SENSOR_MODE = "([a-zA-Z0-9_+]*)"', mtl_, re.I)
self.Satellite = 'Landsat-%s' % re.search(r'LANDSAT[\D]*([0-9])', h2a.group(1), re.I).group(1)
self.Sensor = h2b.group(1)
self.Sensormode = h2c.group(1) if h2c is not None else self.Sensormode # Landsat-8
self.Sensor = 'ETM+' if self.Sensor == 'ETM' else self.Sensor
# EntityID
h2c = re.search('LANDSAT_SCENE_ID = "([A-Z0-9]*)"', mtl_, re.I)
h2c = re.search(r'LANDSAT_SCENE_ID = "([A-Z0-9]*)"', mtl_, re.I)
if h2c:
self.EntityID = h2c.group(1)
# Acquisition Date + Time
h3 = re.search('ACQUISITION_DATE = ([0-9-]*)[\s]*SCENE_CENTER_SCAN_TIME = "?([0-9:]*)"?', mtl_, re.I)
h3 = re.search(r'ACQUISITION_DATE = ([0-9-]*)[\s]*'
r'SCENE_CENTER_SCAN_TIME = "?([0-9:]*)"?',
mtl_, re.I)
if h3 is None:
h3 = re.search('DATE_ACQUIRED = ([0-9-]*)[\s]*SCENE_CENTER_TIME = "?([0-9:]*)"?', mtl_, re.I)
h3 = re.search(r'DATE_ACQUIRED = ([0-9-]*)[\s]*'
r'SCENE_CENTER_TIME = "?([0-9:]*)"?',
mtl_, re.I)
AcqDate = h3.group(1)
AcqTime = h3.group(2)
......@@ -437,15 +450,21 @@ class METADATA(object):
self.PhysUnit = "DN"
# Angles: incidence angle, sunAzimuth, sunElevation, field of view
h5 = re.search("SUN_AZIMUTH = ([\S]*)[\s]*SUN_ELEVATION = ([\S]*)", mtl_, re.I)
h5 = re.search(r"SUN_AZIMUTH = ([\S]*)[\s]*"
r"SUN_ELEVATION = ([\S]*)",
mtl_, re.I)
self.SunAzimuth = float(h5.group(1))
self.SunElevation = float(h5.group(2))
self.FOV = get_FieldOfView(self.GMS_identifier)
# Quality
h6 = re.search("GROUP = CORRECTIONS_APPLIED[\s\S]*END_GROUP = CORRECTIONS_APPLIED", mtl_, re.I)
h6 = re.search(r"GROUP = CORRECTIONS_APPLIED[\s\S]*"
r"END_GROUP = CORRECTIONS_APPLIED",
mtl_, re.I)
if h6 is None:
h6 = re.search("GROUP = IMAGE_ATTRIBUTES[\s\S]*END_GROUP = IMAGE_ATTRIBUTES", mtl_, re.I)
h6 = re.search(r"GROUP = IMAGE_ATTRIBUTES[\s\S]*"
r"END_GROUP = IMAGE_ATTRIBUTES",
mtl_, re.I)
h6_ = h6.group(0)
h61 = (h6_.split("\n"))
......@@ -459,19 +478,19 @@ class METADATA(object):
x += 1
# Additional: coordinate system, geom. Resolution
h7 = re.search("GROUP = PROJECTION_PARAMETERS[\s\S]*END_GROUP = L1_METADATA_FILE", mtl_, re.I)
h7 = re.search(r"GROUP = PROJECTION_PARAMETERS[\s\S]*END_GROUP = L1_METADATA_FILE", mtl_, re.I)
h7_ = h7.group(0)
h71 = (h7_.split("\n"))
for x, i in enumerate(h71):
if re.search("Group", i, re.I):
if re.search(r"Group", i, re.I):
pass
else:
i_ = i.strip().replace('"', "")
self.additional.append(i_.split(" = "))
re_CS_TYPE = re.search('MAP_PROJECTION = "([\w+\s]*)"', h7_, re.I)
re_CS_DATUM = re.search('DATUM = "([\w+\s]*)"', h7_, re.I)
re_CS_UTM_ZONE = re.search('ZONE_NUMBER = ([0-9]*)\n', mtl_, re.I)
re_CS_UTM_ZONE = re.search('UTM_ZONE = ([0-9]*)\n', h7_,
re_CS_TYPE = re.search(r'MAP_PROJECTION = "([\w+\s]*)"', h7_, re.I)