Commit ade94536 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'master' into enhancement/add_license_texts

Conflicts:
	gms_preprocessing/algorithms/classification.py
	gms_preprocessing/processing/multiproc.py
	tests/test_image_classifier.py
parents 04c0f6e7 d982da7b
Pipeline #8018 passed with stage
in 17 minutes and 48 seconds
......@@ -34,6 +34,8 @@ test_gms_preprocessing:
# - cd ../../
# make tests
- pip install git+https://gitext.gfz-potsdam.de/geomultisens/spechomo.git # TODO remove as soon as this is included in the CI container
# run tests
- make nosetests
- make docs
......
This diff is collapsed.
......@@ -27,7 +27,7 @@ Status
:target: https://pyup.io/repos/github/geomultisens/gms_preprocessing/
:alt: Updates
.. image:: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/badges/master/build.svg
.. image:: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/badges/master/pipeline.svg
:target: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/commits/master
.. image:: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/badges/master/coverage.svg
:target: http://geomultisens.gitext.gfz-potsdam.de/gms_preprocessing/coverage/
......@@ -55,7 +55,7 @@ Level-2 processing:
^^^^^^^^^^^^^^^^^^^
* spatial homogenization
* spectral homogenization
* spectral homogenization (using `SpecHomo <https://gitext.gfz-potsdam.de/geomultisens/spechomo>`_
* estimation of accuracy layers
=> application oriented analysis dataset
......
......@@ -97,13 +97,13 @@ class L1A_object(GMS_object):
self.proc_status = proc_status or 'initialized' # if proc_status = 'running' is given by L1A_map
def import_rasterdata(self):
if re.search("ALOS", self.satellite, re.I):
if re.search(r"ALOS", self.satellite, re.I):
'''First 22 lines are nodata: = maybe due to an issue of the GDAL CEOS driver.
But: UL of metadata refers to [row =0, col=21]! So the imported GeoTransform is correct when
the first 21 columns are deleted.'''
self.archive_to_rasObj(self.path_archive, self.path_InFilePreprocessor,
subset=['block', [[None, None], [21, None]]])
elif re.search("Terra", self.satellite, re.I):
elif re.search(r"Terra", self.satellite, re.I):
self.ASTER_HDF_to_rasObj(self.path_archive, path_output=self.path_InFilePreprocessor)
else:
self.archive_to_rasObj(self.path_archive, path_output=self.path_InFilePreprocessor)
......@@ -135,23 +135,23 @@ class L1A_object(GMS_object):
image_files = []
is_ALOS_Landsat_S2 = \
re.search('ALOS', self.satellite) or re.search('Landsat', self.satellite) or \
re.search('Sentinel-2', self.satellite)
re.search(r'ALOS', self.satellite) or re.search(r'Landsat', self.satellite) or \
re.search(r'Sentinel-2', self.satellite)
n_files2search = len(full_LayerBandsAssignment) if is_ALOS_Landsat_S2 else 1
for File in HLP_F.sorted_nicely(files_in_archive):
search_res = \
re.search("IMG-0[0-9]-[\s\S]*", File) if re.search('ALOS', self.satellite) else \
re.search("[\S]*_B[1-9][0-9]?[\S]*.TIF", File) if re.search('Landsat', self.satellite) else \
re.search("[0-9]*.tif", File) if re.search('RapidEye', self.satellite) else \
re.search("imagery.tif", File) if re.search('SPOT', self.satellite) else \
re.search("[\S]*.SAFE/GRANULE/%s/IMG_DATA/[\S]*_B[0-9][\S]*.jp2"
% self.entity_ID, File) if re.search('Sentinel-2', self.satellite) else None
re.search(r"IMG-0[0-9]-[\s\S]*", File) if re.search(r'ALOS', self.satellite) else \
re.search(r"[\S]*_B[1-9][0-9]?[\S]*.TIF", File) if re.search(r'Landsat', self.satellite) else \
re.search(r"[0-9]*.tif", File) if re.search(r'RapidEye', self.satellite) else \
re.search(r"imagery.tif", File) if re.search(r'SPOT', self.satellite) else \
re.search(r"[\S]*.SAFE/GRANULE/%s/IMG_DATA/[\S]*_B[0-9][\S]*.jp2"
% self.entity_ID, File) if re.search(r'Sentinel-2', self.satellite) else None
if search_res:
if re.search('Sentinel-2', self.satellite):
if re.search(r'Sentinel-2', self.satellite):
# add only those files that are corresponding to subsystem (e.g. S2A10: fullLBA = ['2','3','4','8'])
if 1 in [1 if re.search("[\S]*_B[0]?%s.jp2" % LBAn, os.path.basename(File)) else 0
if 1 in [1 if re.search(r"[\S]*_B[0]?%s.jp2" % LBAn, os.path.basename(File)) else 0
for LBAn in full_LayerBandsAssignment]:
image_files.append(File)
else:
......@@ -163,7 +163,7 @@ class L1A_object(GMS_object):
# validate number of expected files #
#####################################
if re.search('ETM+', self.sensor) and self.acq_datetime > datetime.datetime(year=2003, month=5, day=31):
if re.search(r'ETM+', self.sensor) and self.acq_datetime > datetime.datetime(year=2003, month=5, day=31):
expected_files_count = 2 * len(full_LayerBandsAssignment)
else:
expected_files_count = len(full_LayerBandsAssignment)
......@@ -321,9 +321,9 @@ class L1A_object(GMS_object):
self.arr_desc = \
'DN' if self.MetaObj.PhysUnit == 'DN' else \
'Rad' if self.MetaObj.PhysUnit == "W * m-2 * sr-1 * micrometer-1" else \
'TOA_Ref' if re.search('TOA_Reflectance', self.MetaObj.PhysUnit, re.I) else \
'BOA_Ref' if re.search('BOA_Reflectance', self.MetaObj.PhysUnit, re.I) else \
'Temp' if re.search('Degrees Celsius', self.MetaObj.PhysUnit, re.I) else None
'TOA_Ref' if re.search(r'TOA_Reflectance', self.MetaObj.PhysUnit, re.I) else \
'BOA_Ref' if re.search(r'BOA_Reflectance', self.MetaObj.PhysUnit, re.I) else \
'Temp' if re.search(r'Degrees Celsius', self.MetaObj.PhysUnit, re.I) else None
assert self.arr_desc, 'GMS_obj contains an unexpected physical unit: %s' % self.MetaObj.PhysUnit
......@@ -413,7 +413,7 @@ class L1A_object(GMS_object):
(PI * U__earth_sun_distance_correction_factor);
L = (U__earth_sun_distance_correction_factor * rToa * e0__SOLAR_IRRADIANCE_For_band * cos(
Z__Sun_Angles_Grid_Zenith_Values)) / PI;"""
if re.search('Sentinel-2', self.satellite, re.I):
if re.search(r'Sentinel-2', self.satellite, re.I):
warnings.warn('Physical gain values unclear for Sentinel-2! This may cause errors when '
'calculating radiance from TOA Reflectance. ESA provides only 12 gain values for '
'13 bands and it not clear for which bands the gains are provided.')
......@@ -478,7 +478,7 @@ class L1A_object(GMS_object):
os.chdir(os.path.dirname(self.path_archive))
rasObj = GEOP.GEOPROCESSING(self.MetaObj.Dataname, self.logger)
if rasObj.geotransform == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0) and rasObj.projection == '':
if re.search('ALOS', self.satellite) and self.MetaObj.ProcLCode == '1B2':
if re.search(r'ALOS', self.satellite) and self.MetaObj.ProcLCode == '1B2':
self.GeoTransProj_ok, self.GeoAlign_ok = False, True
else:
self.GeoTransProj_ok, self.GeoAlign_ok = False, False
......@@ -569,7 +569,7 @@ class L1A_object(GMS_object):
'mask_nodata') and self.mask_nodata is not None, "The L1A object needs to have a nodata mask."
self.logger.info('Calculating true data corner positions (image and world coordinates)...')
# if re.search('ETM+', self.sensor) and self.acq_datetime > datetime.datetime(year=2003, month=5, day=31,
# if re.search(r'ETM+', self.sensor) and self.acq_datetime > datetime.datetime(year=2003, month=5, day=31,
# tzinfo=datetime.timezone.utc):
if is_dataset_provided_as_fullScene(self.GMS_identifier):
self.trueDataCornerPos = calc_FullDataset_corner_positions(self.mask_nodata, algorithm='numpy',
......@@ -599,7 +599,7 @@ class L1A_object(GMS_object):
else:
if re.search('AVNIR', self.sensor):
if re.search(r'AVNIR', self.sensor):
self.fullSceneCornerPos = calc_FullDataset_corner_positions(self.mask_nodata, algorithm='numpy',
assert_four_corners=False)
# set true data corner positions (lon/lat coordinates)
......@@ -610,7 +610,7 @@ class L1A_object(GMS_object):
else:
# RapidEye or Sentinel-2 data
if re.search('Sentinel-2', self.satellite):
if re.search(r'Sentinel-2', self.satellite):
# get fullScene corner coordinates by database query
# -> calculate footprints for all granules of the same S2 datatake
# -> merge them and calculate overall corner positions
......
......@@ -291,7 +291,7 @@ class AtmCorr(object):
# append AtmCorr object to input L1C objects
# [setattr(L1C_obj, 'AtmCorr', self) for L1C_obj in self.inObjs] # too big for serialization
if not re.search('Sentinel-2', self.inObjs[0].satellite, re.I):
if not re.search(r'Sentinel-2', self.inObjs[0].satellite, re.I):
self.logger.debug('Calculation of acquisition geometry arrays is currently only validated for Sentinel-2!')
# validation possible by comparing S2 angles provided by ESA with own angles # TODO
......@@ -671,7 +671,7 @@ class AtmCorr(object):
:return: <np.ndarray> 2D array (with 20m resolution in case of Sentinel-2)
"""
# determine which input GMS object is used to generate DEM
if re.search('Sentinel-2', self.inObjs[0].satellite):
if re.search(r'Sentinel-2', self.inObjs[0].satellite):
# in case of Sentinel-2 the 20m DEM must be passed
inObj4dem = [obj for obj in self.inObjs if obj.arr.xgsd == 20]
if not inObj4dem:
......
......@@ -99,7 +99,10 @@ class L2B_object(L2A_object):
# perform spectral homogenization of image data #
#################################################
SpH = SpectralHomogenizer(classifier_rootDir=CFG.path_spechomo_classif, logger=self.logger, CPUs=CFG.CPUs)
from ..processing.multiproc import is_mainprocess
SpH = SpectralHomogenizer(classifier_rootDir=CFG.path_spechomo_classif,
logger=self.logger,
CPUs=CFG.CPUs if is_mainprocess() else 1)
if method == 'LI' or CFG.datasetid_spectral_ref is None:
# linear interpolation (if intended by user or in case of custom spectral characteristics of target sensor)
......
......@@ -502,7 +502,7 @@ class Cloud_Mask_Creator(object):
self.GMS_obj.logger.info("Calculating cloud mask based on '%s' algorithm..." % self.algorithm)
if self.algorithm == 'FMASK':
if re.search('Landsat', self.GMS_obj.satellite, re.I):
if re.search(r'Landsat', self.GMS_obj.satellite, re.I):
FMR = FMASK_Runner_Landsat(self.GMS_obj.path_archive, self.GMS_obj.satellite)
else:
......
......@@ -112,7 +112,7 @@ class GEOPROCESSING(object):
# '/vsitar' '.gz': '/vsigzip'}
p1 = [geodata.split(i)[0] + i for i in ['.zip', '.tar', '.tar.gz', '.gz', '.tgz']
if len(geodata.split(i)) > 1 and geodata.split(i)[1].startswith('/')][0]
path2check = os.path.abspath('.' + re.search('/vsi[\s\S]*(/[\s\S,.]*)', p1, re.I).group(1))
path2check = os.path.abspath('.' + re.search(r'/vsi[\s\S]*(/[\s\S,.]*)', p1, re.I).group(1))
assert os.path.exists(path2check), "ERROR: data %s does not exist!" % path2check
assert self.inDs is not None, "ERROR: Could not open %s!" % self.filename
elif isinstance(geodata, gdal.Dataset):
......@@ -704,7 +704,7 @@ class GEOPROCESSING(object):
with open(os.path.splitext(path_output)[0] + '.hdr', 'r') as inF:
lines = inF.readlines()
outContent = ''.join([i for i in lines if not re.search('map info', i, re.I)])
outContent = ''.join([i for i in lines if not re.search(r'map info', i, re.I)])
with open(os.path.splitext(path_output)[0] + '.hdr', 'w') as outF:
outF.write(outContent)
......@@ -718,7 +718,7 @@ class GEOPROCESSING(object):
#
# with open(os.path.splitext(path_output)[0] + '.hdr', 'r') as inF:
# lines = inF.readlines()
# outContent = ''.join([line if not re.search('coordinate system string', line, re.I) else
# outContent = ''.join([line if not re.search(r'coordinate system string', line, re.I) else
# 'coordinate system string = %s' % self.projection for line in lines])
#
# with open(os.path.splitext(path_output)[0] + '.hdr', 'w') as outF:
......
......@@ -604,9 +604,9 @@ def get_filename_by_entityID(conn_DB, entityid, satellite):
:param satellite: <str> satellite name to which the entity ID is belonging
"""
if re.search('Landsat', satellite, re.I):
if re.search(r'Landsat', satellite, re.I):
filename = '%s.tar.gz' % entityid
elif re.search('Sentinel-2', satellite, re.I):
elif re.search(r'Sentinel-2', satellite, re.I):
filename = get_info_from_postgreSQLdb(conn_DB, 'scenes', ['filename'],
{'entityid': entityid}, records2fetch=1)[0][0]
else:
......@@ -794,12 +794,18 @@ class GMS_JOB(object):
if isinstance(datadict['filenames'], str) and datadict['filenames'].endswith('.csv'):
datadict['filenames'] = None # TODO implement csv reader here
raise NotImplementedError
else:
temp_gdf = GeoDataFrame(datadict, columns=['satellite', 'sensor', 'filenames'])
if re.search('Landsat-7', datadict['satellite'], re.I) and re.search('ETM+', datadict['sensor'], re.I):
if re.search(r'Landsat-7', datadict['satellite'], re.I) and \
re.search(r'ETM+', datadict['sensor'], re.I):
from .helper_functions import Landsat_entityID_decrypter as LED
def get_L7_sensor(fN): return LED(fN.split('.tar.gz')[0]).sensorIncSLC
def get_L7_sensor(fN):
return LED(fN.split('.tar.gz')[0]).sensorIncSLC
temp_gdf['sensor'] = list(temp_gdf['filenames'].map(get_L7_sensor))
all_gdfs.append(temp_gdf)
......@@ -1428,7 +1434,7 @@ def archive_exists_on_fileserver(conn_DB, entityID):
archive_fold = os.path.join(CFG.path_archive, satellite, sensor)
assert os.path.exists(archive_fold), 'Archive folder not found: %s.' % archive_fold
if re.search('Landsat', satellite, re.I):
if re.search(r'Landsat', satellite, re.I):
exists = os.path.exists(os.path.join(archive_fold, entityID + '.tar.gz'))
else:
raise NotImplementedError
......
......@@ -52,7 +52,7 @@ def get_GMS_sensorcode(GMS_id):
# type: (GMS_identifier) -> str
Satellite, Sensor, Subsystem = (GMS_id.satellite, GMS_id.sensor, GMS_id.subsystem)
Sensor = Sensor[:-1] if re.match('SPOT', Satellite, re.I) and Sensor[-1] not in ['1', '2'] else Sensor
Sensor = Sensor[:-1] if re.match(r'SPOT', Satellite, re.I) and Sensor[-1] not in ['1', '2'] else Sensor
meta_sensorcode = Satellite + '_' + Sensor + ('_' + Subsystem if Subsystem not in ["", None] else "")
sensorcode_dic = {
'ALOS_AVNIR-2': 'AVNIR-2',
......
......@@ -311,8 +311,8 @@ def get_path_srf_file(GMS_id, bandname=''):
"""
satellite, sensor = GMS_id.satellite, GMS_id.sensor
satellite = 'RapidEye' if re.match('RapidEye', satellite, re.I) else satellite
sensor = sensor[:-1] if re.match('SPOT', satellite, re.I) and sensor[-1] not in ['1', '2'] else sensor
satellite = 'RapidEye' if re.match(r'RapidEye', satellite, re.I) else satellite
sensor = sensor[:-1] if re.match(r'SPOT', satellite, re.I) and sensor[-1] not in ['1', '2'] else sensor
filename = 'band_%s' % bandname if bandname else ''
return os.path.join(CFG.path_SRFs, satellite, sensor, filename)
......@@ -325,8 +325,8 @@ def get_path_snr_model(GMS_id):
"""
satellite, sensor = (GMS_id.satellite, GMS_id.sensor)
satellite = 'RapidEye' if re.match('RapidEye', satellite, re.I) else satellite
sensor = sensor[:-1] if re.match('SPOT', satellite, re.I) and sensor[-1] not in ['1', '2'] else sensor
satellite = 'RapidEye' if re.match(r'RapidEye', satellite, re.I) else satellite
sensor = sensor[:-1] if re.match(r'SPOT', satellite, re.I) and sensor[-1] not in ['1', '2'] else sensor
return os.path.join(CFG.path_SNR_models, satellite, sensor, 'SNR_model.csv')
......
......@@ -83,7 +83,7 @@ class SpatialIndexMediatorServer:
running = 'is running' in outputStr
# get PID
_process_id = re.search('with pid ([\d]*)', outputStr)
_process_id = re.search(r'with pid ([\d]*)', outputStr)
if _process_id and _process_id.group(1):
process_id = int(_process_id.group(1))
else:
......@@ -108,7 +108,7 @@ class SpatialIndexMediatorServer:
def stop(self):
outputStr = self._communicate('stop')
if outputStr == 'success' or re.search('index-mediator-server stopped', outputStr, re.I):
if outputStr == 'success' or re.search(r'index-mediator-server stopped', outputStr, re.I):
return 'stopped'
else:
warnings.warn("\nStopping Spatial Index Mediator Server failed with message '%s'!"
......
......@@ -375,7 +375,7 @@ class GMS_object(object):
def georef(self):
"""Returns True if the current dataset can serve as spatial reference."""
return True if self.image_type == 'RSD' and re.search('OLI', self.sensor, re.I) else False
return True if self.image_type == 'RSD' and re.search(r'OLI', self.sensor, re.I) else False
@property
def coreg_needed(self):
......@@ -1159,7 +1159,7 @@ class GMS_object(object):
# apply cloud mask to image data and all products derived from image data
# (only if image data represents BOA-Ref and cloud areas are not to be filled with TOA-Ref)
if re.search('BOA_Reflectance', GMS_obj_merged.MetaObj.PhysUnit, re.I) and not CFG.ac_fillnonclear_areas:
if re.search(r'BOA_Reflectance', GMS_obj_merged.MetaObj.PhysUnit, re.I) and not CFG.ac_fillnonclear_areas:
# fill non-clear areas with no data values (for all bands)
for pixVal in nonclear_pixVals:
mask_nonclear = GMS_obj_merged.mask_clouds[:] == pixVal
......@@ -1210,7 +1210,7 @@ class GMS_object(object):
@classmethod
def from_tiles(cls, list_GMS_tiles):
# type: (list) -> GMS_object
"""Merge separate GMS objects with different spatial coverage but belonging to the same scene-ID to ONE GMS object.
"""Merge separate GMS objects with different spatial coverage but belonging to one scene-ID to ONE GMS object.
:param list_GMS_tiles: <list> of GMS objects that have been created by cut_GMS_obj_into_blocks()
"""
......
This diff is collapsed.
......@@ -617,7 +617,7 @@ class JobConfig(object):
ds["entity_ID"] = row["entityid"]
ds["filename"] = row["filename"]
ds['sensor'] = 'ETM+' if re.search('ETM+', ds['sensor']) else ds['sensor']
ds['sensor'] = 'ETM+' if re.search(r'ETM+', ds['sensor']) else ds['sensor']
if self.skip_thermal and ds['subsystem'] == 'TIR':
continue # removes ASTER TIR in case of skip_thermal
ds['subsystem'] = '' if ds['subsystem'] is None else ds['subsystem']
......@@ -625,17 +625,17 @@ class JobConfig(object):
if self.skip_pan and ds['sensormode'] == 'P':
continue # removes e.g. SPOT PAN in case of skip_pan
if re.search("Sentinel-2A", ds['satellite'], re.I):
if re.search(r"Sentinel-2A", ds['satellite'], re.I):
for subsystem in ['S2A10', 'S2A20', 'S2A60']:
sub_ds = ds.copy()
sub_ds['subsystem'] = subsystem
data_list.append(sub_ds)
elif re.search("Sentinel-2B", ds['satellite'], re.I):
elif re.search(r"Sentinel-2B", ds['satellite'], re.I):
for subsystem in ['S2B10', 'S2B20', 'S2B60']:
sub_ds = ds.copy()
sub_ds['subsystem'] = subsystem
data_list.append(sub_ds)
elif re.search("Terra", ds['satellite'], re.I):
elif re.search(r"Terra", ds['satellite'], re.I):
for subsystem in ['VNIR1', 'VNIR2', 'SWIR', 'TIR']:
sub_ds = ds.copy()
sub_ds['subsystem'] = subsystem
......
......@@ -21,7 +21,8 @@
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
from multiprocessing import Pool
from multiprocessing import Pool, current_process
from itertools import chain
from ..options.config import GMS_config as CFG
......@@ -93,3 +94,9 @@ def imap_unordered(func, args, CPUs=None, flatten_output=False):
return list(ch)
else:
return list(results)
def is_mainprocess():
# type: () -> bool
"""Return True if the current process is the main process and False if it is a multiprocessing child process."""
return current_process().name == 'MainProcess'
......@@ -180,7 +180,7 @@ class ProcessController(object):
AllWrittenProcL_dueLog = []
else:
logfile = open(path_log, 'r').read()
AllWrittenProcL_dueLog = re.findall(":*(\S*\s*) data successfully saved.", logfile, re.I)
AllWrittenProcL_dueLog = re.findall(r":*(\S*\s*) data successfully saved.", logfile, re.I)
if not AllWrittenProcL_dueLog and path_logfile == path_logfile_merged_ss: # AllWrittenProcL_dueLog = []
self.logger.info('%s: According to logfile no completely processed data exist at any '
'processing level. Dataset has to be reprocessed.' % dataset['entity_ID'])
......
......@@ -2,6 +2,7 @@ py_tools_ds>=0.12.4
geoarray>=0.8.0
arosics>=0.8.5
git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git
git+https://gitext.gfz-potsdam.de/geomultisens/spechomo.git
matplotlib
numpy
scikit-learn
......
......@@ -41,6 +41,7 @@ requirements = [
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.8.0', 'arosics>=0.8.7', 'six', 'tqdm', 'jsmin', 'cerberus',
'nested_dict', 'openpyxl', 'timeout_decorator', 'redis', 'retools', 'redis-semaphore', 'psutil',
'spechomo @ git+https://gitext.gfz-potsdam.de/geomultisens/spechomo#egg=spechomo'
# spectral<0.16 has some problems with writing signed integer 8bit data
# fmask # conda install -c conda-forge python-fmask
# 'pyhdf', # conda install --yes -c conda-forge pyhdf
......@@ -78,6 +79,8 @@ setup(
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
],
test_suite='tests',
tests_require=test_requirements
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment