Commit f4702bf9 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Fix for job creation issues in case of Landsat ETM+ SLC-ON; Fix for exception...

Fix for job creation issues in case of Landsat ETM+ SLC-ON; Fix for exception during reading of AC options file.

- removed installer of ecmwf-api-client since this is now done in SICOR directly

updated __version__ and __versioninfo__
parent c796ea9e
Pipeline #275 passed with stage
in 54 seconds
...@@ -2,8 +2,8 @@ ...@@ -2,8 +2,8 @@
__author__ = """Daniel Scheffler""" __author__ = """Daniel Scheffler"""
__email__ = 'daniel.scheffler@gfz-potsdam.de' __email__ = 'daniel.scheffler@gfz-potsdam.de'
__version__ = '0.4.1' __version__ = '0.4.2'
__versionalias__ = '20170626.01' __versionalias__ = '20170630.01'
from . import algorithms from . import algorithms
......
...@@ -750,6 +750,7 @@ class AtmCorr(object): ...@@ -750,6 +750,7 @@ class AtmCorr(object):
try: try:
rs_image.logger = self.logger rs_image.logger = self.logger
self.results = ac_gms(rs_image, self.options, logger=self.logger, script=script) self.results = ac_gms(rs_image, self.options, logger=self.logger, script=script)
except Exception as e: except Exception as e:
# serialialize AC input # serialialize AC input
if dump_ac_input: if dump_ac_input:
......
...@@ -744,7 +744,7 @@ class GMS_JOB(object): ...@@ -744,7 +744,7 @@ class GMS_JOB(object):
WHERE scenes.id in (%s)""" % ','.join([str(i) for i in list_sceneIDs])) WHERE scenes.id in (%s)""" % ','.join([str(i) for i in list_sceneIDs]))
gdf = GeoDataFrame(cursor.fetchall(), columns=['sceneid', 'satellite', 'sensor', 'filename']) gdf = GeoDataFrame(cursor.fetchall(), columns=['sceneid', 'satellite', 'sensor', 'filename'])
gdf['sensor'] = gdf['sensor'].apply(lambda senN: senN if senN != 'ETM+_SLC_OFF' else 'ETM+') #gdf['sensor'] = gdf['sensor'].apply(lambda senN: senN if senN != 'ETM+_SLC_OFF' else 'ETM+') # FIXME overwriting 'ETM+_SLC_OFF' with 'ETM+' causes _get_validated_sceneInfoGDFs() to fail because the sensorid for ETM+_SLC_OFF cannot be retrieved
gdf = gdf.drop_duplicates() gdf = gdf.drop_duplicates()
if gdf.empty: if gdf.empty:
...@@ -828,6 +828,7 @@ class GMS_JOB(object): ...@@ -828,6 +828,7 @@ class GMS_JOB(object):
all_satellites, all_sensors = zip( all_satellites, all_sensors = zip(
*[i.split('__') for i in (np.unique(gdf['satellite'] + '__' + gdf['sensor']))]) *[i.split('__') for i in (np.unique(gdf['satellite'] + '__' + gdf['sensor']))])
for satellite, sensor in zip(all_satellites, all_sensors): for satellite, sensor in zip(all_satellites, all_sensors):
cur_gdf = gdf.loc[(gdf['satellite'] == satellite) & (gdf['sensor'] == sensor)] cur_gdf = gdf.loc[(gdf['satellite'] == satellite) & (gdf['sensor'] == sensor)]
filenames = list(cur_gdf['filename']) filenames = list(cur_gdf['filename'])
...@@ -837,6 +838,7 @@ class GMS_JOB(object): ...@@ -837,6 +838,7 @@ class GMS_JOB(object):
assert len(satID_res), "No satellite named '%s' found in database." % satellite assert len(satID_res), "No satellite named '%s' found in database." % satellite
assert len(senID_res), "No sensor named '%s' found in database." % sensor assert len(senID_res), "No sensor named '%s' found in database." % sensor
# append sceneid and wkb_hex bounds # append sceneid and wkb_hex bounds
if 'sceneid' in gdf.columns: if 'sceneid' in gdf.columns:
sceneIDs = list(cur_gdf['sceneid']) sceneIDs = list(cur_gdf['sceneid'])
...@@ -852,6 +854,7 @@ class GMS_JOB(object): ...@@ -852,6 +854,7 @@ class GMS_JOB(object):
cur_gdf = cur_gdf.merge(records, on='filename', how="outer", copy=False) cur_gdf = cur_gdf.merge(records, on='filename', how="outer", copy=False)
# separate records with valid matches in database from invalid matches (filename not found in database) # separate records with valid matches in database from invalid matches (filename not found in database)
gdf_recs = cur_gdf[ gdf_recs = cur_gdf[
cur_gdf.sceneid.notnull()].copy() # creates a copy (needed to be able to apply maps later) cur_gdf.sceneid.notnull()].copy() # creates a copy (needed to be able to apply maps later)
...@@ -866,6 +869,7 @@ class GMS_JOB(object): ...@@ -866,6 +869,7 @@ class GMS_JOB(object):
all_gdf_recs.append(gdf_recs) all_gdf_recs.append(gdf_recs)
all_gdf_miss.append(gdf_miss) all_gdf_miss.append(gdf_miss)
# merge all dataframes of all satellite-sensor combinations # merge all dataframes of all satellite-sensor combinations
gdf_recs_compl = GeoDataFrame(pd.concat(all_gdf_recs)) gdf_recs_compl = GeoDataFrame(pd.concat(all_gdf_recs))
gdf_miss_compl = GeoDataFrame(pd.concat(all_gdf_miss)) gdf_miss_compl = GeoDataFrame(pd.concat(all_gdf_miss))
......
...@@ -1779,7 +1779,7 @@ def get_LayerBandsAssignment(GMS_identifier, nBands=None, ignore_usecase=False, ...@@ -1779,7 +1779,7 @@ def get_LayerBandsAssignment(GMS_identifier, nBands=None, ignore_usecase=False,
if CFG.usecase.conversion_type_optical=='BOA_Ref': if CFG.usecase.conversion_type_optical=='BOA_Ref':
path_ac_options = get_path_ac_options(GMS_identifier) path_ac_options = get_path_ac_options(GMS_identifier)
if path_ac_options and os.path.exists(path_ac_options): if path_ac_options and os.path.exists(path_ac_options):
ac_bandNs = get_ac_options(path_ac_options)['AC']['bands'] # FIXME this does not work for L7 ac_bandNs = get_ac_options(path_ac_options, validation=False)['AC']['bands'] # FIXME this does not work for L7 # don't validate because options contain pathes that do not exist on another server
ac_out_LBA = [bN.split('B0')[1] if bN.startswith('B0') else bN.split('B')[1] for bN in ac_bandNs] ac_out_LBA = [bN.split('B0')[1] if bN.startswith('B0') else bN.split('B')[1] for bN in ac_bandNs]
LayerBandsAssignment = [i for i in LayerBandsAssignment if i in ac_out_LBA] LayerBandsAssignment = [i for i in LayerBandsAssignment if i in ac_out_LBA]
else: else:
......
...@@ -289,7 +289,7 @@ class GMS_object(Dataset): ...@@ -289,7 +289,7 @@ class GMS_object(Dataset):
path_ac_options = PG.get_path_ac_options(self.GMS_identifier) path_ac_options = PG.get_path_ac_options(self.GMS_identifier)
if path_ac_options and os.path.exists(path_ac_options): if path_ac_options and os.path.exists(path_ac_options):
opt_dict = get_ac_options(path_ac_options) opt_dict = get_ac_options(path_ac_options, validation=False) # don't validate because options contain pathes that do not exist on another server
# update some file paths depending on the current environment # update some file paths depending on the current environment
opt_dict['DEM']['fn'] = CFG.job.path_dem_proc_srtm_90m opt_dict['DEM']['fn'] = CFG.job.path_dem_proc_srtm_90m
......
...@@ -22,7 +22,7 @@ test_requirements = requirements+ ['coverage'] ...@@ -22,7 +22,7 @@ test_requirements = requirements+ ['coverage']
setup( setup(
name='geomultisens', name='geomultisens',
version='0.4.1', version='0.4.2',
description="GeoMultiSens - Scalable Multi-Sensor Analysis of Remote Sensing Data", description="GeoMultiSens - Scalable Multi-Sensor Analysis of Remote Sensing Data",
long_description=readme + '\n\n' + history, long_description=readme + '\n\n' + history,
author="Daniel Scheffler", author="Daniel Scheffler",
......
...@@ -25,7 +25,6 @@ RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \ ...@@ -25,7 +25,6 @@ RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
conda install --yes -c conda-forge olefile=0.44 ; \ conda install --yes -c conda-forge olefile=0.44 ; \
conda install --yes xerces-c ; \ conda install --yes xerces-c ; \
conda install --yes -c conda-forge glymur pygrib rasterio pyproj cachetools pyhdf ; \ conda install --yes -c conda-forge glymur pygrib rasterio pyproj cachetools pyhdf ; \
pip install https://software.ecmwf.int/wiki/download/attachments/56664858/ecmwf-api-client-python.tgz ; \
pip install shapely geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse \ pip install shapely geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse \
spectral psycopg2 pyorbital pyinstrument geoalchemy2" # must include all the requirements needed to build the docs! spectral psycopg2 pyorbital pyinstrument geoalchemy2" # must include all the requirements needed to build the docs!
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment