Commit b78871dc authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Fix for job creation issues in case of Landsat ETM+ SLC-ON; Fix for exception...

Fix for job creation issues in case of Landsat ETM+ SLC-ON; Fix for exception during reading of AC options file.
- removed installer of ecmwf-api-client since this is now done in SICOR directly
updated __version__ and __versioninfo__
Former-commit-id: f4702bf9
Former-commit-id: a7cbc08b
parent 8f742fbb
......@@ -2,8 +2,8 @@
__author__ = """Daniel Scheffler"""
__email__ = 'daniel.scheffler@gfz-potsdam.de'
__version__ = '0.4.1'
__versionalias__ = '20170626.01'
__version__ = '0.4.2'
__versionalias__ = '20170630.01'
from . import algorithms
......
......@@ -750,6 +750,7 @@ class AtmCorr(object):
try:
rs_image.logger = self.logger
self.results = ac_gms(rs_image, self.options, logger=self.logger, script=script)
except Exception as e:
# serialialize AC input
if dump_ac_input:
......
......@@ -744,7 +744,7 @@ class GMS_JOB(object):
WHERE scenes.id in (%s)""" % ','.join([str(i) for i in list_sceneIDs]))
gdf = GeoDataFrame(cursor.fetchall(), columns=['sceneid', 'satellite', 'sensor', 'filename'])
gdf['sensor'] = gdf['sensor'].apply(lambda senN: senN if senN != 'ETM+_SLC_OFF' else 'ETM+')
#gdf['sensor'] = gdf['sensor'].apply(lambda senN: senN if senN != 'ETM+_SLC_OFF' else 'ETM+') # FIXME overwriting 'ETM+_SLC_OFF' with 'ETM+' causes _get_validated_sceneInfoGDFs() to fail because the sensorid for ETM+_SLC_OFF cannot be retrieved
gdf = gdf.drop_duplicates()
if gdf.empty:
......@@ -828,6 +828,7 @@ class GMS_JOB(object):
all_satellites, all_sensors = zip(
*[i.split('__') for i in (np.unique(gdf['satellite'] + '__' + gdf['sensor']))])
for satellite, sensor in zip(all_satellites, all_sensors):
cur_gdf = gdf.loc[(gdf['satellite'] == satellite) & (gdf['sensor'] == sensor)]
filenames = list(cur_gdf['filename'])
......@@ -837,6 +838,7 @@ class GMS_JOB(object):
assert len(satID_res), "No satellite named '%s' found in database." % satellite
assert len(senID_res), "No sensor named '%s' found in database." % sensor
# append sceneid and wkb_hex bounds
if 'sceneid' in gdf.columns:
sceneIDs = list(cur_gdf['sceneid'])
......@@ -852,6 +854,7 @@ class GMS_JOB(object):
cur_gdf = cur_gdf.merge(records, on='filename', how="outer", copy=False)
# separate records with valid matches in database from invalid matches (filename not found in database)
gdf_recs = cur_gdf[
cur_gdf.sceneid.notnull()].copy() # creates a copy (needed to be able to apply maps later)
......@@ -866,6 +869,7 @@ class GMS_JOB(object):
all_gdf_recs.append(gdf_recs)
all_gdf_miss.append(gdf_miss)
# merge all dataframes of all satellite-sensor combinations
gdf_recs_compl = GeoDataFrame(pd.concat(all_gdf_recs))
gdf_miss_compl = GeoDataFrame(pd.concat(all_gdf_miss))
......
......@@ -1779,7 +1779,7 @@ def get_LayerBandsAssignment(GMS_identifier, nBands=None, ignore_usecase=False,
if CFG.usecase.conversion_type_optical=='BOA_Ref':
path_ac_options = get_path_ac_options(GMS_identifier)
if path_ac_options and os.path.exists(path_ac_options):
ac_bandNs = get_ac_options(path_ac_options)['AC']['bands'] # FIXME this does not work for L7
ac_bandNs = get_ac_options(path_ac_options, validation=False)['AC']['bands'] # FIXME this does not work for L7 # don't validate because options contain pathes that do not exist on another server
ac_out_LBA = [bN.split('B0')[1] if bN.startswith('B0') else bN.split('B')[1] for bN in ac_bandNs]
LayerBandsAssignment = [i for i in LayerBandsAssignment if i in ac_out_LBA]
else:
......
......@@ -289,7 +289,7 @@ class GMS_object(Dataset):
path_ac_options = PG.get_path_ac_options(self.GMS_identifier)
if path_ac_options and os.path.exists(path_ac_options):
opt_dict = get_ac_options(path_ac_options)
opt_dict = get_ac_options(path_ac_options, validation=False) # don't validate because options contain pathes that do not exist on another server
# update some file paths depending on the current environment
opt_dict['DEM']['fn'] = CFG.job.path_dem_proc_srtm_90m
......
......@@ -22,7 +22,7 @@ test_requirements = requirements+ ['coverage']
setup(
name='geomultisens',
version='0.4.1',
version='0.4.2',
description="GeoMultiSens - Scalable Multi-Sensor Analysis of Remote Sensing Data",
long_description=readme + '\n\n' + history,
author="Daniel Scheffler",
......
......@@ -25,7 +25,6 @@ RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
conda install --yes -c conda-forge olefile=0.44 ; \
conda install --yes xerces-c ; \
conda install --yes -c conda-forge glymur pygrib rasterio pyproj cachetools pyhdf ; \
pip install https://software.ecmwf.int/wiki/download/attachments/56664858/ecmwf-api-client-python.tgz ; \
pip install shapely geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse \
spectral psycopg2 pyorbital pyinstrument geoalchemy2" # must include all the requirements needed to build the docs!
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment