Commit 2d2488c4 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Revised output and exception handling. Revised code to get rid of code duplicates. Bug fixes.


Signed-off-by: Daniel Scheffler's avatarDaniel Scheffler <danschef@gfz-potsdam.de>
parent f13dab3d
...@@ -2,6 +2,14 @@ ...@@ -2,6 +2,14 @@
History History
======= =======
0.6.1 (2021-06-18)
------------------
* Revised output and exception handling.
* Revised code to get rid of code duplicates.
* Small bug fixes.
0.6.0 (2021-06-16) 0.6.0 (2021-06-16)
------------------ ------------------
......
...@@ -33,6 +33,7 @@ from multiprocessing import cpu_count ...@@ -33,6 +33,7 @@ from multiprocessing import cpu_count
from threading import Thread from threading import Thread
from queue import Queue from queue import Queue
from subprocess import Popen, PIPE from subprocess import Popen, PIPE
from glob import glob
from qgis.core import ( from qgis.core import (
QgsProcessingAlgorithm, QgsProcessingAlgorithm,
...@@ -42,7 +43,8 @@ from qgis.core import ( ...@@ -42,7 +43,8 @@ from qgis.core import (
QgsProcessingParameterBoolean, QgsProcessingParameterBoolean,
QgsProcessingParameterDefinition, QgsProcessingParameterDefinition,
QgsProcessingParameterRasterLayer, QgsProcessingParameterRasterLayer,
QgsProcessingParameterEnum QgsProcessingParameterEnum,
NULL
) )
from .version import __version__ from .version import __version__
...@@ -450,6 +452,29 @@ class _EnPTBaseAlgorithm(QgsProcessingAlgorithm): ...@@ -450,6 +452,29 @@ class _EnPTBaseAlgorithm(QgsProcessingAlgorithm):
def helpUrl(*args, **kwargs): def helpUrl(*args, **kwargs):
return 'https://enmap.git-pages.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/' return 'https://enmap.git-pages.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/'
@staticmethod
def _get_preprocessed_parameters(parameters):
# replace Enum parameters with corresponding strings (not needed in case of unittest)
for n, opts in [
('output_format', {0: 'GTiff', 1: 'ENVI'}),
('mode_ac', {0: 'land', 1: 'water', 2: 'combined'}),
('deadpix_P_algorithm', {0: 'spectral', 1: 'spatial'}),
('deadpix_P_interp_spectral', {0: 'linear', 1: 'bilinear', 2: 'cubic', 3: 'spline'}),
('deadpix_P_interp_spatial', {0: 'linear', 1: 'bilinear', 2: 'cubic', 3: 'spline'}),
('ortho_resampAlg', {0: 'nearest', 1: 'bilinear', 2: 'gauss'}),
('vswir_overlap_algorithm', {0: 'order_by_wvl', 1: 'average', 2: 'vnir_only', 3: 'swir_only'}),
('target_projection_type', {0: 'UTM', 1: 'Geographic'}),
]:
if isinstance(parameters[n], int):
parameters[n] = opts[parameters[n]]
# remove all parameters not to be forwarded to the EnPT CLI
parameters = {k: v for k, v in parameters.items()
if k not in ['anaconda_root']
and v not in [None, NULL, 'NULL', '']}
return parameters
@staticmethod @staticmethod
def _run_cmd(cmd, qgis_feedback=None, **kwargs): def _run_cmd(cmd, qgis_feedback=None, **kwargs):
"""Execute external command and get its stdout, exitcode and stderr. """Execute external command and get its stdout, exitcode and stderr.
...@@ -499,9 +524,65 @@ class _EnPTBaseAlgorithm(QgsProcessingAlgorithm): ...@@ -499,9 +524,65 @@ class _EnPTBaseAlgorithm(QgsProcessingAlgorithm):
if source.name == stdout_qname: if source.name == stdout_qname:
qgis_feedback.pushInfo(linestr) qgis_feedback.pushInfo(linestr)
if source.name == stderr_qname: elif source.name == stderr_qname:
qgis_feedback.reportError(linestr)
else:
qgis_feedback.reportError(linestr) qgis_feedback.reportError(linestr)
exitcode = process.poll() exitcode = process.poll()
return exitcode return exitcode
def _handle_results(self, parameters: dict, feedback, exitcode: int) -> dict:
success = False
if exitcode:
feedback.reportError("\n" +
"=" * 60 +
"\n" +
"An exception occurred. Processing failed.")
# list output dir
if 'output_dir' in parameters:
outdir = parameters['output_dir']
outraster_matches = \
glob(os.path.join(outdir, '*', '*SPECTRAL_IMAGE.TIF')) or \
glob(os.path.join(outdir, '*', '*SPECTRAL_IMAGE.bsq'))
outraster = outraster_matches[0] if len(outraster_matches) > 0 else None
if os.path.isdir(outdir):
if os.listdir(outdir):
feedback.pushInfo("The output folder '%s' contains:\n" % outdir)
feedback.pushCommandInfo('\n'.join([os.path.basename(f) for f in os.listdir(outdir)]) + '\n')
if outraster:
subdir = os.path.dirname(outraster_matches[0])
feedback.pushInfo(subdir)
feedback.pushInfo("...where the folder '%s' contains:\n" % os.path.split(subdir)[-1])
feedback.pushCommandInfo('\n'.join(sorted([os.path.basename(f)
for f in os.listdir(subdir)])) + '\n')
success = True
else:
feedback.reportError("No output raster was written.")
else:
feedback.reportError("The output folder is empty.")
else:
feedback.reportError("No output folder created.")
# return outputs
if success:
return {
'success': True,
self.P_OUTPUT_RASTER: outraster,
# self.P_OUTPUT_VECTOR: parameters[self.P_OUTPUT_RASTER],
# self.P_OUTPUT_FILE: parameters[self.P_OUTPUT_RASTER],
self.P_OUTPUT_FOLDER: outdir
}
else:
return {'success': False}
else:
feedback.pushInfo('The output was skipped according to user setting.')
return {'success': True}
...@@ -26,7 +26,6 @@ ...@@ -26,7 +26,6 @@
import os import os
from pkgutil import find_loader from pkgutil import find_loader
from glob import glob
from qgis.core import \ from qgis.core import \
(QgsProcessingContext, (QgsProcessingContext,
...@@ -67,33 +66,12 @@ class EnPTAlgorithm(_EnPTBaseAlgorithm): ...@@ -67,33 +66,12 @@ class EnPTAlgorithm(_EnPTBaseAlgorithm):
return enpt_env return enpt_env
def processAlgorithm(self, parameters, context, feedback): def processAlgorithm(self, parameters: dict, context: QgsProcessingContext, feedback: QgsProcessingFeedback):
assert isinstance(parameters, dict)
assert isinstance(context, QgsProcessingContext)
assert isinstance(feedback, QgsProcessingFeedback)
if not find_loader('enpt'): if not find_loader('enpt'):
raise ImportError("enpt", "EnPT must be installed into the QGIS Python environment " raise ImportError("enpt", "EnPT must be installed into the QGIS Python environment "
"when calling 'EnPTAlgorithm'.") "when calling 'EnPTAlgorithm'.")
# replace Enum parameters with corresponding strings (not needed in case of unittest) parameters = self._get_preprocessed_parameters(parameters)
for n, opts in [
('output_format', {0: 'GTiff', 1: 'ENVI'}),
('mode_ac', {0: 'land', 1: 'water', 2: 'combined'}),
('deadpix_P_algorithm', {0: 'spectral', 1: 'spatial'}),
('deadpix_P_interp_spectral', {0: 'linear', 1: 'bilinear', 2: 'cubic', 3: 'spline'}),
('deadpix_P_interp_spatial', {0: 'linear', 1: 'bilinear', 2: 'cubic', 3: 'spline'}),
('ortho_resampAlg', {0: 'nearest', 1: 'bilinear', 2: 'gauss'}),
('vswir_overlap_algorithm', {0: 'order_by_wvl', 1: 'average', 2: 'vnir_only', 3: 'swir_only'}),
('target_projection_type', {0: 'UTM', 1: 'Geographic'}),
]:
if isinstance(parameters[n], int):
parameters[n] = opts[parameters[n]]
# remove all parameters not to be forwarded to the EnPT CLI
parameters = {k: v for k, v in parameters.items()
if k not in ['anaconda_root']
and v not in [None, NULL, 'NULL', '']}
# print parameters and console call to log # print parameters and console call to log
# for key in sorted(parameters): # for key in sorted(parameters):
...@@ -115,36 +93,8 @@ class EnPTAlgorithm(_EnPTBaseAlgorithm): ...@@ -115,36 +93,8 @@ class EnPTAlgorithm(_EnPTBaseAlgorithm):
feedback.pushInfo("The log messages of the EnMAP processing tool are written to the *.log file " feedback.pushInfo("The log messages of the EnMAP processing tool are written to the *.log file "
"in the specified output folder.") "in the specified output folder.")
self._run_cmd(f"enpt {keyval_str}", exitcode = self._run_cmd(f"enpt {keyval_str}",
qgis_feedback=feedback, qgis_feedback=feedback,
env=enpt_env env=enpt_env)
)
return self._handle_results(parameters, feedback, exitcode)
# list output dir
if 'output_dir' in parameters:
outdir = parameters['output_dir']
outraster_matches = glob(os.path.join(outdir, '*', '*SPECTRAL_IMAGE.GEOTIFF'))
outraster = outraster_matches[0] if len(outraster_matches) > 0 else None
feedback.pushInfo("The output folder '%s' contains:\n" % outdir)
feedback.pushCommandInfo('\n'.join([os.path.basename(f) for f in os.listdir(outdir)]) + '\n')
if outraster:
subdir = os.path.dirname(outraster_matches[0])
feedback.pushInfo("...where the folder '%s' contains:\n" % os.path.dirname(subdir))
feedback.pushCommandInfo('\n'.join([os.path.basename(f) for f in os.listdir(subdir)]) + '\n')
# return outputs
return {
'success': True,
self.P_OUTPUT_RASTER: outraster,
# self.P_OUTPUT_VECTOR: parameters[self.P_OUTPUT_RASTER],
# self.P_OUTPUT_FILE: parameters[self.P_OUTPUT_RASTER],
self.P_OUTPUT_FOLDER: outdir
}
else:
feedback.pushInfo('The output was skipped according to user setting.')
# return outputs
return {'success': True}
...@@ -26,7 +26,6 @@ ...@@ -26,7 +26,6 @@
import os import os
from subprocess import check_output, CalledProcessError from subprocess import check_output, CalledProcessError
from glob import glob
from qgis.core import \ from qgis.core import \
(QgsProcessingContext, (QgsProcessingContext,
QgsProcessingFeedback, QgsProcessingFeedback,
...@@ -144,11 +143,7 @@ class ExternalEnPTAlgorithm(_EnPTBaseAlgorithm): ...@@ -144,11 +143,7 @@ class ExternalEnPTAlgorithm(_EnPTBaseAlgorithm):
return enpt_env return enpt_env
def processAlgorithm(self, parameters, context, feedback): def processAlgorithm(self, parameters: dict, context: QgsProcessingContext, feedback: QgsProcessingFeedback):
assert isinstance(parameters, dict)
assert isinstance(context, QgsProcessingContext)
assert isinstance(feedback, QgsProcessingFeedback)
anaconda_root = self._locate_EnPT_Anaconda_environment(parameters[self.P_anaconda_root]) anaconda_root = self._locate_EnPT_Anaconda_environment(parameters[self.P_anaconda_root])
feedback.pushInfo('Found Anaconda installation at %s.' % anaconda_root) feedback.pushInfo('Found Anaconda installation at %s.' % anaconda_root)
...@@ -167,23 +162,7 @@ class ExternalEnPTAlgorithm(_EnPTBaseAlgorithm): ...@@ -167,23 +162,7 @@ class ExternalEnPTAlgorithm(_EnPTBaseAlgorithm):
self.P_OUTPUT_FOLDER: '' self.P_OUTPUT_FOLDER: ''
} }
# remove all parameters not to be forwarded to the EnPT CLI parameters = self._get_preprocessed_parameters(parameters)
parameters = {k: v for k, v in parameters.items()
if k not in ['anaconda_root']}
# replace Enum parameters with corresponding strings (not needed in case of unittest)
for n, opts in [
('output_format', {0: 'GTiff', 1: 'ENVI'}),
('mode_ac', {0: 'land', 1: 'water', 2: 'combined'}),
('deadpix_P_algorithm', {0: 'spectral', 1: 'spatial'}),
('deadpix_P_interp_spectral', {0: 'linear', 1: 'bilinear', 2: 'cubic', 3: 'spline'}),
('deadpix_P_interp_spatial', {0: 'linear', 1: 'bilinear', 2: 'cubic', 3: 'spline'}),
('ortho_resampAlg', {0: 'nearest', 1: 'bilinear', 2: 'gauss'}),
('vswir_overlap_algorithm', {0: 'order_by_wvl', 1: 'average', 2: 'vnir_only', 3: 'swir_only'}),
('target_projection_type', {0: 'UTM', 1: 'Geographic'}),
]:
if isinstance(parameters[n], int):
parameters[n] = opts[parameters[n]]
# print parameters and console call to log # print parameters and console call to log
# for key in sorted(parameters): # for key in sorted(parameters):
...@@ -205,35 +184,8 @@ class ExternalEnPTAlgorithm(_EnPTBaseAlgorithm): ...@@ -205,35 +184,8 @@ class ExternalEnPTAlgorithm(_EnPTBaseAlgorithm):
feedback.pushInfo("The log messages of the EnMAP processing tool are written to the *.log file " feedback.pushInfo("The log messages of the EnMAP processing tool are written to the *.log file "
"in the specified output folder.") "in the specified output folder.")
self._run_cmd("%s %s" % (path_enpt_runscript, keyval_str), exitcode = self._run_cmd(f"{path_enpt_runscript} {keyval_str}",
qgis_feedback=feedback, qgis_feedback=feedback,
env=enpt_env) env=enpt_env)
# list output dir
if 'output_dir' in parameters:
outdir = parameters['output_dir']
outraster_matches = glob(os.path.join(outdir, '*', '*SPECTRAL_IMAGE.GEOTIFF'))
outraster = outraster_matches[0] if len(outraster_matches) > 0 else None
feedback.pushInfo("The output folder '%s' contains:\n" % outdir)
feedback.pushCommandInfo('\n'.join([os.path.basename(f) for f in os.listdir(outdir)]) + '\n')
if outraster:
subdir = os.path.dirname(outraster_matches[0])
feedback.pushInfo("...where the folder '%s' contains:\n" % os.path.dirname(subdir))
feedback.pushCommandInfo('\n'.join([os.path.basename(f) for f in os.listdir(subdir)]) + '\n')
# return outputs
return {
'success': True,
self.P_OUTPUT_RASTER: outraster,
# self.P_OUTPUT_VECTOR: parameters[self.P_OUTPUT_RASTER],
# self.P_OUTPUT_FILE: parameters[self.P_OUTPUT_RASTER],
self.P_OUTPUT_FOLDER: outdir
}
else:
feedback.pushInfo('The output was skipped according to user setting.')
# return outputs return self._handle_results(parameters, feedback, exitcode)
return {'success': True}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment