Unverified Commit 83a55fc1 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Split enpt_enmapboxapp.py into separate modules - one on case EnPT is...

Split  enpt_enmapboxapp.py into separate modules - one on case EnPT is installed externally and one in case it is part of the QGIS environment. Added EnPTAlgorithm for the latter case and respective test.
parent bba66f3a
Pipeline #20796 failed with stage
in 1 minute and 25 seconds
......@@ -6,6 +6,8 @@ History
-------------------
* Removed classifiers for Python<=3.5.
* Split enpt_enmapboxapp.py into separate modules - one on case EnPT is installed externally and
one in case it is part of the QGIS environment. Added EnPTAlgorithm for the latter case and respective test.
0.4.7 (2021-01-11)
......
......@@ -25,7 +25,9 @@
"""Top-level package for enpt_enmapboxapp."""
from .version import __version__, __versionalias__ # noqa (E402 + F401)
from .enpt_enmapboxapp import EnMAPBoxApplication, EnPTAlgorithm
from .enpt_enmapboxapp import EnMAPBoxApplication
from .enpt_algorithm import EnPTAlgorithm
from .enpt_external_algorithm import ExternalEnPTAlgorithm
__author__ = """Daniel Scheffler"""
__email__ = 'danschef@gfz-potsdam.de'
......@@ -34,5 +36,6 @@ __all__ = ['__version__',
'__author__',
'__email__',
'EnMAPBoxApplication',
'EnPTAlgorithm'
'EnPTAlgorithm',
'ExternalEnPTAlgorithm'
]
# -*- coding: utf-8 -*-
# enpt_enmapboxapp, A QGIS EnMAPBox plugin providing a GUI for the EnMAP processing tools (EnPT)
#
# Copyright (C) 2019 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
#
# This software was developed within the context of the EnMAP project supported
# by the DLR Space Administration with funds of the German Federal Ministry of
# Economic Affairs and Energy (on the basis of a decision by the German Bundestag:
# 50 EE 1529) and contributions from DLR, GFZ and OHB System AG.
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""This module provides the base class for EnPTAlgorithm and ExternalEnPTAlgorithm."""
import os
from os.path import expanduser
from datetime import date
from multiprocessing import cpu_count
from qgis.core import \
(QgsProcessingAlgorithm,
Qgis,
QgsProcessingParameterFile,
QgsProcessingParameterNumber,
QgsProcessingParameterFolderDestination,
QgsProcessingParameterBoolean,
QgsProcessingParameterString
)
from .version import __version__
class _EnPTBaseAlgorithm(QgsProcessingAlgorithm):
# NOTE: The parameter assignments made here follow the parameter names in enpt/options/options_schema.py
# Input parameters
P_json_config = 'json_config'
P_CPUs = 'CPUs'
P_path_l1b_enmap_image = 'path_l1b_enmap_image'
P_path_l1b_enmap_image_gapfill = 'path_l1b_enmap_image_gapfill'
P_path_dem = 'path_dem'
P_average_elevation = 'average_elevation'
P_output_dir = 'output_dir'
P_working_dir = 'working_dir'
P_n_lines_to_append = 'n_lines_to_append'
P_disable_progress_bars = 'disable_progress_bars'
P_path_earthSunDist = 'path_earthSunDist'
P_path_solar_irr = 'path_solar_irr'
P_scale_factor_toa_ref = 'scale_factor_toa_ref'
P_enable_keystone_correction = 'enable_keystone_correction'
P_enable_vnir_swir_coreg = 'enable_vnir_swir_coreg'
P_path_reference_image = 'path_reference_image'
P_enable_ac = 'enable_ac'
P_auto_download_ecmwf = 'auto_download_ecmwf'
P_enable_ice_retrieval = 'enable_ice_retrieval'
P_enable_cloud_screening = 'enable_cloud_screening'
P_scale_factor_boa_ref = 'scale_factor_boa_ref'
P_run_smile_P = 'run_smile_P'
P_run_deadpix_P = 'run_deadpix_P'
P_deadpix_P_algorithm = 'deadpix_P_algorithm'
P_deadpix_P_interp_spectral = 'deadpix_P_interp_spectral'
P_deadpix_P_interp_spatial = 'deadpix_P_interp_spatial'
P_ortho_resampAlg = 'ortho_resampAlg'
P_vswir_overlap_algorithm = 'vswir_overlap_algorithm'
# # Output parameters
P_OUTPUT_RASTER = 'outraster'
# P_OUTPUT_VECTOR = 'outvector'
# P_OUTPUT_FILE = 'outfile'
P_OUTPUT_FOLDER = 'outfolder'
def group(self):
return 'Pre-Processing'
def groupId(self):
return 'PreProcessing'
def name(self):
return 'EnPTAlgorithm'
def displayName(self):
return 'EnMAP processing tool algorithm (v%s)' % __version__
def createInstance(self, *args, **kwargs):
return type(self)()
@staticmethod
def _get_default_output_dir():
userhomedir = expanduser('~')
default_enpt_dir = \
os.path.join(userhomedir, 'Documents', 'EnPT', 'Output') if os.name == 'nt' else\
os.path.join(userhomedir, 'EnPT', 'Output')
outdir_nocounter = os.path.join(default_enpt_dir, date.today().strftime('%Y%m%d'))
counter = 1
while os.path.isdir('%s__%s' % (outdir_nocounter, counter)):
counter += 1
return '%s__%s' % (outdir_nocounter, counter)
def initAlgorithm(self, configuration=None):
self.addParameter(QgsProcessingParameterFile(
name=self.P_json_config, description='Configuration JSON template file',
behavior=QgsProcessingParameterFile.File, extension='json',
defaultValue=None,
optional=True))
self.addParameter(QgsProcessingParameterNumber(
name=self.P_CPUs,
description='Number of CPU cores to be used for processing',
type=QgsProcessingParameterNumber.Integer,
defaultValue=cpu_count(), minValue=0, maxValue=cpu_count(),
optional=True))
self.addParameter(QgsProcessingParameterFile(
name=self.P_path_l1b_enmap_image,
description='L1B EnMAP image (zip-archive or root directory)'))
self.addParameter(QgsProcessingParameterFile(
name=self.P_path_l1b_enmap_image_gapfill,
description='Adjacent EnMAP L1B image to be used for gap-filling (zip-archive or root directory)',
optional=True))
self.addParameter(QgsProcessingParameterFile(
name=self.P_path_dem,
description='Input path of digital elevation model in map or sensor geometry; GDAL compatible file '
'format \n(must cover the EnMAP L1B data completely if given in map geometry or must have the '
'same \npixel dimensions like the EnMAP L1B data if given in sensor geometry)',
optional=True))
self.addParameter(QgsProcessingParameterNumber(
name=self.P_average_elevation,
description='Average elevation in meters above sea level \n'
'(may be provided if no DEM is available and ignored if DEM is given)',
type=QgsProcessingParameterNumber.Integer,
defaultValue=0,
optional=True))
self.addParameter(QgsProcessingParameterFolderDestination(
name=self.P_output_dir,
description='Output directory where processed data and log files are saved',
defaultValue=self._get_default_output_dir(),
optional=True))
self.addParameter(QgsProcessingParameterFile(
name=self.P_working_dir,
description='Directory to be used for temporary files',
behavior=QgsProcessingParameterFile.Folder,
defaultValue=None,
optional=True))
self.addParameter(QgsProcessingParameterNumber(
name=self.P_n_lines_to_append,
description='Number of lines to be added to the main image [if not given, use the whole imgap]',
type=QgsProcessingParameterNumber.Integer,
defaultValue=None,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_disable_progress_bars,
description='Disable all progress bars during processing',
defaultValue=True,
optional=True))
self.addParameter(QgsProcessingParameterFile(
name=self.P_path_earthSunDist,
description='Input path of the earth sun distance model',
defaultValue=None,
optional=True))
self.addParameter(QgsProcessingParameterFile(
name=self.P_path_solar_irr,
description='Input path of the solar irradiance model',
defaultValue=None,
optional=True))
self.addParameter(QgsProcessingParameterNumber(
name=self.P_scale_factor_toa_ref,
description='Scale factor to be applied to TOA reflectance result',
type=QgsProcessingParameterNumber.Integer,
defaultValue=10000,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_enable_keystone_correction,
description='Keystone correction',
defaultValue=False,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_enable_vnir_swir_coreg,
description='VNIR/SWIR co-registration',
defaultValue=False,
optional=True))
self.addParameter(QgsProcessingParameterFile(
name=self.P_path_reference_image,
description='Reference image for co-registration.',
defaultValue=None,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_enable_ac,
description='Enable atmospheric correction using SICOR algorithm',
defaultValue=True,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_auto_download_ecmwf,
description='Automatically download ECMWF data for atmospheric correction',
defaultValue=False,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_enable_ice_retrieval,
description='Enable ice retrieval (increases accuracy of water vapour retrieval)',
defaultValue=True,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_enable_cloud_screening,
description='Cloud screening during atmospheric correction',
defaultValue=False,
optional=True))
self.addParameter(QgsProcessingParameterNumber(
name=self.P_scale_factor_boa_ref,
description='Scale factor to be applied to BOA reflectance result',
type=QgsProcessingParameterNumber.Integer,
defaultValue=10000,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_run_smile_P,
description='Smile detection and correction (provider smile coefficients are ignored)',
defaultValue=False,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_run_deadpix_P,
description='Dead pixel correction',
defaultValue=True,
optional=True))
self.addParameter(QgsProcessingParameterString(
name=self.P_deadpix_P_algorithm,
description="Algorithm for dead pixel correction ('spectral' or 'spatial')",
defaultValue='spectral',
multiLine=False,
optional=True))
self.addParameter(QgsProcessingParameterString(
name=self.P_deadpix_P_interp_spectral,
description="Spectral interpolation algorithm to be used during dead pixel correction "
"('linear', 'bilinear', 'cubic', 'spline')",
defaultValue='linear',
multiLine=False,
optional=True))
self.addParameter(QgsProcessingParameterString(
name=self.P_deadpix_P_interp_spatial,
description="Spatial interpolation algorithm to be used during dead pixel correction "
"('linear', 'bilinear', 'cubic', 'spline')",
defaultValue='linear',
multiLine=False,
optional=True))
self.addParameter(QgsProcessingParameterString(
name=self.P_ortho_resampAlg,
description="Ortho-rectification resampling algorithm ('nearest', 'bilinear', 'gauss')",
defaultValue='bilinear',
multiLine=False,
optional=True))
self.addParameter(QgsProcessingParameterString(
name=self.P_vswir_overlap_algorithm,
description="Algorithm specifying how to deal with the spectral bands in the VNIR/SWIR spectral overlap "
"region ('order_by_wvl', 'average', 'vnir_only', 'swir_only')",
defaultValue='swir_only',
multiLine=False,
optional=True))
@staticmethod
def shortHelpString(*args, **kwargs):
"""Example:
'<p>Here comes the HTML documentation.</p>' \
'<h3>With Headers...</h3>' \
'<p>and Hyperlinks: <a href="www.google.de">Google</a></p>'
:param args:
:param kwargs:
"""
text = \
'<p>General information about this EnMAP box app can be found ' \
'<a href="https://enmap.git-pages.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/">here</a>' \
'.</p>' \
'<p>Type <i>python enpt_cli.py -h</i> into a shell to get further information about individual ' \
'parameters.</p>'
return text
def helpString(self):
return self.shortHelpString()
@staticmethod
def helpUrl(*args, **kwargs):
return 'https://enmap.git-pages.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/'
# -*- coding: utf-8 -*-
# enpt_enmapboxapp, A QGIS EnMAPBox plugin providing a GUI for the EnMAP processing tools (EnPT)
#
# Copyright (C) 2019 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
#
# This software was developed within the context of the EnMAP project supported
# by the DLR Space Administration with funds of the German Federal Ministry of
# Economic Affairs and Energy (on the basis of a decision by the German Bundestag:
# 50 EE 1529) and contributions from DLR, GFZ and OHB System AG.
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""This module provides the EnPTAlgorithm which is used in case EnPT is installed into QGIS Python environment."""
import os
from pkgutil import find_loader
from glob import glob
from qgis.core import \
(QgsProcessingContext,
QgsProcessingFeedback,
NULL
)
from ._enpt_alg_base import _EnPTBaseAlgorithm
class EnPTAlgorithm(_EnPTBaseAlgorithm):
def processAlgorithm(self, parameters, context, feedback):
assert isinstance(parameters, dict)
assert isinstance(context, QgsProcessingContext)
assert isinstance(feedback, QgsProcessingFeedback)
if not find_loader('enpt'):
raise ImportError("enpt", "EnPT must be installed into the QGIS Python environment "
"when calling 'EnPTAlgorithm'.")
from enpt.options.config import EnPTConfig
from enpt.execution.controller import EnPT_Controller
feedback.pushInfo("The log messages of the EnMAP processing tool are written to the *.log file "
"in the specified output folder.")
# remove all parameters not to be forwarded to the EnPT CLI
parameters = {k: v for k, v in parameters.items()
if k not in ['anaconda_root']
and v not in [None, NULL, 'NULL', '']}
# run EnPT in subprocess that activates the EnPT Anaconda environment
# os.environ['PYTHONUNBUFFERED'] = '1'
# from contextlib import redirect_stdout
CTR = EnPT_Controller(EnPTConfig(**parameters))
CTR.run_all_processors()
# list output dir
outdir = parameters['output_dir']
outraster_matches = glob(os.path.join(outdir, '*', '*SPECTRAL_IMAGE.GEOTIFF'))
outraster = outraster_matches[0] if len(outraster_matches) > 0 else None
feedback.pushInfo("The output folder '%s' contains:\n" % outdir)
feedback.pushCommandInfo('\n'.join([os.path.basename(f) for f in os.listdir(outdir)]) + '\n')
if outraster:
subdir = os.path.dirname(outraster_matches[0])
feedback.pushInfo("...where the folder '%s' contains:\n" % os.path.dirname(subdir))
feedback.pushCommandInfo('\n'.join([os.path.basename(f) for f in os.listdir(subdir)]) + '\n')
# return outputs
return {
'success': True,
self.P_OUTPUT_RASTER: outraster,
# self.P_OUTPUT_VECTOR: parameters[self.P_OUTPUT_RASTER],
# self.P_OUTPUT_FILE: parameters[self.P_OUTPUT_RASTER],
self.P_OUTPUT_FOLDER: outdir
}
This diff is collapsed.
# -*- coding: utf-8 -*-
# enpt_enmapboxapp, A QGIS EnMAPBox plugin providing a GUI for the EnMAP processing tools (EnPT)
#
# Copyright (C) 2019 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
#
# This software was developed within the context of the EnMAP project supported
# by the DLR Space Administration with funds of the German Federal Ministry of
# Economic Affairs and Energy (on the basis of a decision by the German Bundestag:
# 50 EE 1529) and contributions from DLR, GFZ and OHB System AG.
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""This module provides the ExternalEnPTAlgorithm which is used in case EnPT is installed into separate environment."""
import os
import psutil
from subprocess import Popen, PIPE, check_output, CalledProcessError
from threading import Thread
from queue import Queue
from glob import glob
from qgis.core import \
(QgsProcessingContext,
QgsProcessingFeedback,
QgsProcessingParameterFile,
NULL
)
from ._enpt_alg_base import _EnPTBaseAlgorithm
class ExternalEnPTAlgorithm(_EnPTBaseAlgorithm):
# Input parameters
P_anaconda_root = 'anaconda_root'
def initAlgorithm(self, configuration=None):
self.addParameter(QgsProcessingParameterFile(
name=self.P_anaconda_root,
description='Anaconda root directory (which contains the EnPT Python environment in a subdirectory)',
behavior=QgsProcessingParameterFile.Folder,
defaultValue=self._get_default_anaconda_root(),
optional=True))
super().initAlgorithm(configuration=configuration)
@staticmethod
def _get_default_anaconda_root():
if os.getenv('ANACONDA_ROOT') and os.path.exists(os.getenv('ANACONDA_ROOT')):
return os.getenv('ANACONDA_ROOT')
elif os.name == 'nt':
return 'C:\\ProgramData\\Anaconda3'
else:
return '' # FIXME is there a default location in Linux/OSX?
@staticmethod
def _run_cmd(cmd, qgis_feedback=None, **kwargs):
"""Execute external command and get its stdout, exitcode and stderr.
Code based on: https://stackoverflow.com/a/31867499
:param cmd: a normal shell command including parameters
"""
def reader(pipe, queue):
try:
with pipe:
for line in iter(pipe.readline, b''):
queue.put((pipe, line))
finally:
queue.put(None)
process = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, **kwargs)
q = Queue()
Thread(target=reader, args=[process.stdout, q]).start()
Thread(target=reader, args=[process.stderr, q]).start()
# for _ in range(2):
for source, line in iter(q.get, None):
if qgis_feedback.isCanceled():
# qgis_feedback.reportError('CANCELED')
proc2kill = psutil.Process(process.pid)
for proc in proc2kill.children(recursive=True):
proc.kill()
proc2kill.kill()
raise KeyboardInterrupt
linestr = line.decode('latin-1').rstrip()
# print("%s: %s" % (source, linestr))
if source.name == 3:
qgis_feedback.pushInfo(linestr)
if source.name == 4:
qgis_feedback.reportError(linestr)
exitcode = process.poll()
return exitcode
@staticmethod
def _locate_EnPT_Anaconda_environment(user_root):
anaconda_rootdir = None
if user_root and os.path.exists(user_root):
anaconda_rootdir = user_root
elif os.getenv('ANACONDA_ROOT') and os.path.exists(os.getenv('ANACONDA_ROOT')):
anaconda_rootdir = os.getenv('ANACONDA_ROOT')
else:
possPaths = [
'C:\\ProgramData\\Anaconda3',
'C:\\Users\\%s\\Anaconda3' % os.getenv('username')
] if os.name == 'nt' else \
[]
for rootDir in possPaths:
if os.path.exists(rootDir):
anaconda_rootdir = rootDir
if not anaconda_rootdir:
raise NotADirectoryError("No valid Anaconda root directory given - "
"neither via the GUI, nor via the 'ANACONDA_ROOT' environment variable.")
# set ENPT_PYENV_ACTIVATION environment variable
os.environ['ENPT_PYENV_ACTIVATION'] = \
os.path.join(anaconda_rootdir, 'Scripts', 'activate.bat') if os.name == 'nt' else \
os.path.join(anaconda_rootdir, 'bin', 'activate')
if not os.path.exists(os.getenv('ENPT_PYENV_ACTIVATION')):
raise FileNotFoundError(os.getenv('ENPT_PYENV_ACTIVATION'))
return anaconda_rootdir
@staticmethod
def _is_enpt_environment_present(anaconda_rootdir):
return os.path.exists(os.path.join(anaconda_rootdir, 'envs', 'enpt'))
@staticmethod
def _locate_enpt_run_script():
try:
if os.name == 'nt':
# Windows
return check_output('where enpt_run_cmd.bat', shell=True).decode('UTF-8').strip()
# return "D:\\Daten\\Code\\python\\enpt_enmapboxapp\\bin\\enpt_run_cmd.bat"
else:
# Linux / OSX
return check_output('which enpt_run_cmd.sh', shell=True).decode('UTF-8').strip()
# return 'enpt_run_cmd.sh '
except CalledProcessError:
raise EnvironmentError('The EnPT run script could not be found. Please make sure, that enpt_enmapboxapp '