Commit 0aa41dc1 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'feature/adapt_to_single_env' into 'master'

Split  enpt_enmapboxapp.py into separate modules - one on case EnPT is...

See merge request !13
parents 950aa73b 25ba21bc
Pipeline #24019 failed with stages
in 17 minutes and 1 second
...@@ -2,11 +2,17 @@ ...@@ -2,11 +2,17 @@
History History
======= =======
0.4.8 (coming soon) 0.5.0 (coming soon)
------------------- -------------------
* 'make lint' now additionally prints the log outputs. * 'make lint' now additionally prints the log outputs.
* Replaced deprecated URLs. Fixed 'make lint'. * Replaced deprecated URLs. Fixed 'make lint'.
* Removed classifiers for Python<=3.5.
* Split enpt_enmapboxapp.py into separate modules - one on case EnPT is installed externally and
one in case it is part of the QGIS environment. Added EnPTAlgorithm for the latter case and respective test.
* Adapted new --exclude-patterns parameter of urlchecker.
* The EnPTAlgorithm class now also uses a subcommand to run EnPT to be able to use multiprocessing.
* Updated EnPT entry point.
0.4.7 (2021-01-11) 0.4.7 (2021-01-11)
......
...@@ -66,7 +66,7 @@ lint: ## check style with flake8 ...@@ -66,7 +66,7 @@ lint: ## check style with flake8
urlcheck: ## check for dead URLs urlcheck: ## check for dead URLs
urlchecker check . \ urlchecker check . \
--file-types .py,.rst,.md,.json \ --file-types .py,.rst,.md,.json \
--white-listed-patterns www.enmap.org --exclude-patterns www.enmap.org # certificate checks fail although URLs work
test: ## run tests quickly with the default Python test: ## run tests quickly with the default Python
python setup.py test python setup.py test
......
...@@ -37,11 +37,11 @@ call %ENPT_PYENV_ACTIVATION% enpt ...@@ -37,11 +37,11 @@ call %ENPT_PYENV_ACTIVATION% enpt
:: echo %* :: echo %*
:: run enpt_cli.py with the provided arguments :: run enpt_cli.py with the provided arguments
FOR /F %%i IN ('where enpt_cli.py') do (SET PATH_ENPT_CLI=%%i) FOR /F %%i IN ('where enpt') do (SET PATH_ENPT_CLI=%%i)
echo. echo.
echo _______________________________________ echo _______________________________________
python %PATH_ENPT_CLI% %* %PATH_ENPT_CLI% %*
echo _______________________________________ echo _______________________________________
echo. echo.
...@@ -26,4 +26,4 @@ ...@@ -26,4 +26,4 @@
# activate EnPT Python environment # activate EnPT Python environment
source $ENPT_PYENV_ACTIVATION enpt source $ENPT_PYENV_ACTIVATION enpt
enpt_cli.py "$@" enpt "$@"
...@@ -25,7 +25,9 @@ ...@@ -25,7 +25,9 @@
"""Top-level package for enpt_enmapboxapp.""" """Top-level package for enpt_enmapboxapp."""
from .version import __version__, __versionalias__ # noqa (E402 + F401) from .version import __version__, __versionalias__ # noqa (E402 + F401)
from .enpt_enmapboxapp import EnMAPBoxApplication, EnPTAlgorithm from .enpt_enmapboxapp import EnMAPBoxApplication
from .enpt_algorithm import EnPTAlgorithm
from .enpt_external_algorithm import ExternalEnPTAlgorithm
__author__ = """Daniel Scheffler""" __author__ = """Daniel Scheffler"""
__email__ = 'danschef@gfz-potsdam.de' __email__ = 'danschef@gfz-potsdam.de'
...@@ -34,5 +36,6 @@ __all__ = ['__version__', ...@@ -34,5 +36,6 @@ __all__ = ['__version__',
'__author__', '__author__',
'__email__', '__email__',
'EnMAPBoxApplication', 'EnMAPBoxApplication',
'EnPTAlgorithm' 'EnPTAlgorithm',
'ExternalEnPTAlgorithm'
] ]
# -*- coding: utf-8 -*-
# enpt_enmapboxapp, A QGIS EnMAPBox plugin providing a GUI for the EnMAP processing tools (EnPT)
#
# Copyright (C) 2019 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
#
# This software was developed within the context of the EnMAP project supported
# by the DLR Space Administration with funds of the German Federal Ministry of
# Economic Affairs and Energy (on the basis of a decision by the German Bundestag:
# 50 EE 1529) and contributions from DLR, GFZ and OHB System AG.
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""This module provides the base class for EnPTAlgorithm and ExternalEnPTAlgorithm."""
import os
from os.path import expanduser
import psutil
from datetime import date
from multiprocessing import cpu_count
from threading import Thread
from queue import Queue
from subprocess import Popen, PIPE
from qgis.core import \
(QgsProcessingAlgorithm,
QgsProcessingParameterFile,
QgsProcessingParameterNumber,
QgsProcessingParameterFolderDestination,
QgsProcessingParameterBoolean,
QgsProcessingParameterString,
QgsProcessingParameterDefinition,
QgsProcessingParameterRasterLayer
)
from .version import __version__
class _EnPTBaseAlgorithm(QgsProcessingAlgorithm):
# NOTE: The parameter assignments made here follow the parameter names in enpt/options/options_schema.py
# Input parameters
P_json_config = 'json_config'
P_CPUs = 'CPUs'
P_path_l1b_enmap_image = 'path_l1b_enmap_image'
P_path_l1b_enmap_image_gapfill = 'path_l1b_enmap_image_gapfill'
P_path_dem = 'path_dem'
P_average_elevation = 'average_elevation'
P_output_dir = 'output_dir'
P_working_dir = 'working_dir'
P_n_lines_to_append = 'n_lines_to_append'
P_disable_progress_bars = 'disable_progress_bars'
P_path_earthSunDist = 'path_earthSunDist'
P_path_solar_irr = 'path_solar_irr'
P_scale_factor_toa_ref = 'scale_factor_toa_ref'
P_enable_keystone_correction = 'enable_keystone_correction'
P_enable_vnir_swir_coreg = 'enable_vnir_swir_coreg'
P_path_reference_image = 'path_reference_image'
P_enable_ac = 'enable_ac'
P_auto_download_ecmwf = 'auto_download_ecmwf'
P_enable_ice_retrieval = 'enable_ice_retrieval'
P_enable_cloud_screening = 'enable_cloud_screening'
P_scale_factor_boa_ref = 'scale_factor_boa_ref'
P_run_smile_P = 'run_smile_P'
P_run_deadpix_P = 'run_deadpix_P'
P_deadpix_P_algorithm = 'deadpix_P_algorithm'
P_deadpix_P_interp_spectral = 'deadpix_P_interp_spectral'
P_deadpix_P_interp_spatial = 'deadpix_P_interp_spatial'
P_ortho_resampAlg = 'ortho_resampAlg'
P_vswir_overlap_algorithm = 'vswir_overlap_algorithm'
# # Output parameters
P_OUTPUT_RASTER = 'outraster'
# P_OUTPUT_VECTOR = 'outvector'
# P_OUTPUT_FILE = 'outfile'
P_OUTPUT_FOLDER = 'outfolder'
def group(self):
return 'Pre-Processing'
def groupId(self):
return 'PreProcessing'
def name(self):
return 'EnPTAlgorithm'
def displayName(self):
return 'EnMAP processing tool algorithm (v%s)' % __version__
def createInstance(self, *args, **kwargs):
return type(self)()
@staticmethod
def _get_default_output_dir():
userhomedir = expanduser('~')
default_enpt_dir = \
os.path.join(userhomedir, 'Documents', 'EnPT', 'Output') if os.name == 'nt' else\
os.path.join(userhomedir, 'EnPT', 'Output')
outdir_nocounter = os.path.join(default_enpt_dir, date.today().strftime('%Y%m%d'))
counter = 1
while os.path.isdir('%s__%s' % (outdir_nocounter, counter)):
counter += 1
return '%s__%s' % (outdir_nocounter, counter)
def addParameter(self, param, *args, advanced=False, **kwargs):
"""Add a parameter to the QgsProcessingAlgorithm.
This overrides the parent method to make it accept an 'advanced' parameter.
:param param: the parameter to be added
:param args: arguments to be passed to the parent method
:param advanced: whether the parameter should be flagged as 'advanced'
:param kwargs: keyword arguments to be passed to the parent method
"""
if advanced:
param.setFlags(param.flags() | QgsProcessingParameterDefinition.FlagAdvanced)
super(_EnPTBaseAlgorithm, self).addParameter(param, *args, **kwargs)
def initAlgorithm(self, configuration=None):
self.addParameter(QgsProcessingParameterFile(
name=self.P_json_config, description='Configuration JSON template file',
behavior=QgsProcessingParameterFile.File, extension='json',
defaultValue=None,
optional=True))
self.addParameter(QgsProcessingParameterNumber(
name=self.P_CPUs,
description='Number of CPU cores to be used for processing',
type=QgsProcessingParameterNumber.Integer,
defaultValue=cpu_count(), minValue=0, maxValue=cpu_count(),
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterFile(
name=self.P_path_l1b_enmap_image,
description='EnMAP Level-1B image (zip-archive or root directory)'))
self.addParameter(QgsProcessingParameterFile(
name=self.P_path_l1b_enmap_image_gapfill,
description='Adjacent EnMAP Level-1B image to be used for gap-filling (zip-archive or root directory)',
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterRasterLayer(
name=self.P_path_dem,
description='Input path of digital elevation model in map or sensor geometry; GDAL compatible file '
'format \n(must cover the EnMAP L1B data completely if given in map geometry or must have the '
'same \npixel dimensions like the EnMAP L1B data if given in sensor geometry)',
optional=True))
self.addParameter(QgsProcessingParameterNumber(
name=self.P_average_elevation,
description='Average elevation in meters above sea level \n'
'(may be provided if no DEM is available and ignored if DEM is given)',
type=QgsProcessingParameterNumber.Integer,
defaultValue=0,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterFolderDestination(
name=self.P_output_dir,
description='Output directory where processed data and log files are saved',
defaultValue=self._get_default_output_dir(),
optional=True))
self.addParameter(QgsProcessingParameterFile(
name=self.P_working_dir,
description='Directory to be used for temporary files',
behavior=QgsProcessingParameterFile.Folder,
defaultValue=None,
optional=True))
self.addParameter(QgsProcessingParameterNumber(
name=self.P_n_lines_to_append,
description='Number of lines to be added to the main image [if not given, use the whole imgap]',
type=QgsProcessingParameterNumber.Integer,
defaultValue=None,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_disable_progress_bars,
description='Disable all progress bars during processing',
defaultValue=True,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterFile(
name=self.P_path_earthSunDist,
description='Input path of the earth sun distance model',
defaultValue=None,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterFile(
name=self.P_path_solar_irr,
description='Input path of the solar irradiance model',
defaultValue=None,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterNumber(
name=self.P_scale_factor_toa_ref,
description='Scale factor to be applied to TOA reflectance result',
type=QgsProcessingParameterNumber.Integer,
defaultValue=10000,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_enable_keystone_correction,
description='Keystone correction',
defaultValue=False,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_enable_vnir_swir_coreg,
description='VNIR/SWIR co-registration',
defaultValue=False,
optional=True))
self.addParameter(QgsProcessingParameterRasterLayer(
name=self.P_path_reference_image,
description='Reference image for co-registration.',
defaultValue=None,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_enable_ac,
description='Enable atmospheric correction using SICOR algorithm',
defaultValue=True,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_auto_download_ecmwf,
description='Automatically download ECMWF data for atmospheric correction',
defaultValue=False,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_enable_ice_retrieval,
description='Enable ice retrieval (increases accuracy of water vapour retrieval)',
defaultValue=True,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_enable_cloud_screening,
description='Cloud screening during atmospheric correction',
defaultValue=False,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterNumber(
name=self.P_scale_factor_boa_ref,
description='Scale factor to be applied to BOA reflectance result',
type=QgsProcessingParameterNumber.Integer,
defaultValue=10000,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_run_smile_P,
description='Smile detection and correction (provider smile coefficients are ignored)',
defaultValue=False,
optional=True))
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_run_deadpix_P,
description='Dead pixel correction',
defaultValue=True,
optional=True))
self.addParameter(QgsProcessingParameterString(
name=self.P_deadpix_P_algorithm,
description="Algorithm for dead pixel correction ('spectral' or 'spatial')",
defaultValue='spectral',
multiLine=False,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterString(
name=self.P_deadpix_P_interp_spectral,
description="Spectral interpolation algorithm to be used during dead pixel correction "
"('linear', 'bilinear', 'cubic', 'spline')",
defaultValue='linear',
multiLine=False,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterString(
name=self.P_deadpix_P_interp_spatial,
description="Spatial interpolation algorithm to be used during dead pixel correction "
"('linear', 'bilinear', 'cubic', 'spline')",
defaultValue='linear',
multiLine=False,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterString(
name=self.P_ortho_resampAlg,
description="Ortho-rectification resampling algorithm ('nearest', 'bilinear', 'gauss')",
defaultValue='bilinear',
multiLine=False,
optional=True),
advanced=True)
self.addParameter(QgsProcessingParameterString(
name=self.P_vswir_overlap_algorithm,
description="Algorithm specifying how to deal with the spectral bands in the VNIR/SWIR spectral overlap "
"region ('order_by_wvl', 'average', 'vnir_only', 'swir_only')",
defaultValue='swir_only',
multiLine=False,
optional=True),
advanced=True)
@staticmethod
def shortHelpString(*args, **kwargs):
"""Example:
'<p>Here comes the HTML documentation.</p>' \
'<h3>With Headers...</h3>' \
'<p>and Hyperlinks: <a href="www.google.de">Google</a></p>'
:param args:
:param kwargs:
"""
text = \
'<p>General information about this EnMAP box app can be found ' \
'<a href="https://enmap.git-pages.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/">here</a>' \
'.</p>' \
'<p>Type <i>enpt -h</i> into a shell to get further information about individual ' \
'parameters.</p>'
return text
def helpString(self):
return self.shortHelpString()
@staticmethod
def helpUrl(*args, **kwargs):
return 'https://enmap.git-pages.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/'
@staticmethod
def _run_cmd(cmd, qgis_feedback=None, **kwargs):
"""Execute external command and get its stdout, exitcode and stderr.
Code based on: https://stackoverflow.com/a/31867499
:param cmd: a normal shell command including parameters
"""
def reader(pipe, queue):
try:
with pipe:
for line in iter(pipe.readline, b''):
queue.put((pipe, line))
finally:
queue.put(None)
process = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, **kwargs)
q = Queue()
Thread(target=reader, args=[process.stdout, q]).start()
Thread(target=reader, args=[process.stderr, q]).start()
# for _ in range(2):
for source, line in iter(q.get, None):
if qgis_feedback.isCanceled():
# qgis_feedback.reportError('CANCELED')
proc2kill = psutil.Process(process.pid)
for proc in proc2kill.children(recursive=True):
proc.kill()
proc2kill.kill()
raise KeyboardInterrupt
linestr = line.decode('latin-1').rstrip()
# print("%s: %s" % (source, linestr))
if source.name == 3:
qgis_feedback.pushInfo(linestr)
if source.name == 4:
qgis_feedback.reportError(linestr)
exitcode = process.poll()
return exitcode
# -*- coding: utf-8 -*-
# enpt_enmapboxapp, A QGIS EnMAPBox plugin providing a GUI for the EnMAP processing tools (EnPT)
#
# Copyright (C) 2019 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
#
# This software was developed within the context of the EnMAP project supported
# by the DLR Space Administration with funds of the German Federal Ministry of
# Economic Affairs and Energy (on the basis of a decision by the German Bundestag:
# 50 EE 1529) and contributions from DLR, GFZ and OHB System AG.
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""This module provides the EnPTAlgorithm which is used in case EnPT is installed into QGIS Python environment."""
import os
from pkgutil import find_loader
from glob import glob
from qgis.core import \
(QgsProcessingContext,
QgsProcessingFeedback,
NULL
)
from ._enpt_alg_base import _EnPTBaseAlgorithm
class EnPTAlgorithm(_EnPTBaseAlgorithm):
@staticmethod
def _prepare_enpt_environment() -> dict:
os.environ['PYTHONUNBUFFERED'] = '1'
enpt_env = os.environ.copy()
enpt_env["PATH"] = ';'.join([i for i in enpt_env["PATH"].split(';') if 'OSGEO' not in i]) # actually not needed
if "PYTHONHOME" in enpt_env.keys():
del enpt_env["PYTHONHOME"]
if "PYTHONPATH" in enpt_env.keys():
del enpt_env["PYTHONPATH"]
# FIXME is this needed?
enpt_env['IPYTHONENABLE'] = 'True'
enpt_env['PROMPT'] = '$P$G'
enpt_env['PYTHONDONTWRITEBYTECODE'] = '1'
enpt_env['PYTHONIOENCODING'] = 'UTF-8'
enpt_env['TEAMCITY_VERSION'] = 'LOCAL'
enpt_env['O4W_QT_DOC'] = 'C:/OSGEO4~3/apps/Qt5/doc'
if 'SESSIONNAME' in enpt_env.keys():
del enpt_env['SESSIONNAME']
# import pprint
# s = pprint.pformat(enpt_env)
# with open('D:\\env.json', 'w') as fp:
# fp.write(s)
return enpt_env
def processAlgorithm(self, parameters, context, feedback):
assert isinstance(parameters, dict)
assert isinstance(context, QgsProcessingContext)
assert isinstance(feedback, QgsProcessingFeedback)
if not find_loader('enpt'):
raise ImportError("enpt", "EnPT must be installed into the QGIS Python environment "
"when calling 'EnPTAlgorithm'.")
feedback.pushInfo("The log messages of the EnMAP processing tool are written to the *.log file "
"in the specified output folder.")
# remove all parameters not to be forwarded to the EnPT CLI
parameters = {k: v for k, v in parameters.items()
if k not in ['anaconda_root']
and v not in [None, NULL, 'NULL', '']}
# print parameters and console call to log
# for key in sorted(parameters):
# feedback.pushInfo('{} = {}'.format(key, repr(parameters[key])))
keyval_str = ' '.join(['--{} {}'.format(key, parameters[key])
for key in sorted(parameters)
if parameters[key] not in [None, NULL, 'NULL', '']])
print(parameters)
print(keyval_str + '\n\n')
feedback.pushInfo("\nCalling EnPT with the following command:\n"
"enpt %s\n\n" % keyval_str)
# prepare environment for subprocess
enpt_env = self._prepare_enpt_environment()
# path_enpt_runscript = self._locate_enpt_run_script()
# run EnPT in subprocess that activates the EnPT Anaconda environment
# feedback.pushDebugInfo('Using %s to start EnPT.' % path_enpt_runscript)
feedback.pushInfo("The log messages of the EnMAP processing tool are written to the *.log file "
"in the specified output folder.")
self._run_cmd(f"enpt {keyval_str}",
qgis_feedback=feedback,
env=enpt_env
)
# list output dir
outdir = parameters['output_dir']