Commit a6e41afd authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge remote-tracking branch 'origin/master' into master

# Conflicts:
#	tests/gitlab_CI_docker/build_enpt_enmapboxapp_testsuite_image.sh
#	tests/gitlab_CI_docker/context/enpt_enmapboxapp_ci.docker
#	tests/gitlab_CI_docker/context/environment_enpt_enmapboxapp.yml
parents c44f8302 3c1033f3
Pipeline #12267 failed with stages
in 7 minutes and 59 seconds
......@@ -12,15 +12,42 @@ test_enpt_enmapboxapp:
script:
- source /root/miniconda3/bin/activate ci_env
# install enpt_enmapboxapp, otherwise the executable scripts are not in place
- pip install -e .
# update EnPT # TODO revise as soon as EnPT is in PyPI
# - source /root/miniconda3/bin/activate enpt
# - rm -rf context/enpt
# - git clone git@gitext.gfz-potsdam.de:EnMAP/GFZ_Tools_EnMAP_BOX/EnPT.git ./context/enpt
# - cd ./context/enpt
# - pip install -e .
# - cd ../../
# - source /root/miniconda3/bin/activate ci_env
# simulate DISPLAY for Qt
- Xvfb :1 -screen 0 1024x768x16 &> xvfb.log & DISPLAY=:1.0
- export DISPLAY=:1.0
# - QT_QPA_PLATFORM=offscreen
# - export QT_QPA_PLATFORM
# set environment variables
- export IS_CI_ENV=1 # to notify tests that they are running within a docker CI system
- export ANACONDA_ROOT=/root/miniconda3/
- export IS_ENPT_GUI_TEST=1
# run nosetests
- make nosetests # test are called here
# create the docs
- pip install sphinx_rtd_theme # Read-the-docs theme for SPHINX documentation
- make docs
artifacts:
paths:
# - htmlcov/
- htmlcov/
- docs/_build/html/
# - nosetests.html
# - nosetests.xml
- nosetests.html
- nosetests.xml
- tests/linting
when: always
......@@ -51,14 +78,31 @@ test_enpt_enmapboxapp_install:
- conda create -y -q --name enpt_enmapboxapp_test python=3
- source activate enpt_enmapboxapp_test
# install enpt
- pip install -e .
# install qgis
- conda install -c conda-forge qgis
# install enmapbox
- cd ..
- rm -rf enmap-box
- git clone https://bitbucket.org/hu-geomatics/enmap-box.git
- pip install -r https://bitbucket.org/hu-geomatics/enmap-box/raw/develop/requirements.txt ; \
- cd enmap-box
- pip install .
- cd ../enpt_enmapboxapp
# install enpt_enmapboxapp
- pip install -e .
- pwd
- ls
# importability cannot be tested here because environment does not contain QGIS and enmapbox
# - python -c "import enpt_enmapboxapp; print(enpt_enmapboxapp)"
# echo installed packages
- conda list qgis
- conda list enmapbox
# test importability
- cd ../enmap-box # we need to jump into that folder, otherwise enmapbox is not importable (due to an issue with its setup.py)
- python -c "import enpt_enmapboxapp; print(enpt_enmapboxapp)"
- python -c "from enpt_enmapboxapp import EnPTAlgorithm, EnMAPBoxApplication"
only:
- master
- dev
......@@ -73,27 +117,41 @@ pages:
- rm -rf public
- mkdir public
- mkdir -p public/doc
# - mkdir -p public/coverage
# - mkdir -p public/nosetests_reports
- mkdir -p public/coverage
- mkdir -p public/nosetests_reports
# Copy over the docs
- cp -r docs/_build/html/* public/doc/
# Copy over the coverage reports
# - cp -r htmlcov/* public/coverage/
- cp -r htmlcov/* public/coverage/
# Copy over the nosetests reports
# - cp nosetests.* public/nosetests_reports/
- cp nosetests.* public/nosetests_reports/
# Check if everything is working great
- ls -al public
- ls -al public/doc
# - ls -al public/coverage
# - ls -al public/nosetests_reports
- ls -al public/coverage
- ls -al public/nosetests_reports
artifacts:
paths:
- public
expire_in: 30 days
only:
- master
- feature/add_docs
deploy_pypi:
stage: deploy
dependencies:
- test_enpt_enmapboxapp
script:
- source /root/miniconda3/bin/activate ci_env
- pip install -U twine
- python setup.py sdist
- twine upload dist/* # requires creds as environment variables
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- dev
......@@ -2,6 +2,45 @@
History
=======
0.4.3 (2020-03-26)
------------------
* Fixed broken 'pip install enpt_enmapboxapp' on Windows (fixes issue #17).
0.4.2 (2020-03-26)
------------------
* added parameter 'vswir_overlap_algorithm'
0.4.1 (2020-03-26)
------------------
* nosetests are now properly working:
EnPT is called with the given GUI parameters and sends back a file containing all received parameters
-> fixes issue #13 (closed)
* fixed Linux implementation
* improved error messages in case not all software components are properly installed
0.4.0 (2020-03-25)
------------------
* EnPT can now be interrupted by pressing the cancel button.
* Replaced placeholder app with a link to start the GUI.
* Added an About-Dialog.
* The package is now publicly available.
* Added PyPI upload.
0.3.0 (2020-01-28)
------------------
* The EnPT output is now properly displayed in the log window during EnPT runtime
* Code improvements
* Some minor documentation improvements
0.2.0 (2020-01-17)
------------------
......
......@@ -4,6 +4,7 @@ include HISTORY.rst
include LICENSE
include README.rst
recursive-include bin *
recursive-include tests *
recursive-exclude * __pycache__
recursive-exclude * *.py[co]
......
......@@ -62,10 +62,19 @@ test-all: ## run tests on every Python version with tox
tox
coverage: ## check code coverage quickly with the default Python
coverage run --source enpt_enmapboxapp setup.py test
coverage erase
coverage run --source enpt_enmapboxapp --source bin setup.py test
coverage combine # must be called in order to make coverage work in multiprocessing
coverage report -m
coverage html
$(BROWSER) htmlcov/index.html
# $(BROWSER) htmlcov/index.html
nosetests: clean-test ## Runs nosetests with coverage, xUnit and nose-html-output
## - puts the coverage results in the folder 'htmlcov'
## - generates 'nosetests.html' (--with-html)
## - generates 'nosetests.xml' (--with-xunit) which is currently not visualizable by GitLab
nosetests -vv --with-coverage --cover-package=enpt_enmapboxapp --cover-erase --cover-html --cover-html-dir=htmlcov \
--with-html --with-xunit --rednose --force-color
docs: ## generate Sphinx HTML documentation, including API docs
rm -f docs/enpt_enmapboxapp.rst
......
================
enpt_enmapboxapp
================
.. image:: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/badges/master/pipeline.svg
:target: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/commits/master
.. image:: https://img.shields.io/pypi/v/enpt_enmapboxapp.svg
:target: https://pypi.python.org/pypi/enpt_enmapboxapp
.. image:: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/badges/master/coverage.svg
:target: coverage_
.. image:: https://img.shields.io/static/v1?label=Documentation&message=GitLab%20Pages&color=orange
:target: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/
.. image:: https://img.shields.io/pypi/l/enpt_enmapboxapp.svg
:target: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/blob/master/LICENSE
.. image:: https://img.shields.io/pypi/pyversions/enpt_enmapboxapp.svg
:target: https://img.shields.io/pypi/pyversions/enpt_enmapboxapp.svg
.. image:: https://img.shields.io/pypi/dm/enpt_enmapboxapp.svg
:target: https://pypi.python.org/pypi/enpt_enmapboxapp
.. .. image:: https://img.shields.io/travis/danschef/enpt_enmapboxapp.svg
:target: https://travis-ci.org/danschef/enpt_enmapboxapp
......@@ -16,37 +20,32 @@ enpt_enmapboxapp
:target: https://enpt-enmapboxapp.readthedocs.io/en/latest/?badge=latest
:alt: Documentation Status
.. .. image:: https://pyup.io/repos/github/danschef/enpt_enmapboxapp/shield.svg
:target: https://pyup.io/repos/github/danschef/enpt_enmapboxapp/
:alt: Updates
================
enpt_enmapboxapp
================
A QGIS EnMAPBox plugin providing a GUI for the EnMAP processing tools (EnPT).
* Free software: GNU General Public License v3
* Documentation: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/
See also the latest coverage_ report and the nosetests_ HTML report.
How the GUI looks like
----------------------
.. image:: ./docs/images/screenshot_enpt_enmapboxapp_874x1267.png
:width: 874 px
:height: 1267 px
:scale: 100 %
Installation
------------
Clone the repository and install enpt_enmapboxapp to your local QGIS python:
.. code-block:: console
git clone https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp.git
cd enpt_enmapboxapp
pip install .
Credits
-------
This software was developed within the context of the EnMAP project supported by the DLR Space Administration with
......@@ -57,3 +56,5 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack
.. _Cookiecutter: https://github.com/audreyr/cookiecutter
.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
.. _coverage: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/coverage/
.. _nosetests: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/nosetests_reports/nosetests.html
......@@ -20,6 +20,7 @@
:: You should have received a copy of the GNU Lesser General Public License along
:: with this program. If not, see <http://www.gnu.org/licenses/>.
@echo off
:: activate the EnPT Anaconda environment located at %ENPT_PYENV_ACTIVATION%
:: echo %PATH%
......@@ -37,4 +38,10 @@ call %ENPT_PYENV_ACTIVATION% enpt
:: run enpt_cli.py with the provided arguments
FOR /F %%i IN ('where enpt_cli.py') do (SET PATH_ENPT_CLI=%%i)
echo.
echo _______________________________________
python %PATH_ENPT_CLI% %*
echo _______________________________________
echo.
......@@ -9,13 +9,11 @@ QGIS_ including the EnMAP-Box plugin. If QGIS_ or the EnMAP-Box_ is not yet inst
follow the installation instructions
`here <https://enmap-box.readthedocs.io/en/latest/usr_section/usr_installation.html>`__.
The enpt_enmapboxapp package is then installed into the QGIS_ Python environment.
The enpt_enmapboxapp package is then installed into the QGIS_ Python environment:
.. code-block:: console
git clone https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp.git
cd enpt_enmapboxapp
pip install .
pip install enpt_enmapboxapp
.. note::
......
......@@ -3,13 +3,15 @@ Usage
=====
The enpt_enmapboxapp GUI can be found in
:menuselection:`QGIS 3.xx --> EnMAP-Box --> Processing Toolbox --> Pre-Processing --> EnMAP processing tools algorithm`
:menuselection:`QGIS 3.xx --> EnMAP-Box --> Processing Toolbox --> Pre-Processing --> EnMAP processing tool algorithm`
.. image:: ./images/sceenshot_how_to_start_1342x1029.png
:scale: 75 %
Use the GUI to parameterize EnPT_ according to your input dataset and the desired processing settings.
The input parameters are explained
`here <http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/doc/usage.html#enpt-cli-py>`__.
.. hint::
......
......@@ -25,11 +25,14 @@
"""Top-level package for enpt_enmapboxapp."""
from .version import __version__, __versionalias__ # noqa (E402 + F401)
from .enpt_enmapboxapp import EnMAPBoxApplication, EnPTAlgorithm
__author__ = """Daniel Scheffler"""
__email__ = 'danschef@gfz-potsdam.de'
__all__ = ['__version__',
'__versionalias__',
'__author__',
'__email__'
'__email__',
'EnMAPBoxApplication',
'EnPTAlgorithm'
]
......@@ -23,19 +23,23 @@
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""This module provides a QGIS EnMAPBox GUI for the EnMAP processing tools (EnPT)."""
import os
import traceback
import psutil
from os.path import expanduser
from datetime import date
from subprocess import Popen, PIPE, check_output
from subprocess import Popen, PIPE, check_output, CalledProcessError
from threading import Thread
from queue import Queue
from multiprocessing import cpu_count
from glob import glob
from qgis.PyQt.QtGui import QIcon
from qgis.PyQt.QtWidgets import QMenu, QAction, QDialog, QHBoxLayout, QLabel, QPushButton
from qgis.PyQt.QtWidgets import QMenu, QAction, QMessageBox
from enmapbox.gui.applications import EnMAPBoxApplication
from qgis.core import \
(QgsProcessingAlgorithm,
QgsApplication,
Qgis,
QgsProcessingParameterFile,
QgsProcessingParameterNumber,
QgsProcessingParameterFolderDestination,
......@@ -47,11 +51,9 @@ from qgis.core import \
)
from .version import __version__
VERSION = __version__
LICENSE = 'GNU GPL-3'
APP_DIR = os.path.dirname(__file__)
APP_NAME = 'EnPT EnMAPBox App'
......@@ -89,11 +91,13 @@ class EnPTEnMAPBoxApp(EnMAPBoxApplication):
# this way you can add your QMenu/QAction to an other menu entry, e.g. 'Tools'
# appMenu = self.enmapbox.menu('Tools')
menu = appMenu.addMenu('EnPT (EnMAP Processing Tools)')
menu = appMenu.addMenu('EnPT (EnMAP Processing Tool)')
menu.setIcon(self.icon())
# add a QAction that starts a process of your application.
# In this case it will open your GUI.
a = menu.addAction('About EnPT')
a.triggered.connect(self.showAboutDialog)
a = menu.addAction('Start EnPT GUI')
assert isinstance(a, QAction)
a.triggered.connect(self.startGUI)
......@@ -101,53 +105,34 @@ class EnPTEnMAPBoxApp(EnMAPBoxApplication):
return menu
def showAboutDialog(self):
QMessageBox.information(None, self.name,
'Version {}'.format(self.version))
def processingAlgorithms(self):
"""
This function returns the QGIS Processing Framework GeoAlgorithms specified by your application
:return: [list-of-GeoAlgorithms]
"""
return [EnPTAlgorithm(), ]
return [EnPTAlgorithm()]
def startGUI(self):
"""
Opens a GUI
:param args:
:return:
"""
w = ExampleAppGUI(self.enmapbox.ui)
w.show()
class ExampleAppGUI(QDialog):
"""
A minimal graphical user interface
"""
def __init__(self, parent=None):
super(ExampleAppGUI, self).__init__(parent)
self.setWindowTitle(APP_NAME)
self.setWindowIcon(QIcon(os.path.join(APP_DIR, 'icon.png')))
self.setMinimumWidth(400)
layout = QHBoxLayout()
self.setLayout(layout)
layout.addWidget(QLabel('Hello World'))
self.btn = QPushButton('Click me')
# clicking the button will print "Hello World" to the python CLI
self.btn.clicked.connect(lambda: print('Hello World'))
layout.addWidget(self.btn)
self.setMinimumSize(self.sizeHint())
def printDictionary(parameters):
"""
An algorithm that just prints the provided parameter dictionary
"""
print('Parameters:')
for key, parameter in parameters.items():
print('{} = {}'.format(key, parameter))
try:
from processing.gui.AlgorithmDialog import AlgorithmDialog
alg = QgsApplication.processingRegistry().algorithmById('enmapbox:EnPTAlgorithm')
assert isinstance(alg, EnPTAlgorithm)
dlg = AlgorithmDialog(alg.create(), in_place=False, parent=self.enmapbox.ui)
dlg.show()
return dlg
except Exception as ex:
msg = str(ex)
msg += '\n' + str(traceback.format_exc())
self.enmapbox.messageBar().pushMessage(APP_NAME, 'Error', msg, level=Qgis.Critical, duration=10)
return None
class EnPTAlgorithm(QgsProcessingAlgorithm):
......@@ -182,6 +167,7 @@ class EnPTAlgorithm(QgsProcessingAlgorithm):
P_deadpix_P_interp_spectral = 'deadpix_P_interp_spectral'
P_deadpix_P_interp_spatial = 'deadpix_P_interp_spatial'
P_ortho_resampAlg = 'ortho_resampAlg'
P_vswir_overlap_algorithm = 'vswir_overlap_algorithm'
# # Output parameters
P_OUTPUT_RASTER = 'outraster'
......@@ -199,14 +185,16 @@ class EnPTAlgorithm(QgsProcessingAlgorithm):
return 'EnPTAlgorithm'
def displayName(self):
return 'EnMAP processing tools algorithm'
return 'EnMAP processing tool algorithm (v%s)' % __version__
def createInstance(self, *args, **kwargs):
return type(self)()
@staticmethod
def _get_default_anaconda_root():
if os.name == 'nt':
if os.getenv('ANACONDA_ROOT') and os.path.exists(os.getenv('ANACONDA_ROOT')):
return os.getenv('ANACONDA_ROOT')
elif os.name == 'nt':
return 'C:\\ProgramData\\Anaconda3'
else:
return '' # FIXME is there a default location in Linux/OSX?
......@@ -295,7 +283,7 @@ class EnPTAlgorithm(QgsProcessingAlgorithm):
self.addParameter(QgsProcessingParameterBoolean(
name=self.P_disable_progress_bars,
description='Disable all progress bars during processing',
defaultValue=False,
defaultValue=True,
optional=True))
self.addParameter(QgsProcessingParameterFile(
......@@ -408,33 +396,72 @@ class EnPTAlgorithm(QgsProcessingAlgorithm):
multiLine=False,
optional=True))
self.addParameter(QgsProcessingParameterString(
name=self.P_vswir_overlap_algorithm,
description="Algorithm specifying how to deal with the spectral bands in the VNIR/SWIR spectral overlap "
"region ('order_by_wvl', 'average', 'vnir_only', 'swir_only')",
defaultValue='swir_only',
multiLine=False,
optional=True))
@staticmethod
def _run_cmd(cmd, no_stdout=False, no_stderr=False, **kwargs):
def _run_cmd(cmd, qgis_feedback=None, **kwargs):
"""Execute external command and get its stdout, exitcode and stderr.
Code based on: https://stackoverflow.com/a/31867499
:param cmd: a normal shell command including parameters
"""
proc = Popen(cmd,
stdout=None if no_stdout else PIPE,
stderr=None if no_stderr else PIPE,
shell=True,
**kwargs)
out, err = proc.communicate()
exitcode = proc.returncode
def reader(pipe, queue):
try:
with pipe:
for line in iter(pipe.readline, b''):
queue.put((pipe, line))
finally:
queue.put(None)
process = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, **kwargs)
q = Queue()
Thread(target=reader, args=[process.stdout, q]).start()
Thread(target=reader, args=[process.stderr, q]).start()
# for _ in range(2):
for source, line in iter(q.get, None):
if qgis_feedback.isCanceled():
# qgis_feedback.reportError('CANCELED')
proc2kill = psutil.Process(process.pid)
for proc in proc2kill.children(recursive=True):
proc.kill()
proc2kill.kill()
raise KeyboardInterrupt
linestr = line.decode('latin-1').rstrip()
# print("%s: %s" % (source, linestr))
if source.name == 3:
qgis_feedback.pushInfo(linestr)
if source.name == 4:
qgis_feedback.reportError(linestr)
exitcode = process.poll()
return exitcode
return out, exitcode, err
def locate_EnPT_Anaconda_environment(self):
@staticmethod
def _locate_EnPT_Anaconda_environment(user_root):
anaconda_rootdir = None
if self.P_anaconda_root and os.path.exists(self.P_anaconda_root):
anaconda_rootdir = self.P_anaconda_root
if user_root and os.path.exists(user_root):
anaconda_rootdir = user_root
elif 'ANACONDA_ROOT' in os.environ and os.path.exists(os.environ['ANACONDA_ROOT']):
anaconda_rootdir = os.environ['ANACONDA_ROOT']
elif os.getenv('ANACONDA_ROOT') and os.path.exists(os.getenv('ANACONDA_ROOT')):
anaconda_rootdir = os.getenv('ANACONDA_ROOT')
else:
possPaths = \
['C:\\ProgramData\\Anaconda3',
'C:\\Users\\%s\\Anaconda3' % os.getenv('username')
] if os.name == 'nt' else \
[]
......@@ -442,23 +469,91 @@ class EnPTAlgorithm(QgsProcessingAlgorithm):
if os.path.exists(rootDir):
anaconda_rootdir = rootDir
if not anaconda_rootdir and self.P_anaconda_root or 'ANACONDA_ROOT' in os.environ:
if not anaconda_rootdir:
raise NotADirectoryError("No valid Anaconda root directory given - "
"neither via the GUI, nor via the 'ANACONDA_ROOT' environment variable.")
# set ENPT_PYENV_ACTIVATION environment variable
os.environ['ENPT_PYENV_ACTIVATION'] = os.path.join(anaconda_rootdir, 'Scripts', 'activate.bat')
os.environ['ENPT_PYENV_ACTIVATION'] = \
os.path.join(anaconda_rootdir, 'Scripts', 'activate.bat') if os.name == 'nt' else \
os.path.join(anaconda_rootdir, 'bin', 'activate')
if not os.path.exists(os.getenv('ENPT_PYENV_ACTIVATION')):
raise FileNotFoundError(os.getenv('ENPT_PYENV_ACTIVATION'))
return anaconda_rootdir
@staticmethod
def _is_enpt_environment_present(anaconda_rootdir):
return os.path.exists(os.path.join(anaconda_rootdir, 'envs', 'enpt'))
@staticmethod
def _locate_enpt_run_script():
try: