Commit d019eaed authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'bugfix/fix_docker_ci' into 'master'

Bugfix/fix docker ci

Closes #13

See merge request !11
parents 057cf178 602abaae
Pipeline #7866 failed with stages
in 8 minutes and 37 seconds
......@@ -12,8 +12,32 @@ test_enpt_enmapboxapp:
script:
- source /root/miniconda3/bin/activate ci_env
# install enpt_enmapboxapp, otherwise the executable scripts are not in place
- pip install -e .
# update EnPT # TODO revise as soon as EnPT is in PyPI
# - source /root/miniconda3/bin/activate enpt
# - rm -rf context/enpt
# - git clone git@gitext.gfz-potsdam.de:EnMAP/GFZ_Tools_EnMAP_BOX/EnPT.git ./context/enpt
# - cd ./context/enpt
# - pip install -e .
# - cd ../../
# - source /root/miniconda3/bin/activate ci_env
# simulate DISPLAY for Qt
- Xvfb :1 -screen 0 1024x768x16 &> xvfb.log & DISPLAY=:1.0
- export DISPLAY=:1.0
# - QT_QPA_PLATFORM=offscreen
# - export QT_QPA_PLATFORM
# set environment variables
- export IS_CI_ENV=1 # to notify tests that they are running within a docker CI system
- export ANACONDA_ROOT=/root/miniconda3/
- export IS_ENPT_GUI_TEST=1
# run nosetests
# - make nosetests # test are called here
- make nosetests # test are called here
# create the docs
- pip install sphinx_rtd_theme # Read-the-docs theme for SPHINX documentation
......@@ -22,8 +46,8 @@ test_enpt_enmapboxapp:
paths:
# - htmlcov/
- docs/_build/html/
# - nosetests.html
# - nosetests.xml
- nosetests.html
- nosetests.xml
- tests/linting
when: always
......@@ -94,23 +118,23 @@ pages:
- rm -rf public
- mkdir public
- mkdir -p public/doc
# - mkdir -p public/coverage
# - mkdir -p public/nosetests_reports
- mkdir -p public/coverage
- mkdir -p public/nosetests_reports
# Copy over the docs
- cp -r docs/_build/html/* public/doc/
# Copy over the coverage reports
# - cp -r htmlcov/* public/coverage/
- cp -r htmlcov/* public/coverage/
# Copy over the nosetests reports
# - cp nosetests.* public/nosetests_reports/
- cp nosetests.* public/nosetests_reports/
# Check if everything is working great
- ls -al public
- ls -al public/doc
# - ls -al public/coverage
# - ls -al public/nosetests_reports
- ls -al public/coverage
- ls -al public/nosetests_reports
artifacts:
paths:
- public
......
......@@ -2,12 +2,26 @@
enpt_enmapboxapp
================
A QGIS EnMAPBox plugin providing a GUI for the EnMAP processing tools (EnPT).
* Free software: GNU General Public License v3
* Documentation: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/
.. image:: ./docs/images/screenshot_enpt_enmapboxapp_874x1267.png
:width: 874 px
:height: 1267 px
:scale: 100 %
Status
------
.. image:: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/badges/master/pipeline.svg
:target: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/commits/master
.. image:: https://img.shields.io/pypi/v/enpt_enmapboxapp.svg
:target: https://pypi.python.org/pypi/enpt_enmapboxapp
.. .. image:: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/badges/master/coverage.svg
.. image:: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/badges/master/coverage.svg
:target: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/coverage/
.. image:: https://img.shields.io/static/v1?label=Documentation&message=GitLab%20Pages&color=orange
:target: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/
......@@ -29,18 +43,7 @@ enpt_enmapboxapp
:target: https://pyup.io/repos/github/danschef/enpt_enmapboxapp/
:alt: Updates
A QGIS EnMAPBox plugin providing a GUI for the EnMAP processing tools (EnPT).
* Free software: GNU General Public License v3
* Documentation: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/
.. image:: ./docs/images/screenshot_enpt_enmapboxapp_874x1267.png
:width: 874 px
:height: 1267 px
:scale: 100 %
See also the latest coverage_ report and the nosetests_ HTML report.
Installation
......@@ -65,3 +68,5 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack
.. _Cookiecutter: https://github.com/audreyr/cookiecutter
.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
.. _coverage: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/coverage/
.. _nosetests: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/nosetests_reports/nosetests.html
......@@ -191,7 +191,9 @@ class EnPTAlgorithm(QgsProcessingAlgorithm):
@staticmethod
def _get_default_anaconda_root():
if os.name == 'nt':
if os.getenv('ANACONDA_ROOT') and os.path.exists(os.getenv('ANACONDA_ROOT')):
return os.getenv('ANACONDA_ROOT')
elif os.name == 'nt':
return 'C:\\ProgramData\\Anaconda3'
else:
return '' # FIXME is there a default location in Linux/OSX?
......@@ -409,7 +411,7 @@ class EnPTAlgorithm(QgsProcessingAlgorithm):
finally:
queue.put(None)
process = Popen(cmd, stdout=PIPE, stderr=PIPE, bufsize=1, shell=True, **kwargs)
process = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, **kwargs)
q = Queue()
Thread(target=reader, args=[process.stdout, q]).start()
Thread(target=reader, args=[process.stderr, q]).start()
......@@ -444,8 +446,8 @@ class EnPTAlgorithm(QgsProcessingAlgorithm):
if user_root and os.path.exists(user_root):
anaconda_rootdir = user_root
elif 'ANACONDA_ROOT' in os.environ and os.path.exists(os.environ['ANACONDA_ROOT']):
anaconda_rootdir = os.environ['ANACONDA_ROOT']
elif os.getenv('ANACONDA_ROOT') and os.path.exists(os.getenv('ANACONDA_ROOT')):
anaconda_rootdir = os.getenv('ANACONDA_ROOT')
else:
possPaths = \
......@@ -458,15 +460,24 @@ class EnPTAlgorithm(QgsProcessingAlgorithm):
if os.path.exists(rootDir):
anaconda_rootdir = rootDir
if not anaconda_rootdir and user_root or 'ANACONDA_ROOT' in os.environ:
if not anaconda_rootdir:
raise NotADirectoryError("No valid Anaconda root directory given - "
"neither via the GUI, nor via the 'ANACONDA_ROOT' environment variable.")
# set ENPT_PYENV_ACTIVATION environment variable
os.environ['ENPT_PYENV_ACTIVATION'] = os.path.join(anaconda_rootdir, 'Scripts', 'activate.bat')
os.environ['ENPT_PYENV_ACTIVATION'] = \
os.path.join(anaconda_rootdir, 'Scripts', 'activate.bat') if os.name == 'nt' else \
os.path.join(anaconda_rootdir, 'bin', 'activate')
if not os.path.exists(os.getenv('ENPT_PYENV_ACTIVATION')):
raise FileNotFoundError(os.getenv('ENPT_PYENV_ACTIVATION'))
return anaconda_rootdir
@staticmethod
def _is_enpt_environment_present(anaconda_rootdir):
return os.path.exists(os.path.join(anaconda_rootdir, 'envs', 'enpt'))
@staticmethod
def _locate_enpt_run_script():
try:
......@@ -519,6 +530,21 @@ class EnPTAlgorithm(QgsProcessingAlgorithm):
anaconda_root = self._locate_EnPT_Anaconda_environment(parameters[self.P_anaconda_root])
feedback.pushInfo('Found Anaconda installation at %s.' % anaconda_root)
if self._is_enpt_environment_present(anaconda_root):
feedback.pushInfo("The Anaconda installation contains the 'enpt' environment as expected.")
else:
feedback.reportError("The Anaconda installation has no environment called 'enpt'. Please follow the EnPT "
"installation instructions to install the EnMAP processing tool backend code "
"(see http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/doc/"
"installation.html). This is needed to run EnPT from this GUI.")
return {
'success': False,
self.P_OUTPUT_RASTER: '',
# self.P_OUTPUT_VECTOR: parameters[self.P_OUTPUT_RASTER],
# self.P_OUTPUT_FILE: parameters[self.P_OUTPUT_RASTER],
self.P_OUTPUT_FOLDER: ''
}
# remove all parameters not to be forwarded to the EnPT CLI
parameters = {k: v for k, v in parameters.items() if k not in ['anaconda_root']}
......@@ -561,6 +587,7 @@ class EnPTAlgorithm(QgsProcessingAlgorithm):
# return outputs
return {
'success': True,
self.P_OUTPUT_RASTER: outraster,
# self.P_OUTPUT_VECTOR: parameters[self.P_OUTPUT_RASTER],
# self.P_OUTPUT_FILE: parameters[self.P_OUTPUT_RASTER],
......
......@@ -2,7 +2,7 @@
context_dir="./context"
dockerfile="enpt_enmapboxapp_ci.docker"
tag="enpt_enmapboxapp_ci:0.3.2"
tag="enpt_enmapboxapp_ci:0.4.0"
gitlab_runner="enpt_enmapboxapp_gitlab_CI_runner"
# get enmapbox project
......@@ -10,6 +10,14 @@ rm -rf context/enmapbox
git clone https://bitbucket.org/hu-geomatics/enmap-box.git ./context/enmapbox
# git clone https://bitbucket.org/hu-geomatics/enmap-box.git --branch develop --single-branch ./context/enmapbox
# get EnPT project
rm -rf context/enpt
git clone git@gitext.gfz-potsdam.de:EnMAP/GFZ_Tools_EnMAP_BOX/EnPT.git ./context/enpt
# get SICOR project
rm -rf context/sicor
git clone git@gitext.gfz-potsdam.de:EnMAP/sicor.git ./context/sicor
echo "#### Build runner docker image"
sudo docker rmi ${tag}
sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir}
......
FROM gms_base_centos:0.3
# install Xvfb to be able to assign a DISPLAY for Qt tests
RUN yum install -y xorg-x11-server-Xvfb
# copy some needed stuff to /root
COPY *.yml /root/
......@@ -19,14 +22,26 @@ RUN /bin/bash -i -c "\
source activate ci_env; \
conda env update -n ci_env -f /root/environment_enpt_enmapboxapp.yml"
# copy enmapbox code to /tmp
# copy enmapbox, enpt and sicor code to /tmp
COPY enmapbox /tmp/enmapbox
COPY enpt /tmp/enpt
COPY sicor /tmp/sicor
# install enmapbox (in pip development mode)
RUN bash -i -c "\
source /root/miniconda3/bin/activate ; \
source activate ci_env; \
source /root/miniconda3/bin/activate ci_env ; \
cd /tmp/enmapbox/ ; \
pip install -r https://bitbucket.org/hu-geomatics/enmap-box/raw/develop/requirements.txt ; \
pip install -e . ; \
conda list"
# install enpt (into separate conda environment; in pip development mode)
RUN bash -i -c "\
source /root/miniconda3/bin/activate ; \
conda env update -f /tmp/enpt/tests/gitlab_CI_docker/context/environment_enpt.yml ; \
source activate enpt ; \
cd /tmp/sicor/ ; \
pip install -e . ; \
cd /tmp/enpt/ ; \
pip install -e . ; \
conda list"
......@@ -25,9 +25,11 @@
"""Tests for `enpt_enmapboxapp` package."""
# import os
import os
from tempfile import TemporaryDirectory
import pickle
from unittest import TestCase
from unittest import TestCase, skipIf
from qgis.core import QgsProcessingAlgorithm, QgsProcessingContext, QgsProcessingFeedback, QgsProcessingProvider, NULL
from enmapbox.testing import initQgisApplication
......@@ -90,6 +92,7 @@ class TestExampleEnMAPBoxApp(TestCase):
# self.assertEqual(kwds['foo'], 'bar')
def test_processingAlgorithms(self):
os.environ['IS_ENPT_GUI_TEST'] = '1'
alg = EnPTAlgorithm()
self.assertIsInstance(alg, QgsProcessingAlgorithm)
......@@ -97,11 +100,28 @@ class TestExampleEnMAPBoxApp(TestCase):
alg2 = alg.createInstance()
self.assertIsInstance(alg2, QgsProcessingAlgorithm)
outputs = alg.processAlgorithm(enpt_test_parameters,
QgsProcessingContext(),
QgsProcessingFeedback())
self.assertIsInstance(outputs, dict)
self.assertTrue(outputs['args'] == (enpt_test_parameters,))
with TemporaryDirectory() as td:
params = enpt_test_parameters.copy()
params['output_dir'] = td
params['path_l1b_enmap_image'] = os.path.join('dummy', 'path', 'to', 'EnMAP_file.zip')
params['path_dem'] = os.path.join('dummy', 'path', 'to', 'DEM.bsq')
outputs = alg.processAlgorithm(params,
QgsProcessingContext(),
QgsProcessingFeedback())
self.assertIsInstance(outputs, dict)
self.assertTrue(outputs['success'] is True,
'EnPT could not be called or did not output the expected results.')
with open(os.path.join(td, 'received_args_kwargs.pkl'), 'rb') as inF:
content = pickle.load(inF)
none_params = [k for k, v in params.items() if params[k] in [None, NULL]]
for k, v in params.items():
if k not in ['anaconda_root', 'json_config'] + none_params:
self.assertTrue(k in content['kwargs'], "Missing key '%s' in received parameters." % k)
self.assertEqual(v, content['kwargs'][k])
# def test_dialog(self):
# """
......@@ -120,6 +140,7 @@ class TestExampleEnMAPBoxApp(TestCase):
# if SHOW_GUI:
# APP.exec_()
@skipIf(os.getenv('IS_CI_ENV'), reason='to be tested manually')
def test_with_EnMAPBox(self):
"""
Finally, test if your application can be added into the EnMAP-Box
......@@ -145,4 +166,11 @@ class TestExampleEnMAPBoxApp(TestCase):
if __name__ == "__main__":
import unittest
SHOW_GUI = False
# quiet matplotlib
import logging
mpl_logger = logging.getLogger('matplotlib')
mpl_logger.setLevel(logging.WARNING)
# run tests
unittest.main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment