Commit 03b528cb authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'nosetests' into dev

parents e5b43c89 667175cd
Pipeline #551 passed with stages
in 6 minutes and 30 seconds
......@@ -40,6 +40,7 @@ htmlcov/
.coverage
.coverage.*
.cache
nosetests.html
nosetests.xml
coverage.xml
*,cover
......@@ -67,6 +68,8 @@ target/
BAK/
OLD/
/geomultisens/database/cloud_classifier/
/tests/data/output_mgrs_tiles/
/tests/data/output_scenes/
/tests/data/sampledata/
/geomultisens/database/metadata/
/geomultisens/database/tiling_system/
......
......@@ -6,12 +6,14 @@ test_geomultisens:
- source /root/anaconda3/bin/activate
- export GDAL_DATA=/root/anaconda3/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- make coverage
- make nosetests
- make docs
artifacts:
paths:
- htmlcov/
- docs/_build/html/
- nosetests.html
- nosetests.xml
pages:
stage: deploy
......@@ -20,6 +22,8 @@ pages:
script:
- mkdir -p public/coverage
- cp -r htmlcov/* public/coverage/
- mkdir -p public/nosetests_reports
- cp nosetests.* public/nosetests_reports/
- mkdir -p public/doc
- cp -r docs/_build/html/* public/doc/
......
......@@ -6,6 +6,7 @@ Development Lead
----------------
* Daniel Scheffler <daniel.scheffler@gfz-potsdam.de>
** <http://www.gfz-potsdam.de/en/section/remote-sensing/staff/profil/daniel-scheffler/>
Contributors
......
.PHONY: clean clean-test clean-pyc clean-build docs help
.PHONY: clean clean-test clean-pyc clean-build docs help nosetests
.DEFAULT_GOAL := help
define BROWSER_PYSCRIPT
import os, webbrowser, sys
......@@ -28,7 +28,6 @@ help:
clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
clean-build: ## remove build artifacts
rm -fr build/
rm -fr dist/
......@@ -43,9 +42,12 @@ clean-pyc: ## remove Python file artifacts
find . -name '__pycache__' -exec rm -fr {} +
clean-test: ## remove test and coverage artifacts
coverage erase
rm -fr .tox/
rm -f .coverage
rm -fr htmlcov/
rm -fr nosetests.html
rm -fr nosetests.xml
lint: ## check style with flake8
flake8 geomultisens tests
......@@ -56,12 +58,18 @@ test: ## run tests quickly with the default Python
test-all: ## run tests on every Python version with tox
tox
coverage: ## check code coverage quickly with the default Python
coverage: clean-test ## check code coverage quickly with the default Python
coverage run --source geomultisens setup.py test
coverage combine # must be called in order to make coverage work in multiprocessing
coverage report -m
coverage html
$(BROWSER) htmlcov/index.html
# $(BROWSER) htmlcov/index.html
nosetests: clean-test ## Runs nosetests with coverage, xUnit and nose-html-output
## - puts the coverage results in the folder 'htmlcov'
## - generates 'nosetests.html' (--with-html)
## - generates 'nosetests.xml' (--with-xunit) which is currently not visualizable by GitLab
nosetests -vv --with-coverage --cover-package=geomultisens --cover-erase --cover-html --cover-html-dir=htmlcov --with-html --with-xunit
docs: ## generate Sphinx HTML documentation, including API docs
rm -f docs/geomultisens.rst
......@@ -69,7 +77,7 @@ docs: ## generate Sphinx HTML documentation, including API docs
sphinx-apidoc -o docs/ geomultisens
$(MAKE) -C docs clean
$(MAKE) -C docs html
$(BROWSER) docs/_build/html/index.html
#$(BROWSER) docs/_build/html/index.html
servedocs: docs ## compile the docs watching for changes
watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
......
......@@ -32,7 +32,7 @@ Status
.. image:: https://gitext.gfz-potsdam.de/danschef/GeoMultiSens/badges/master/coverage.svg
:target: http://danschef.gitext.gfz-potsdam.de/GeoMultiSens/coverage/
See also the latest coverage_ report.
See also the latest coverage_ report and the nosetests_ HTML report.
Features
--------
......@@ -67,3 +67,4 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack
.. _Cookiecutter: https://github.com/audreyr/cookiecutter
.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
.. _coverage: http://danschef.gitext.gfz-potsdam.de/GeoMultiSens/coverage/
.. _nosetests: http://danschef.gitext.gfz-potsdam.de/GeoMultiSens/nosetests_reports/nosetests.html
......@@ -2,7 +2,7 @@
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXOPTS = -T
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = _build
......
......@@ -2,8 +2,8 @@
__author__ = """Daniel Scheffler"""
__email__ = 'daniel.scheffler@gfz-potsdam.de'
__version__ = '0.5.1'
__versionalias__ = '20170718.01'
__version__ = '0.5.4'
__versionalias__ = '20170721.01'
from . import algorithms
......
# -*- coding: utf-8 -*-
__author__='Daniel Scheffler'
import multiprocessing
from multiprocessing import Pool
from itertools import chain
from ..config import GMS_config as CFG
......@@ -23,8 +23,8 @@ def MAP(func, args, CPUs=None, flatten_output=False):
CPUs = CPUs if CPUs is not None else CFG.job.CPUs
CPUs = CPUs if CPUs <= CFG.job.CPUs else CFG.job.CPUs # treat CFG.job.CPUs as maximum number of CPUs
if CPUs is not None and CPUs > 1:
with multiprocessing.Pool(CPUs) as pool:
if CPUs is not None and CPUs > 1 and len(args) > 1:
with Pool(CPUs) as pool:
results = pool.map(func,args) # always returns a list
else:
results = [func(argset) for argset in args] # generator does not always work properly here
......
......@@ -8,4 +8,6 @@ coverage==4.1
Sphinx==1.4.8
cryptography==1.7
PyYAML==3.11
nose
nose2
nose-htmloutput
......@@ -18,11 +18,11 @@ requirements = [
#'sicor', # pip install git+https://gitext.gfz-potsdam.de/hollstei/sicor.git
]
setup_requirements = [] # TODO(danschef): put setup requirements (distutils extensions, etc.) here
test_requirements = requirements+ ['coverage']
test_requirements = requirements+ ['coverage', 'nose', 'nose-htmloutput']
setup(
name='geomultisens',
version='0.5.1',
version='0.5.4',
description="GeoMultiSens - Scalable Multi-Sensor Analysis of Remote Sensing Data",
long_description=readme + '\n\n' + history,
author="Daniel Scheffler",
......
......@@ -5,18 +5,6 @@ dockerfile="geomultisens_ci.docker"
tag="geomultisens_ci:latest"
gitlab_runner="geomultisens_gitlab_CI_runner"
# get py_tools_ds project # TODO move this to setup.py as soon as package is public
rm -rf context/py_tools_ds
git clone https://gitext.gfz-potsdam.de/danschef/py_tools_ds.git ./context/py_tools_ds
# get geoarray project # TODO move this to setup.py as soon as package is public
rm -rf context/geoarray
git clone https://gitext.gfz-potsdam.de/danschef/geoarray.git ./context/geoarray
# get arosics project # TODO move this to setup.py as soon as package is public
rm -rf context/arosics
git clone https://gitext.gfz-potsdam.de/danschef/arosics.git ./context/arosics
# get sicor project # TODO move this to setup.py as soon as package is public
rm -rf context/sicor
git clone https://gitext.gfz-potsdam.de/hollstei/sicor.git ./context/sicor
......
FROM centos:7
RUN yum update -y && \
yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git # TODO
yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git gdb# TODO
ENV anaconda_dl 'Anaconda3-4.3.1-Linux-x86_64.sh'
RUN /bin/bash -i -c "cd /root; wget https://repo.continuum.io/archive/$anaconda_dl ; \
......@@ -24,18 +24,15 @@ RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
conda install --yes -c conda-forge libnetcdf=4.4.1.1 ; \
conda install --yes -c conda-forge olefile=0.44 ; \
conda install --yes xerces-c ; \
conda install --yes -c conda-forge glymur pygrib rasterio pyproj cachetools pyhdf ; \
pip install shapely geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse \
spectral psycopg2 pyorbital pyinstrument geoalchemy2" # must include all the requirements needed to build the docs!
spectral psycopg2 pyorbital pyinstrument geoalchemy2 py_tools_ds geoarray arosics \
nose nose2 nose-htmloutput coverage" # must include all the requirements needed to build the docs!
# copy some needed stuff to /root
#COPY *.pkl /root/ # EXAMPLE
#COPY *.h5 /root/ # EXAMPLE
# copy py_tools_ds, geoarray, arosics and sicor code to /tmp
COPY py_tools_ds /tmp/py_tools_ds
COPY geoarray /tmp/geoarray
COPY arosics /tmp/arosics
# copy sicor code to /tmp
COPY sicor /tmp/sicor
# install git lfs
......@@ -55,8 +52,5 @@ RUN bash -i -c 'source /root/anaconda3/bin/activate; \
sed -i "/backend : qt5agg/c\backend : Agg" $(python -c "import matplotlib; print(matplotlib.matplotlib_fname())")'
RUN bash -i -c "source /root/anaconda3/bin/activate; \
cd /tmp/py_tools_ds/; python setup.py install; \
cd /tmp/geoarray/; python setup.py install; \
cd /tmp/arosics/; python setup.py install; \
cd /tmp/sicor/; python setup.py install; \
cd /tmp/sicor/; make install; \
" # TODO move this to setup.py as soon as package is public
# -*- coding: utf-8 -*-
"""Unit test package for testproject."""
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment