Commit d890e57f authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'Tests'

Conflicts:
	tests/CI_docker/context/geoarray_ci.docker
parents 5df31eb9 2fcf9777
......@@ -43,6 +43,7 @@ clean-pyc: ## remove Python file artifacts
find . -name '__pycache__' -exec rm -fr {} +
clean-test: ## remove test and coverage artifacts
coverage erase
rm -fr .tox/
rm -f .coverage
rm -fr htmlcov/
......@@ -51,19 +52,16 @@ lint: ## check style with flake8
flake8 geoarray tests
test: ## run tests quickly with the default Python
python setup.py test
python setup.py test
test-all: ## run tests on every Python version with tox
tox
coverage: ## check code coverage quickly with the default Python
coverage run --source geoarray setup.py test
coverage report -m
coverage html
$(BROWSER) htmlcov/index.html
coverage: clean-test ## check code coverage quickly with the default Python
coverage run --source geoarray setup.py test
coverage report -m
coverage html
# $(BROWSER) htmlcov/index.html
docs: ## generate Sphinx HTML documentation, including API docs
rm -f docs/geoarray.rst
......@@ -73,7 +71,7 @@ docs: ## generate Sphinx HTML documentation, including API docs
$(MAKE) -C docs html
# $(MAKE) -C docs latex
# $(MAKE) -C docs latexpdf
$(BROWSER) docs/_build/html/index.html
# $(BROWSER) docs/_build/html/index.html
servedocs: docs ## compile the docs watching for changes
watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
......
......@@ -6,5 +6,9 @@ flake8==2.6.0
tox==2.3.1
coverage==4.1
Sphinx==1.4.8
nose
nose2
nose-htmloutput
coverage
rednose
......@@ -17,7 +17,7 @@ requirements = [
# git+https://github.com/matplotlib/basemap.git # conda install --yes -c conda-forge basemap
]
test_requirements = ["coverage"]
test_requirements = ["coverage", "nose", "nose2", "nose-htmloutput", "rednose"]
setup(
name='geoarray',
......
......@@ -2,16 +2,13 @@
context_dir="./context"
dockerfile="geoarray_ci.docker"
tag="geoarray_ci:latest"
gitlab_runner="danschef_gitlab_CI_runner"
# get py_tools_ds project # TODO move this to setup.py as soon as package is public
rm -rf context/py_tools_ds
git clone https://gitext.gfz-potsdam.de/danschef/py_tools_ds.git ./context/py_tools_ds
tag="geoarray_ci:0.4"
gitlab_runner="geoarray_gitlab_CI_runner"
echo "#### Build runner docker image"
sudo docker rmi ${tag}
sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir}
#sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir}
sudo docker build --no-cache -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir}
echo "#### Create gitlab-runner (daemon) container with tag; ${tag}"
sudo docker stop ${gitlab_runner}
......
FROM centos:7
RUN yum update -y && \
yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git
yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git python-devel gdb
ENV anaconda_dl 'Anaconda3-4.3.1-Linux-x86_64.sh'
RUN /bin/bash -i -c "cd /root; wget https://repo.continuum.io/archive/$anaconda_dl ; \
......@@ -9,22 +9,19 @@ RUN /bin/bash -i -c "cd /root; wget https://repo.continuum.io/archive/$anaconda_
# 'conda install --yes -c conda-forge 'icu=58.*' lxml' fixing a bug for conda-forge gdal build, as of 06/2017
RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
conda install --yes pyqt coverage ; \
conda install --yes pyqt ; \
conda install --yes -c ioam holoviews=1.8.0 ; \
conda install --yes -c ioam bokeh ; \
conda install --yes -c conda-forge gdal ; \
conda install --yes -c conda-forge 'icu=58.*' lxml ; \
conda install --yes -c conda-forge glymur pygrib rasterio pyproj cachetools basemap; \
pip install shapely geopandas pandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse \
folium geojson"
folium geojson py_tools_ds nose nose2 nose-htmloutput coverage rednose"
# copy some needed stuff to /root
#COPY *.pkl /root/ # EXAMPLE
#COPY *.h5 /root/ # EXAMPLE
# copy py_tools_ds code to /tmp # TODO move this to setup.py as soon as package is public
COPY py_tools_ds /tmp/py_tools_ds
# install git lfs
RUN /bin/bash -i -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | bash"
......@@ -40,7 +37,3 @@ RUN bash -i -c 'source /root/anaconda3/bin/activate; \
sed -i "/backend : tkagg/c\backend : Agg" $(python -c "import matplotlib; print(matplotlib.matplotlib_fname())"); \
sed -i "/backend : Qt4Agg/c\backend : Agg" $(python -c "import matplotlib; print(matplotlib.matplotlib_fname())"); \
sed -i "/backend : qt5agg/c\backend : Agg" $(python -c "import matplotlib; print(matplotlib.matplotlib_fname())")'
RUN bash -i -c "source /root/anaconda3/bin/activate; \
cd /tmp/py_tools_ds/; python setup.py install;" # TODO move this to setup.py as soon as package is public
......@@ -84,6 +84,7 @@ class Test_GeoarrayAppliedOnPathArray(unittest.TestCase):
given_pszProj4_string = '+proj=utm +zone=33 +datum=WGS84 +units=m +no_defs'
expected_epsg = 32633
given_nodata = -9999.0
given_bandnames = ['B1', 'B2']
# Variable for the parametrization of the test case.
k = 0
......@@ -104,8 +105,10 @@ class Test_GeoarrayAppliedOnPathArray(unittest.TestCase):
cls.L8_2bands_extract10x11 = "".join((tests_path,"/tests/data/L8_2bands_extract10x11.tif"))
cls.testtiff = GeoArray(cls.L8_2bands_extract10x11)
# Serialize the "GeoArray"-class to the "data"-directory
with open("".join((tests_path,"/tests/data/testtiff_path.tmp")), "wb") as f:
# Serialize the "GeoArray"-class to the "../tests/data/output"-directory
serialized_testtiff_path = "".join((tests_path,"/tests/data/output/testtiff_path.tmp"))
os.makedirs(os.path.dirname(serialized_testtiff_path), exist_ok=True)
with open(serialized_testtiff_path, "wb") as f:
dill.dump(cls.testtiff, f)
if cls.k == 1:
......@@ -119,11 +122,14 @@ class Test_GeoarrayAppliedOnPathArray(unittest.TestCase):
cls.given_projection = srs.ExportToWkt()
# Creating the "GeoArray"-instance with a NUMPY ARRAY.
# The notdata-value will be set in the test "test_NoDataValueOfTiff".
cls.testtiff = GeoArray(cls.L8_2bands_extract10x11, geotransform=cls.given_geotransform,
projection=cls.given_projection, nodata=cls.given_nodata)
projection=cls.given_projection, bandnames=cls.given_bandnames)
# Serialize the "GeoArray"-class to the "data"-directory
with open("".join((tests_path,"/tests/data/testtiff_array.tmp")),"wb") as f:
# Serialize the "GeoArray"-class to the "../tests/data/output"-directory
serialized_testtiff_array = "".join((tests_path, "/tests/data/output/testtiff_array.tmp"))
os.makedirs(os.path.dirname(serialized_testtiff_array), exist_ok=True)
with open(serialized_testtiff_array,"wb") as f:
dill.dump(cls.testtiff, f)
......@@ -197,9 +203,8 @@ class Test_GeoarrayAppliedOnPathArray(unittest.TestCase):
def test_Bandnames(self):
"""
Testing the function: bandnames.
Test, if the default band names were correctly assigned.
Test, if the default (for k=0) and set (for k=1) band names respectively were correctly assigned.
"""
self.assertEqual(self.testtiff.bandnames, self.expected_bandnames,
msg="The bandnames of the Tiff-file are different than ['B1', 'B2'] (format: OrderedDict)")
......@@ -309,15 +314,20 @@ class Test_GeoarrayAppliedOnPathArray(unittest.TestCase):
def test_NoDataValueOfTiff(self):
"""
Testing the function: nodata,
indirect testing of the function: set_gdalDataset_meta(!).
Test, if the nodata value of the image was correctly assigned.
indirect testing of the functions: find_noDataVal(), set_gdalDataset_meta(!).
Test, if the set/default nodata value of the GeoArray-instance was correctly assigned.
"""
if self.k == 1:
self.assertIsNone(self.testtiff.nodata,
msg="The nodata-value of the 'GeoArray'-object instanced with a numpy.array and without "
"declaring a nodata-value is not automatically set to 'None'! "
"The remaining assertion in the test 'test_NoDataValueOfTiff' will not be executed.")
self.testtiff.nodata = self.given_nodata
self.assertEqual(self.testtiff.nodata, self.given_nodata,
msg="The nodata-value of the tested Tiff-file (%s) is not as expected (%s)!"
%(self.testtiff.nodata, self.given_nodata))
%(self.testtiff.nodata, self.given_nodata))
......@@ -369,23 +379,25 @@ class Test_GeoarrayFunctions(unittest.TestCase):
# Opening the temporary serialized variables (see setUpClass of test case 1) to re-use in the new test case
# without the need to inherit the variables from test case 1.
if cls.k == 0:
with open("".join((tests_path, "/tests/data/testtiff_path.tmp")), "rb") as f:
with open("".join((tests_path, "/tests/data/output/testtiff_path.tmp")), "rb") as f:
cls.testtiff = dill.load(f)
if cls.k == 1:
with open("".join((tests_path, "/tests/data/testtiff_array.tmp")), "rb") as f:
with open("".join((tests_path, "/tests/data/output/testtiff_array.tmp")), "rb") as f:
cls.testtiff = dill.load(f)
@classmethod
def tearDownClass(cls):
# Removing the temporary serialized variables from the 'data'-directory. If test case 2 is not executed,
# the files will be removed at the end of this script in the "if __name__ == '__main__'"-code segment.
if cls.k == 0:
os.remove("".join((tests_path, "/tests/data/testtiff_path.tmp")))
if cls.k == 1:
os.remove("".join((tests_path, "/tests/data/testtiff_array.tmp")))
# Removing the "../tests/data/output"-directory with all files. If test case 2 is not executed, the files
# will be removed at the end of this script in the "if __name__ == '__main__'"-code segment.
data_dir = "".join((tests_path, "/tests/data/output/"))
data_dir_list = os.listdir(data_dir)
for files in data_dir_list:
os.remove(path.join(data_dir, files))
os.rmdir(data_dir)
......@@ -410,15 +422,23 @@ class Test_GeoarrayFunctions(unittest.TestCase):
def test_MaskBaddataIsNone(self):
# TODO: Setting a baddata mask and testing, if mask_baddata is an instance of the BadDataMask-class (masks.py).
def test_MaskBaddataOffTiff(self):
"""
Testing the function: mask_baddata.
Test, if the output of the mask_baddata-function is "None" since a baddata-mask is not set within the used
TIFF-image.
Test,
a) if the output of the mask_baddata-function is "None", when the baddata-mask is not set, and
b) if the output of the mask_baddata-function is an instance of "BadDataMask"(class, defined in masks.py).
"""
self.L8_BadDataMask10x11 = "".join((tests_path, "/tests/data/L8_BadDataMask10x11.tif"))
bdm = masks.BadDataMask(self.L8_BadDataMask10x11)
self.assertIsNone(self.testtiff.mask_baddata)
for i in range(0, 2, 1):
with self.subTest(i=1):
if i == 0:
self.assertIsNone(self.testtiff.mask_baddata)
if i == 1:
self.testtiff.mask_baddata = bdm
self.assertIsInstance(self.testtiff.mask_baddata, masks.BadDataMask)
......@@ -435,7 +455,8 @@ class Test_GeoarrayFunctions(unittest.TestCase):
def test_MetadataIsInstanceOfGeodataframe(self):
# TODO test, if the metadata-function gives an output
# TODO: Create a metadata-file for the tested TIFF-Image.
# TODO: Test, if the metadata-function gives an output
"""
Testing the function: metadata.
Test, if the output of the metadata-function is an instance of GeoDataFrame.
......@@ -445,6 +466,54 @@ class Test_GeoarrayFunctions(unittest.TestCase):
def test_SaveTiffToDisk(self):
"""
Testing the function: save,
test with 2 stages.
Stage 1: After saving the original TIFF-Image (output format "GTiff") to the "../tests/data/output"-directory
with the "save"-function and then instancing the copied TIFF-Image as "GeoArray"-object, it is
first tested, if the file exists in the aforementioned directory.
When the file exists, induction of stage 2...
Stage 2: Testing, whether a) the object "testtiff_copy" is an instance of the "GeoArray"-class, and b) if the
numpy.array of the copied TIFF-Image is identical to the numpy.array of the original TIFF-Image.
If the newly created file does not exist in the "../tests/data/output"-directory (Stage 1), the whole test
will be skipped.
"""
# Saving the "GeoArray"-instance "testtiff" with the "save"-function of the "GeoArray"-class to the
# "../tests/data/output"-directory of this repository and instancing it again as a "GeoArray"-object.
L8_2bands_extract10x11_copy = "".join((tests_path, "/tests/data/output/L8_BadDataMask10x11_copy.tif"))
self.testtiff.save(L8_2bands_extract10x11_copy, fmt="GTiff")
testtiff_copy = GeoArray(L8_2bands_extract10x11_copy)
if path.exists(L8_2bands_extract10x11_copy):
assert isinstance(testtiff_copy, GeoArray) and \
np.array_equal(self.testtiff[:], testtiff_copy[:]), \
"The copy of the original TIFF-Image, saved by the 'GeoArray' save-function, %s an instance of " \
"class 'GeoArray'. Its numpy.array %s identical to the numpy.array of the original TIFF-Image." \
% ("IS" if isinstance(testtiff_copy, GeoArray) else "IS NOT",
"IS" if np.array_equal(self.testtiff[:], testtiff_copy[:]) else "IS NOT")
else:
self.skipTest("The file '%s' does not exists in the directory '%s'. "
"The test 'test_SaveTiffToDisk' will be skipped."
%(path.basename(L8_2bands_extract10x11_copy), path.dirname(L8_2bands_extract10x11_copy)))
def test_PlottingFunctions(self):
# FIXME: This test-function is not yet complete! + TODO: Use other parameters of the plot functions!
# TODO: Idea: Testing of plot-functions with "from matplotlib.testing.decorators import image_comparison" (?),
# TODO example of matplotlib.testing.decorators under: https://github.com/ketch/griddle/blob/master/tests/plot_tests.py
self.testtiff.show()
# self.testtiff.show(interactive=True) # only works if test is started with ipython.
# self.testtiff.show_map()
# self.testtiff.show_map_utm() # Function still under construction.
self.testtiff.show_histogram()
# ----> TODO: Write tests for the remaining functions!
......@@ -487,8 +556,8 @@ if __name__ == '__main__':
# If-else loop: Creating and executing the second test suite (created with the tests of the test case 2), only
# if the first test case was successful and no test was skipped. Otherwise, a note will be printed as output to
# inform the user that test case 2 was skipped. Additionally, the serialized variable "testtiff" from test
# case 1 will be removed in the else-statement.
# inform the user that test case 2 was skipped. Additionally, the "../tests/data/output"-directory with files
# will be removed in the else-statement.
if testResult.wasSuccessful() and testResult.skipped == []:
other_suite = unittest.TestSuite()
more_test = None
......@@ -504,10 +573,13 @@ if __name__ == '__main__':
# TODO: Change the verbosity of the TextTestRunner to 1
else:
if k == 0:
os.remove("".join((tests_path, "/tests/data/testtiff_path.tmp")))
if k == 1:
os.remove("".join((tests_path, "/tests/data/testtiff_array.tmp")))
data_dir = "".join((tests_path, "/tests/data/output/"))
data_dir_list = os.listdir(data_dir)
for files in data_dir_list:
os.remove(path.join(data_dir, files))
os.rmdir(data_dir)
print("Test case 2: Since %s error/ %s failure/ %s skip occured in the first test case the second test "
"case 'Test_GeoarrayFunctions' will be skipped."
%(len(testResult.errors), len(testResult.failures), len(testResult.skipped)))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment