Commit 11d00576 authored by Daniel Scheffler's avatar Daniel Scheffler Committed by Daniel Scheffler
Browse files

Revised docker container setup. Verified Windows compatibility for tests. ...

Revised docker container setup. Verified Windows compatibility for tests.  Updated minimal versions of py_tools_ds and geoarray. Updated README files. Some PEP8-editing. Disabled multiprocessing in case of Python 2. Verified Python 2 compatibility.
parent 26a76f7e
Pipeline #1103 failed with stages
in 2 minutes and 8 seconds
......@@ -42,8 +42,6 @@ test_arosics_install:
- export GDAL_DATA=/root/anaconda3/share/gdal
- conda create -y -q --name arosics python=3.5
- source activate arosics
# resolve some requirements with conda
- conda install --yes -q -c conda-forge numpy gdal scikit-image matplotlib pyproj rasterio fiona shapely
- python setup.py install
- cd ..
- pwd
......
......@@ -58,7 +58,7 @@ Using [conda](https://conda.io/docs/), the recommended approach is:
# create virtual environment for arosics, this is optional
conda create -y -q --name arosics python=3
source activate arosics
conda install -y -q -c conda-forge numpy gdal scikit-image matplotlib pyproj rasterio fiona shapely
conda install -y -q -c conda-forge numpy gdal scikit-image matplotlib pyproj rasterio shapely
conda install -y -q -c conda-forge pyfftw basemap pykrige # these libraries are optional
```
......@@ -81,7 +81,6 @@ PATH=$PATH:/path/to/your/installation/folder/arosics:/path/to/your/installation/
AROSICS has been tested with Python 3.4+ and Python 2.7. It should be fully compatible to all Python versions above 2.7.
Before running AROSICS, make sure your GDAL_DATA environment variable is set!
# Modules
......
......@@ -52,7 +52,7 @@ Using conda_, the recommended approach is:
# create virtual environment for arosics, this is optional
conda create -y -q --name arosics python=3
source activate arosics
conda install -y -q -c conda-forge numpy gdal scikit-image matplotlib pyproj rasterio fiona shapely
conda install -y -q -c conda-forge numpy gdal scikit-image matplotlib pyproj rasterio shapely
conda install -y -q -c conda-forge pyfftw basemap pykrige # these libraries are optional
......@@ -73,8 +73,6 @@ Or clone the repository via GIT and update the PATH environment variable:
git clone https://gitext.gfz-potsdam.de/danschef/py_tools_ds.git
PATH=$PATH:/path/to/your/installation/folder/arosics:/path/to/your/installation/folder/geoarray:/path/to/your/installation/folder/py_tools_ds
Before running AROSICS, make sure your GDAL_DATA environment variable is set!
Credits
-------
......
......@@ -1507,7 +1507,8 @@ class COREG(object):
pathVRT = os.path.splitext(os.path.basename(self.shift.path))[0] + '.vrt'
ds_im2shift = gdal.Open(self.shift.path)
dst_ds = (lambda drv: drv.CreateCopy(pathVRT, ds_im2shift, 0))(gdal.GetDriverByName('VRT'))
drv = gdal.GetDriverByName('VRT')
dst_ds = drv.CreateCopy(pathVRT, ds_im2shift, 0)
dst_ds.SetGeoTransform(gt_shifted)
del dst_ds, ds_im2shift
......
......@@ -3,6 +3,7 @@
import warnings
import os
from copy import copy
from six import PY3
# custom
try:
......@@ -145,12 +146,15 @@ class COREG_LOCAL(object):
:param ignore_errors(bool): Useful for batch processing. (default: False)
"""
# assertions
# assertions / input validation
assert gdal.GetDriverByName(fmt_out), "'%s' is not a supported GDAL driver." % fmt_out
if match_gsd and out_gsd:
warnings.warn("'-out_gsd' is ignored because '-match_gsd' is set.\n")
if out_gsd:
assert isinstance(out_gsd, list) and len(out_gsd) == 2, 'out_gsd must be a list with two values.'
if (not PY3 and CPUs is None) or (isinstance(CPUs, int) and CPUs > 1):
CPUs = 1
warnings.warn('Multiprocessing is currently not supported under Python 2. Using singleprocessing.')
self.params = dict([x for x in locals().items() if x[0] != "self" and not x[0].startswith('__')])
......
......@@ -4,18 +4,21 @@ import numpy as np
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
def subplot_2dline(XY_tuples, titles=None, shapetuple=None, grid=False):
shapetuple = (1,len(XY_tuples)) if shapetuple is None else shapetuple
assert titles is None or len(titles)==len(XY_tuples), \
shapetuple = (1, len(XY_tuples)) if shapetuple is None else shapetuple
assert titles is None or len(titles) == len(XY_tuples), \
'List in titles keyword must have the same length as the passed XY_tuples.'
norm = lambda array, normto: [float(i)*(normto/max(array)) for i in array]
fig = plt.figure(figsize=norm(plt.figaspect(shapetuple[0]/shapetuple[1]*1.), 10))
for i,XY in enumerate(XY_tuples):
ax = fig.add_subplot(shapetuple[0], shapetuple[1], i+1)
X,Y = XY
ax.plot(X,Y,linestyle='-')
if titles is not None: ax.set_title(titles[i])
if grid: ax.grid(which='major', axis='both', linestyle='-')
norm = lambda array, normto: [float(i) * (normto / max(array)) for i in array]
fig = plt.figure(figsize=norm(plt.figaspect(shapetuple[0] / shapetuple[1] * 1.), 10))
for i, XY in enumerate(XY_tuples):
ax = fig.add_subplot(shapetuple[0], shapetuple[1], i + 1)
X, Y = XY
ax.plot(X, Y, linestyle='-')
if titles is not None:
ax.set_title(titles[i])
if grid:
ax.grid(which='major', axis='both', linestyle='-')
plt.tight_layout()
plt.show(block=True)
......@@ -23,14 +26,15 @@ def subplot_2dline(XY_tuples, titles=None, shapetuple=None, grid=False):
def subplot_imshow(ims, titles=None, shapetuple=None, grid=False):
ims = [ims] if not isinstance(ims,list) else ims
assert titles is None or len(titles)==len(ims), 'Error: Got more or less titles than images.'
shapetuple = (1,len(ims)) if shapetuple is None else shapetuple
norm = lambda array, normto: [float(i)*(normto/max(array)) for i in array]
fig,axes = plt.subplots(shapetuple[0],shapetuple[1], figsize=norm(plt.figaspect(shapetuple[0]/shapetuple[1]*1.), 20))
[axes[i].imshow(im,cmap='gray',interpolation='none', vmin=np.percentile(im,2), vmax = np.percentile(im,98)) \
for i,im in enumerate(ims)]
ims = [ims] if not isinstance(ims, list) else ims
assert titles is None or len(titles) == len(ims), 'Error: Got more or less titles than images.'
shapetuple = (1, len(ims)) if shapetuple is None else shapetuple
norm = lambda array, normto: [float(i) * (normto / max(array)) for i in array]
fig, axes = plt.subplots(shapetuple[0], shapetuple[1],
figsize=norm(plt.figaspect(shapetuple[0] / shapetuple[1] * 1.), 20))
[axes[i].imshow(im, cmap='gray', interpolation='none', vmin=np.percentile(im, 2), vmax=np.percentile(im, 98))
for i, im in enumerate(ims)]
if titles is not None:
[axes[i].set_title(titles[i]) for i in range(len(ims))]
if grid:
......@@ -41,21 +45,23 @@ def subplot_imshow(ims, titles=None, shapetuple=None, grid=False):
return fig
def subplot_3dsurface(ims,shapetuple=None):
ims = [ims] if not isinstance(ims,list) else ims
shapetuple = (1,len(ims)) if shapetuple is None else shapetuple
norm = lambda array, normto: [float(i)*(normto/max(array)) for i in array]
fig = plt.figure(figsize=norm(plt.figaspect((shapetuple[0]/2.)/shapetuple[1]*1.), 20))
for i,im in enumerate(ims):
ax = fig.add_subplot(shapetuple[0], shapetuple[1], i+1, projection='3d')
x = np.arange(0, im.shape[0], 1)
y = np.arange(0, im.shape[1], 1)
X, Y = np.meshgrid(x, y)
Z = im.reshape(X.shape)
ax.plot_surface(X, Y, Z, cmap = plt.cm.hot)
def subplot_3dsurface(ims, shapetuple=None):
ims = [ims] if not isinstance(ims, list) else ims
shapetuple = (1, len(ims)) if shapetuple is None else shapetuple
norm = lambda array, normto: [float(i) * (normto / max(array)) for i in array]
fig = plt.figure(figsize=norm(plt.figaspect((shapetuple[0] / 2.) / shapetuple[1] * 1.), 20))
for i, im in enumerate(ims):
ax = fig.add_subplot(shapetuple[0], shapetuple[1], i + 1, projection='3d')
x = np.arange(0, im.shape[0], 1)
y = np.arange(0, im.shape[1], 1)
X, Y = np.meshgrid(x, y)
Z = im.reshape(X.shape)
ax.plot_surface(X, Y, Z, cmap=plt.cm.hot)
ax.contour(X, Y, Z, zdir='x', cmap=plt.cm.coolwarm, offset=0)
ax.contour(X, Y, Z, zdir='y', cmap=plt.cm.coolwarm, offset=im.shape[1])
ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z')
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')
plt.tight_layout()
plt.show(block=True)
......
......@@ -14,7 +14,7 @@ Using conda_, the recommended approach is:
# create virtual environment for arosics, this is optional
conda create -y -q --name arosics python=3
source activate arosics
conda install -y -q -c conda-forge numpy gdal scikit-image matplotlib pyproj rasterio fiona shapely
conda install -y -q -c conda-forge numpy gdal scikit-image matplotlib pyproj rasterio shapely
conda install -y -q -c conda-forge pyfftw basemap pykrige # these libraries are optional
......
......@@ -12,8 +12,8 @@ with open('README.rst') as readme_file:
with open('HISTORY.rst') as history_file:
history = history_file.read()
requirements = ['numpy', 'gdal', 'shapely', 'scikit-image', 'matplotlib', 'geopandas', 'spectral', 'geoarray>=0.5.11',
'py_tools_ds>=0.4.6', 'plotly', 'cmocean'
requirements = ['numpy', 'gdal', 'shapely', 'scikit-image', 'matplotlib', 'geopandas', 'spectral', 'geoarray>=0.5.14',
'py_tools_ds>=0.6.0', 'plotly', 'cmocean', 'six',
# 'pykrige' # conda install --yes -c conda-forge pykrige
# 'pyfftw', # conda install --yes -c conda-forge pyfftw=0.10.4 ; \
# 'basemap', # conda install --yes -c conda-forge basemap; \
......@@ -23,7 +23,7 @@ setup_requirements = [
# TODO(danschef): put setup requirements (distutils extensions, etc.) here
]
test_requirements = ['coverage']
test_requirements = ['coverage', 'nose', 'nose-htmloutput', 'rednose']
setup(
name='arosics',
......
......@@ -2,7 +2,7 @@
context_dir="./context"
dockerfile="arosics_ci.docker"
tag="arosics_ci:latest"
tag="arosics_ci:0.4.30"
gitlab_runner="arosics_gitlab_CI_runner"
echo "#### Build runner docker image"
......
......@@ -13,15 +13,14 @@ RUN /bin/bash -i -c "cd /root; wget https://repo.continuum.io/archive/$anaconda_
# 'conda install --yes -c conda-forge 'icu=58.*' lxml' fixing a bug for conda-forge gdal build, as of 06/2017
# holoviews and bokeh is only needed for geoarray because its not installable via pip during geoarray.setup.py
RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
conda install --yes pyqt coverage; \
conda install --yes -c conda-forge gdal ; \
conda install --yes pyqt; \
conda install --yes -c conda-forge pyfftw=0.10.4 ; \
conda install --yes -c ioam holoviews=1.8.0 ; \
conda install --yes -c ioam bokeh ; \
conda install --yes -c conda-forge numpy gdal scikit-image matplotlib pyproj rasterio shapely basemap pykrige; \
conda install --yes -c conda-forge 'icu=58.*' lxml ; \
conda install --yes -c conda-forge glymur pygrib rasterio pyproj cachetools basemap; \
pip install shapely geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse spectral \
geoarray py_tools_ds plotly flake8 pycodestyle pylint pydocstyle nose nose2 nose-htmloutput rednose" # must include all the requirements needed to build the docs!
pip install geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse spectral \
geoarray py_tools_ds plotly flake8 pycodestyle pylint pydocstyle coverage nose nose2 nose-htmloutput rednose" # must include all the requirements needed to build the docs!
# copy some needed stuff to /root
#COPY *.pkl /root/ # EXAMPLE
......
......@@ -6,31 +6,31 @@ import os
# custom
import arosics
tests_path = os.path.abspath(os.path.join(arosics.__file__,"../../tests"))
tests_path = os.path.abspath(os.path.join(arosics.__file__, "..", "..", "tests"))
# define test data pathes
test_cases = dict(
INTER1=dict(
ref_path = os.path.join(tests_path, 'data/testcase_inter1_S2A_S2A/ref_S2A_20160608T153121_T33UUU_sub.tif'),
tgt_path = os.path.join(tests_path, 'data/testcase_inter1_S2A_S2A/tgt_S2A_20160529T153631_T33UUU_sub.tif'),
kwargs_global = dict(
path_out = os.path.join(tests_path, 'output/testcase_inter1_S2A_S2A/'
'tgt_S2A_20160529T153631_T33UUU_sub_CR_global.bsq'),
footprint_poly_ref= 'POLYGON ((340870 5862000, 354000 5862000, 354000 5830000, 331320 5830000, '
'340870 5862000))',
footprint_poly_tgt= 'POLYGON ((341890 5866490, 356180 5866490, 356180 5834970, 335440 5834970, '
'335490 5845270, 341890 5866490))',
progress = False,
v = 0),
wp_inside = (344720, 5848485),# inside of overlap
wp_covering_nodata = (339611, 5856426), # close to the image edge of the input images -> win>64px covers nodata
wp_close_to_edge = (353810, 5840516), # close to the image edge of the input images -> win>64px covers nodata
wp_cloudy = (353308, 5859404), # at a cloudy position of the target image
wp_outside = (349533, 5818862), # outside of overlap
kwargs_local = dict(
grid_res = 100,
path_out=os.path.join(tests_path, 'output/testcase_inter1_S2A_S2A/'
ref_path=os.path.join(tests_path, 'data', 'testcase_inter1_S2A_S2A', 'ref_S2A_20160608T153121_T33UUU_sub.tif'),
tgt_path=os.path.join(tests_path, 'data', 'testcase_inter1_S2A_S2A', 'tgt_S2A_20160529T153631_T33UUU_sub.tif'),
kwargs_global=dict(
path_out=os.path.join(tests_path, 'output', 'testcase_inter1_S2A_S2A/'
'tgt_S2A_20160529T153631_T33UUU_sub_CR_global.bsq'),
footprint_poly_ref='POLYGON ((340870 5862000, 354000 5862000, 354000 5830000, 331320 5830000, '
'340870 5862000))',
footprint_poly_tgt='POLYGON ((341890 5866490, 356180 5866490, 356180 5834970, 335440 5834970, '
'335490 5845270, 341890 5866490))',
progress=False,
v=0),
wp_inside=(344720, 5848485), # inside of overlap
wp_covering_nodata=(339611, 5856426), # close to the image edge of the input images -> win>64px covers nodata
wp_close_to_edge=(353810, 5840516), # close to the image edge of the input images -> win>64px covers nodata
wp_cloudy=(353308, 5859404), # at a cloudy position of the target image
wp_outside=(349533, 5818862), # outside of overlap
kwargs_local=dict(
grid_res=100,
path_out=os.path.join(tests_path, 'output', 'testcase_inter1_S2A_S2A',
'tgt_S2A_20160529T153631_T33UUU_sub_CR_local.bsq'),
progress = False)
progress=False)
)
)
......@@ -3,12 +3,10 @@
"""Tests for the global co-registration module of AROSICS."""
import unittest
import shutil
import os
# custom
from .cases import test_cases
from arosics import COREG
......@@ -24,7 +22,6 @@ class COREG_GLOBAL_init(unittest.TestCase):
self.coreg_kwargs = test_cases['INTER1']['kwargs_global']
self.coreg_kwargs['wp'] = test_cases['INTER1']['wp_inside']
def test_coreg_init_from_disk(self):
self.CRL = COREG(self.ref_path, self.tgt_path, **self.coreg_kwargs)
......@@ -41,7 +38,6 @@ class COREG_GLOBAL_init(unittest.TestCase):
self.CRL = COREG(self.ref_gA, self.tgt_gA, **self.coreg_kwargs)
class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase):
"""Test case for the complete workflow of global co-registration based on two Sentinel-2 datasets, one with
~25% cloud cover, the other one without any clouds. The subsets cover the S2A tiles only partly (nodata areas
......@@ -53,14 +49,12 @@ class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase):
self.tgt_path = test_cases['INTER1']['tgt_path']
self.coreg_kwargs = test_cases['INTER1']['kwargs_global']
def tearDown(self):
"""Delete output."""
dir_out = os.path.dirname(self.coreg_kwargs['path_out'])
if os.path.isdir(dir_out):
shutil.rmtree(dir_out)
def run_shift_detection_correction(self, ref, tgt, **params):
# get instance of COREG_LOCAL object
CR = COREG(ref, tgt, **params)
......@@ -77,7 +71,6 @@ class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase):
return CR
def test_shift_calculation_with_default_params(self):
"""Test with default parameters - should compute X/Y shifts properly ad write the de-shifted target image."""
......@@ -87,7 +80,7 @@ class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase):
footprint_poly_tgt=None))
self.assertTrue(CR.success)
#@unittest.SkipTest
# @unittest.SkipTest
def test_shift_calculation_verboseMode(self):
"""Test the verbose mode - runs the functions of the plotting submodule."""
......@@ -95,7 +88,6 @@ class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase):
**dict(self.coreg_kwargs, v=True))
self.assertTrue(CR.success)
def test_shift_calculation_windowCoveringNodata(self):
"""Test shift detection in case the given matching window (defined by 'wp' and 'ws' covers the nodata area
of an input image.
......@@ -107,11 +99,10 @@ class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase):
# TODO compare to expected results
CR = self.run_shift_detection_correction(self.ref_path, self.tgt_path,
**dict(self.coreg_kwargs,
wp = test_cases['INTER1']['wp_covering_nodata'],
ws = (256,256)))
wp=test_cases['INTER1']['wp_covering_nodata'],
ws=(256, 256)))
self.assertTrue(CR.success)
def test_shift_calculation_windowAtImageEdge(self):
"""Test shift detection in case the given matching window is close to an image edge without covering any nodata
area.
......@@ -123,10 +114,9 @@ class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase):
# TODO compare to expected results
CR = self.run_shift_detection_correction(self.ref_path, self.tgt_path,
**dict(self.coreg_kwargs,
wp=test_cases['INTER1']['wp_close_to_edge'], ws=(256,256)))
wp=test_cases['INTER1']['wp_close_to_edge'], ws=(256, 256)))
self.assertTrue(CR.success)
def test_shift_calculation_windowOutside(self):
"""Test if shift computation properly raises a ValueError if the given window position is outside of the image
overlap."""
......@@ -136,7 +126,6 @@ class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase):
**dict(self.coreg_kwargs,
wp=test_cases['INTER1']['wp_outside']))
def test_shift_calculation_windowAtClouds(self):
"""Test if shift computation properly raises a RunTimeError if the matching window is centered at a cloudy
image position.
......@@ -145,30 +134,27 @@ class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase):
with self.assertRaises(RuntimeError):
self.run_shift_detection_correction(self.ref_path, self.tgt_path,
**dict(self.coreg_kwargs,
wp=test_cases['INTER1']['wp_cloudy'], ws=(256,256)))
wp=test_cases['INTER1']['wp_cloudy'], ws=(256, 256)))
def test_shift_calculation_differentInputGrids(self):
""""""
self.skipTest('Not yet implemented.')
def test_shift_calculation_withoutPyFFTW(self):
""""""
self.skipTest('Not yet implemented.')
def test_shift_calculation_SSIMdecreases(self):
""""""
self.skipTest('Not yet implemented.')
#@unittest.SkipTest
def test_plotting_after_shift_calculation(self):#, mock_show):
# @unittest.SkipTest
def test_plotting_after_shift_calculation(self): # , mock_show):
""""""
#mock_show.return_value = None # probably not necessary here in your case
# mock_show.return_value = None # probably not necessary here in your case
CR = self.run_shift_detection_correction(self.ref_path, self.tgt_path, **self.coreg_kwargs)
self.assertTrue(CR.success)
......@@ -181,6 +167,5 @@ class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase):
# CR.show_matchWin(interactive=False, deshifted=True)
CR.show_image_footprints()
#if __name__ == '__main__':
# if __name__ == '__main__':
# unittest.main(argv=['first-arg-is-ignored'],exit=False, verbosity=2)
......@@ -3,7 +3,6 @@
"""Tests for the local co-registration module of AROSICS."""
import unittest
import shutil
import os
......@@ -71,7 +70,6 @@ class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase):
self.assertTrue(os.path.exists(self.coreg_kwargs['path_out']),
'Output of local co-registration has not been written.')
# if __name__ == '__main__':
# unittest.main(argv=['first-arg-is-ignored'],exit=False, verbosity=2)
#
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment