PEP8 editing. Added linting.

.gitlab-ci.yml

@@ -10,8 +10,7 @@ variables:
 
 stages:
   - test
-  - deploy_pages
-  - deploy_to_pypi
+  - deploy
   - cleanup
 
 
@@ -33,6 +32,21 @@ test_geoarray:
     when: always
 
 
+test_styles:
+  stage: test
+  script:
+    - source /root/anaconda3/bin/activate
+    - export GDAL_DATA=/root/anaconda3/share/gdal
+    - export PYTHONPATH=$PYTHONPATH:/root  # /root <- directory needed later
+    - make lint
+  artifacts:
+    paths:
+    - tests/linting/flake8.log
+    - tests/linting/pycodestyle.log
+    - tests/linting/pydocstyle.log
+    when: always
+
+
 test_geoarray_install:
   stage: test
   script:
@@ -50,8 +64,8 @@ test_geoarray_install:
     - python -c "from geoarray import GeoArray"
 
 
-pages:
-  stage: deploy_pages
+deploy_pages:
+  stage: deploy
   dependencies:
     - test_geoarray
   script:
@@ -71,7 +85,7 @@ pages:
 
 
 deploy_pypi:
-  stage: deploy_to_pypi
+  stage: deploy
   dependencies:
     - test_geoarray
   script:   # Configure the PyPI credentials, then push the package, and cleanup the creds.

Makefile

@@ -50,7 +50,9 @@ clean-test: ## remove test and coverage artifacts
 	rm -fr nosetests.xml
 
 lint: ## check style with flake8
-	flake8 geoarray tests
+	flake8 --max-line-length=120 geoarray tests > ./tests/linting/flake8.log
+	pycodestyle geoarray --exclude="*.ipynb,*.ipynb*" --max-line-length=120 > ./tests/linting/pycodestyle.log
+	-pydocstyle geoarray > ./tests/linting/pydocstyle.log
 
 test: ## run tests quickly with the default Python
 	python setup.py test

geoarray/__init__.py

@@ -4,10 +4,10 @@ import os
 if 'MPLBACKEND' not in os.environ:
     os.environ['MPLBACKEND'] = 'Agg'
 
-from .baseclasses import GeoArray
-from .masks import BadDataMask
-from .masks import NoDataMask
-from .masks import CloudMask
+from .baseclasses import GeoArray  # noqa: E402
+from .masks import BadDataMask  # noqa: E402
+from .masks import NoDataMask  # noqa: E402
+from .masks import CloudMask  # noqa: E402
 
 
 __author__ = """Daniel Scheffler"""

geoarray/masks.py

 # -*- coding: utf-8 -*-
-__author__='Daniel Scheffler'
 
 import numpy as np
 
 # internal imports
 from .baseclasses import GeoArray
 
+__author__ = 'Daniel Scheffler'
+
 
 class BadDataMask(GeoArray):
     def __init__(self, path_or_array, geotransform=None, projection=None, bandnames=None, nodata=False, progress=True,
                  q=False):
-
         super(BadDataMask, self).__init__(path_or_array, geotransform=geotransform, projection=projection,
                                           bandnames=bandnames, nodata=nodata, progress=progress, q=q)
 
@@ -19,8 +19,7 @@ class BadDataMask(GeoArray):
             self._validate_array_values(self.arr)
             self.arr = self.arr.astype(np.bool)
 
-        # del self._mask_baddata, self.mask_baddata # TODO delete property (requires deleter)
-
+            # del self._mask_baddata, self.mask_baddata # TODO delete property (requires deleter)
 
     @property
     def arr(self):
@@ -37,17 +36,14 @@ class BadDataMask(GeoArray):
         assert len(pixelVals_in_mask) <= 2, 'Bad data mask must have only two pixel values (boolean) - 0 and 1 or ' \
                                             'False and True! The given mask for %s contains the values %s.' \
                                             % (self.basename, pixelVals_in_mask)
-        assert pixelVals_in_mask in [[0, 1], [0],[1], [False, True], [False], [True]],\
+        assert pixelVals_in_mask in [[0, 1], [0], [1], [False, True], [False], [True]], \
             'Found unsupported pixel values in the given bad data mask for %s: %s. Only the values True, False, 0 ' \
             'and 1 are supported. ' % (self.basename, pixelVals_in_mask)
 
 
-
-
 class NoDataMask(GeoArray):
     def __init__(self, path_or_array, geotransform=None, projection=None, bandnames=None, nodata=False, progress=True,
                  q=False):
-
         super(NoDataMask, self).__init__(path_or_array, geotransform=geotransform, projection=projection,
                                          bandnames=bandnames, nodata=nodata, progress=progress, q=q)
 
@@ -56,9 +52,8 @@ class NoDataMask(GeoArray):
             self._validate_array_values(self.arr)
             self.arr = self.arr.astype(np.bool)
 
-        # del self._mask_nodata, self.mask_nodata # TODO delete property (requires deleter)
-        # TODO disk-mode: init must check the numbers of bands, and ideally also the pixel values in mask
-
+            # del self._mask_nodata, self.mask_nodata # TODO delete property (requires deleter)
+            # TODO disk-mode: init must check the numbers of bands, and ideally also the pixel values in mask
 
     @property
     def arr(self):
@@ -80,20 +75,16 @@ class NoDataMask(GeoArray):
             'and 1 are supported. ' % (self.basename, pixelVals_in_mask)
 
 
-
-
 class CloudMask(GeoArray):
     def __init__(self, path_or_array, geotransform=None, projection=None, bandnames=None, nodata=None, progress=True,
                  q=False):
-
         # TODO implement class definitions and specific metadata
 
-        super(CloudMask,self).__init__(path_or_array, geotransform=geotransform, projection=projection,
+        super(CloudMask, self).__init__(path_or_array, geotransform=geotransform, projection=projection,
                                         bandnames=bandnames, nodata=nodata, progress=progress, q=q)
 
         # del self._mask_nodata, self.mask_nodata # TODO delete property (requires deleter)
         # TODO check that: "Automatically detected nodata value for CloudMask 'IN_MEM': 1.0"
 
-
     def to_ENVI_classification(self):
-        raise NotImplementedError # TODO
+        raise NotImplementedError  # TODO

geoarray/subsetting.py

 # -*- coding: utf-8 -*-
-__author__='Daniel Scheffler'
-
 
 import warnings
 import numpy as np
+from typing import Union, TypeVar
 from shapely.geometry import box, Polygon
 from py_tools_ds.geo.coord_calc import get_corner_coordinates, calc_FullDataset_corner_positions
 from py_tools_ds.geo.coord_grid import snap_bounds_to_pixGrid
@@ -12,10 +11,14 @@ from py_tools_ds.geo.projection import prj_equal
 from py_tools_ds.geo.vector.topology import get_overlap_polygon
 from py_tools_ds.numeric.array import get_outFillZeroSaturated
 
+from .baseclasses import GeoArray
+
+__author__ = 'Daniel Scheffler'
+T_ndA_gA = TypeVar(Union[np.ndarray, GeoArray])
 
 
 def _clip_array_at_mapPos(arr, mapBounds, arr_gt, band2clip=None, fillVal=0):
-    # type: (np.ndarray, tuple, tuple, int, int) -> (np.ndarray, tuple)
+    # type: (T_ndA_gA, tuple, tuple, int, int) -> (np.ndarray, tuple)
     """
     NOTE: asserts that mapBounds have the same projection like the coordinates in arr_gt
 
@@ -28,39 +31,39 @@ def _clip_array_at_mapPos(arr, mapBounds, arr_gt, band2clip=None, fillVal=0):
     """
 
     # assertions
-    assert isinstance(arr_gt, (tuple,list))
+    assert isinstance(arr_gt, (tuple, list))
     assert isinstance(band2clip, int) or band2clip is None
 
     # get array metadata
-    rows, cols             = arr.shape[:2]
-    bands                  = arr.shape[2] if arr.ndim == 3 else 1
-    arr_dtype              = arr.dtype
+    rows, cols = arr.shape[:2]
+    bands = arr.shape[2] if arr.ndim == 3 else 1
+    arr_dtype = arr.dtype
     ULxy, LLxy, LRxy, URxy = get_corner_coordinates(gt=arr_gt, rows=rows, cols=cols)
-    arrBounds              = ULxy[0], LRxy[1], LRxy[0], ULxy[1]
+    arrBounds = ULxy[0], LRxy[1], LRxy[0], ULxy[1]
 
     # snap mapBounds to the grid of the array
-    mapBounds              = snap_bounds_to_pixGrid(mapBounds, arr_gt)
+    mapBounds = snap_bounds_to_pixGrid(mapBounds, arr_gt)
     xmin, ymin, xmax, ymax = mapBounds
 
     # get out_gt and out_prj
-    out_gt               = list(arr_gt)
+    out_gt = list(arr_gt)
     out_gt[0], out_gt[3] = xmin, ymax
 
     # get image area to read
-    cS, rS = [int(i)   for i in mapXY2imXY((xmin, ymax), arr_gt)] # UL
-    cE, rE = [int(i)-1 for i in mapXY2imXY((xmax, ymin), arr_gt)] # LR
+    cS, rS = [int(i) for i in mapXY2imXY((xmin, ymax), arr_gt)]  # UL
+    cE, rE = [int(i) - 1 for i in mapXY2imXY((xmax, ymin), arr_gt)]  # LR
 
     if 0 <= rS <= rows - 1 and 0 <= rE <= rows - 1 and 0 <= cS <= cols - 1 and 0 <= cE <= cols - 1:
         """requested area is within the input array"""
-        if bands==1:
+        if bands == 1:
             out_arr = arr[rS:rE + 1, cS:cE + 1]
         else:
             out_arr = arr[rS:rE + 1, cS:cE + 1, band2clip] if band2clip is not None else arr[rS:rE + 1, cS:cE + 1, :]
     else:
         """requested area is not completely within the input array"""
         # create array according to size of mapBounds + fill with nodata
-        tgt_rows  = int(abs((ymax - ymin) / arr_gt[5]))
-        tgt_cols  = int(abs((xmax - xmin) / arr_gt[1]))
+        tgt_rows = int(abs((ymax - ymin) / arr_gt[5]))
+        tgt_cols = int(abs((xmax - xmin) / arr_gt[1]))
         tgt_bands = bands if band2clip is None else 1
         tgt_shape = (tgt_rows, tgt_cols, tgt_bands) if tgt_bands > 1 else (tgt_rows, tgt_cols)
 
@@ -68,31 +71,33 @@ def _clip_array_at_mapPos(arr, mapBounds, arr_gt, band2clip=None, fillVal=0):
             fillVal = fillVal if fillVal is not None else get_outFillZeroSaturated(arr)[0]
             out_arr = np.full(tgt_shape, fillVal, arr_dtype)
         except MemoryError:
-            raise MemoryError('Calculated target dimensions are %s. Check your inputs!' %str(tgt_shape))
+            raise MemoryError('Calculated target dimensions are %s. Check your inputs!' % str(tgt_shape))
 
         # calculate image area to be read from input array
         overlap_poly = get_overlap_polygon(box(*arrBounds), box(*mapBounds))['overlap poly']
         assert overlap_poly, 'The input array and the requested geo area have no spatial overlap.'
         xmin_in, ymin_in, xmax_in, ymax_in = overlap_poly.bounds
-        cS_in, rS_in = [int(i)   for i in mapXY2imXY((xmin_in, ymax_in), arr_gt)]
-        cE_in, rE_in = [int(i)-1 for i in mapXY2imXY((xmax_in, ymin_in), arr_gt)]  # -1 because max values do not represent pixel origins
+        cS_in, rS_in = [int(i) for i in mapXY2imXY((xmin_in, ymax_in), arr_gt)]
+        cE_in, rE_in = [int(i) - 1 for i in
+                        mapXY2imXY((xmax_in, ymin_in), arr_gt)]  # -1 because max values do not represent pixel origins
 
         # read a subset of the input array
         if bands == 1:
             data = arr[rS_in:rE_in + 1, cS_in:cE_in + 1]
         else:
             data = arr[rS_in:rE_in + 1, cS_in:cE_in + 1, band2clip] if band2clip is not None else \
-                   arr[rS_in:rE_in + 1, cS_in:cE_in + 1, :]
+                arr[rS_in:rE_in + 1, cS_in:cE_in + 1, :]
 
         # calculate correct area of out_arr to be filled and fill it with read data from input array
-        cS_out, rS_out = [int(i)   for i in mapXY2imXY((xmin_in, ymax_in), out_gt)]
-        cE_out, rE_out = [int(i)-1 for i in mapXY2imXY((xmax_in, ymin_in), out_gt)]  # -1 because max values do not represent pixel origins
+        cS_out, rS_out = [int(i) for i in mapXY2imXY((xmin_in, ymax_in), out_gt)]
+        # -1 because max values do not represent pixel origins
+        cE_out, rE_out = [int(i) - 1 for i in mapXY2imXY((xmax_in, ymin_in), out_gt)]
 
         # fill newly created array with read data from input array
-        if tgt_bands==1:
-            out_arr[rS_out:rE_out + 1, cS_out:cE_out + 1]   = data if data.ndim == 2 else data[:,:,0]
+        if tgt_bands == 1:
+            out_arr[rS_out:rE_out + 1, cS_out:cE_out + 1] = data if data.ndim == 2 else data[:, :, 0]
         else:
-            out_arr[rS_out:rE_out + 1, cS_out:cE_out + 1,:] = data
+            out_arr[rS_out:rE_out + 1, cS_out:cE_out + 1, :] = data
 
     return out_arr, out_gt
 
@@ -111,63 +116,67 @@ def get_array_at_mapPosOLD(arr, arr_gt, arr_prj, mapBounds, mapBounds_prj, band2
     :return:
     """
 
-    #[print(i,'\n') for i in [arr, arr_gt, arr_prj, mapBounds, mapBounds_prj]]
+    # [print(i,'\n') for i in [arr, arr_gt, arr_prj, mapBounds, mapBounds_prj]]
     # check if requested bounds have the same projection like the array
     samePrj = prj_equal(arr_prj, mapBounds_prj)
 
     if samePrj:
-        out_prj         = arr_prj
+        out_prj = arr_prj
         out_arr, out_gt = _clip_array_at_mapPos(arr, mapBounds, arr_gt, band2clip=band2get, fillVal=fillVal)
 
     else:
-        # calculate requested corner coordinates in the same projection like the input array (bounds are not sufficient due to projection rotation)
+        # calculate requested corner coordinates in the same projection like the input array
+        # (bounds are not sufficient due to projection rotation)
         xmin, ymin, xmax, ymax = mapBounds
         ULxy, URxy, LRxy, LLxy = (xmin, ymax), (xmax, ymax), (xmax, ymin), (xmin, ymin)
         ULxy, URxy, LRxy, LLxy = [transform_any_prj(mapBounds_prj, arr_prj, *xy) for xy in [ULxy, URxy, LRxy, LLxy]]
-        mapBounds_arrPrj       = Polygon([ULxy, URxy, LRxy, LLxy]).buffer(arr_gt[1]).bounds
+        mapBounds_arrPrj = Polygon([ULxy, URxy, LRxy, LLxy]).buffer(arr_gt[1]).bounds
 
         # read subset of input array as temporary data (that has to be reprojected later)
         temp_arr, temp_gt = _clip_array_at_mapPos(arr, mapBounds_arrPrj, arr_gt, band2clip=band2get, fillVal=fillVal)
 
         # eliminate no data area for faster warping
         try:
-            oneBandArr     = np.all(np.where(temp_arr == fillVal, 0, 1), axis=2) \
-                                if len(temp_arr.shape) > 2 else np.where(temp_arr == fillVal, 0, 1)
-            corners        = [(i[1], i[0]) for i in
-                                calc_FullDataset_corner_positions(oneBandArr, assert_four_corners=False)]
-            bounds         = [int(i) for i in Polygon(corners).bounds]
+            oneBandArr = np.all(np.where(temp_arr == fillVal, 0, 1), axis=2) \
+                if len(temp_arr.shape) > 2 else np.where(temp_arr == fillVal, 0, 1)
+            corners = [(i[1], i[0]) for i in
+                       calc_FullDataset_corner_positions(oneBandArr, assert_four_corners=False)]
+            bounds = [int(i) for i in Polygon(corners).bounds]
             cS, rS, cE, rE = bounds
 
-            temp_arr               = temp_arr[rS:rE + 1, cS:cE + 1]
+            temp_arr = temp_arr[rS:rE + 1, cS:cE + 1]
             temp_gt[0], temp_gt[3] = [int(i) for i in imXY2mapXY((cS, rS), temp_gt)]
-        except:
+        except Exception:
             warnings.warn('Could not eliminate no data area for faster warping. '
                           'Result will not be affected but processing takes a bit longer..')
 
-        #from matplotlib import pyplot as plt
-        #plt.figure()
-        #plt.imshow(temp_arr[:,:])
+        # from matplotlib import pyplot as plt
+        # plt.figure()
+        # plt.imshow(temp_arr[:,:])
 
         # calculate requested geo bounds in the target projection, snapped to the output array grid
         mapBounds = snap_bounds_to_pixGrid(mapBounds, arr_gt)
         xmin, ymin, xmax, ymax = mapBounds
-        out_gt   = list(arr_gt)
+        out_gt = list(arr_gt)
         out_gt[0], out_gt[3] = xmin, ymax
         out_rows = int(abs((ymax - ymin) / arr_gt[5]))
-        out_cols = int(abs((xmax - xmin) / arr_gt[1])) # FIXME using out_gt and outRowsCols is a workaround for not beeing able to pass output extent in the OUTPUT projection
+        # FIXME using out_gt and outRowsCols is a workaround for not beeing able to pass output extent in the OUTPUT
+        # FIXME projection
+        out_cols = int(abs((xmax - xmin) / arr_gt[1]))
 
         # reproject temporary data to target projection (the projection of mapBounds)
         from py_tools_ds.geo.raster.reproject import warp_ndarray
         out_arr, out_gt, out_prj = warp_ndarray(temp_arr, temp_gt, arr_prj, mapBounds_prj,
                                                 in_nodata=fillVal, out_nodata=fillVal, out_gt=out_gt,
-                                                outRowsCols=(out_rows, out_cols), outExtent_within=True,rsp_alg=0)  # FIXME resampling alg
+                                                outRowsCols=(out_rows, out_cols), outExtent_within=True,
+                                                rsp_alg=0)  # FIXME resampling alg
 
     return out_arr, out_gt, out_prj
 
 
 def get_array_at_mapPos(arr, arr_gt, arr_prj, out_prj, mapBounds, mapBounds_prj=None, out_gsd=None, band2get=None,
                         fillVal=0, rspAlg='near', progress=True):
-    # type: (np.ndarray, tuple, str, str, tuple, str, tuple, int, int, str, bool) -> (np.ndarray, tuple, str)
+    # type: (T_ndA_gA, tuple, str, str, tuple, str, tuple, int, int, str, bool) -> (np.ndarray, tuple, str)
     """
 
     :param arr:
@@ -187,7 +196,7 @@ def get_array_at_mapPos(arr, arr_gt, arr_prj, out_prj, mapBounds, mapBounds_prj=
 
     # check if reprojection is needed
     mapBounds_prj = mapBounds_prj if mapBounds_prj else out_prj
-    samePrj       = prj_equal(arr_prj, out_prj)
+    samePrj = prj_equal(arr_prj, out_prj)
 
     if samePrj:
         # output array is requested in the same projection like input array => no reprojection needed
@@ -196,10 +205,10 @@ def get_array_at_mapPos(arr, arr_gt, arr_prj, out_prj, mapBounds, mapBounds_prj=
         if not prj_equal(arr_prj, mapBounds_prj):
             xmin, ymin, xmax, ymax = mapBounds
             (xmin, ymin), (xmax, ymax) = \
-                [transform_any_prj(mapBounds_prj, arr_prj, X, Y) for X,Y in [(xmin, ymin), (xmax, ymax)]]
+                [transform_any_prj(mapBounds_prj, arr_prj, X, Y) for X, Y in [(xmin, ymin), (xmax, ymax)]]
             mapBounds = xmin, ymin, xmax, ymax
 
-        out_prj         = arr_prj
+        out_prj = arr_prj
         out_arr, out_gt = _clip_array_at_mapPos(arr, mapBounds, arr_gt, band2clip=band2get, fillVal=fillVal)
 
     else:
@@ -208,7 +217,7 @@ def get_array_at_mapPos(arr, arr_gt, arr_prj, out_prj, mapBounds, mapBounds_prj=
         # calculate requested geo bounds in the target projection, snapped to the output array grid
         mapBounds = snap_bounds_to_pixGrid(mapBounds, arr_gt)
 
-        arr = arr[:,:,band2get] if band2get else arr[:] # also converts GeoArray to numpy.ndarray
+        arr = arr[:, :, band2get] if band2get else arr[:]  # also converts GeoArray to numpy.ndarray
 
         from py_tools_ds.geo.raster.reproject import warp_ndarray
         out_arr, out_gt, out_prj = \

tests/test_geoarray.py

@@ -29,7 +29,6 @@ import numpy as np
 import os
 from os import path
 import osgeo.osr
-import shapely
 from shapely.geometry import Polygon
 import time
 import unittest
@@ -407,7 +406,7 @@ class Test_GeoarrayFunctions(unittest.TestCase):
         Test, if the output of the footprint_poly-function is an instance of shapely.geometry.
         """
 
-        self.assertIsInstance(self.testtiff.footprint_poly, shapely.geometry.Polygon)
+        self.assertIsInstance(self.testtiff.footprint_poly, Polygon)
 
     def test_MetadataIsInstanceOfGeodataframe(self):
         # TODO: Create a metadata-file for the tested TIFF-Image.