Commit d1a17054 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

PEP8 editing. Added linting.

parent 533db50b
Pipeline #1181 passed with stages
in 5 minutes and 58 seconds
......@@ -10,8 +10,7 @@ variables:
stages:
- test
- deploy_pages
- deploy_to_pypi
- deploy
- cleanup
......@@ -32,6 +31,22 @@ test_py_tools_ds:
when: always
test_styles:
stage: test
script:
- source /root/anaconda3/bin/activate
- export GDAL_DATA=/root/anaconda3/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- pip install flake8 pycodestyle pylint pydocstyle # TODO remove as soon as docker container is rebuilt
- make lint
artifacts:
paths:
- tests/linting/flake8.log
- tests/linting/pycodestyle.log
- tests/linting/pydocstyle.log
when: always
test_py_tools_ds_install:
stage: test
script:
......@@ -49,8 +64,8 @@ test_py_tools_ds_install:
- python -c "from py_tools_ds.geo.vector.geometry import boxObj"
pages:
stage: deploy_pages
deploy_pages:
stage: deploy
dependencies:
- test_py_tools_ds
script:
......@@ -60,7 +75,6 @@ pages:
- cp nosetests.* public/nosetests_reports/
- mkdir -p public/doc
- cp -r docs/_build/html/* public/doc/
artifacts:
paths:
- public
......@@ -70,7 +84,7 @@ pages:
deploy_pypi:
stage: deploy_to_pypi
stage: deploy
dependencies:
- test_py_tools_ds
script: # Configure the PyPI credentials, then push the package, and cleanup the creds.
......
......@@ -52,7 +52,9 @@ clean-test: ## remove test and coverage artifacts
rm -fr nosetests.xml
lint: ## check style with flake8
flake8 py_tools_ds tests
flake8 --max-line-length=120 py_tools_ds tests > ./tests/linting/flake8.log
pycodestyle py_tools_ds --exclude="*.ipynb,*.ipynb*" --max-line-length=120 > ./tests/linting/pycodestyle.log
-pydocstyle py_tools_ds > ./tests/linting/pydocstyle.log
test: ## run tests quickly with the default Python
python setup.py test
......
# -*- coding: utf-8 -*-
from __future__ import (division, print_function, absolute_import, unicode_literals)
from . import compatibility
from . import convenience
from . import dtypes
from . import geo
from . import io
from . import numeric
from . import similarity
from . import web
__version__ = '0.7.1'
__versionalias__ = '20170913_01'
......
__author__='Daniel Scheffler'
from . import gdal
from . import python
__all__=['python'
'gdal',
'gdalnumeric']
\ No newline at end of file
# -*- coding: utf-8 -*-
__author__ = 'Daniel Scheffler'
......@@ -5,36 +5,32 @@ import os
try:
from osgeo import gdal
from osgeo import gdalnumeric
from osgeo import gdalconst
except ImportError:
import gdal
import gdalnumeric
import gdalconst
from ..processing.shell import subcall_with_output
from ..io.pathgen import get_tempfile
from ..io.pathgen import get_tempfile
__author__ = "Daniel Scheffler"
def Warp(destNameOrDestDS, srcDSOrSrcDSTab, options = '', format = 'GTiff',
outputBounds = None,
outputBoundsSRS = None,
xRes = None, yRes = None, targetAlignedPixels = False,
width = 0, height = 0,
srcSRS = None, dstSRS = None,
srcAlpha = False, dstAlpha = False,
warpOptions = None, errorThreshold = None,
warpMemoryLimit = None, creationOptions = None, outputType = gdal.GDT_Unknown,
workingType = gdal.GDT_Unknown, resampleAlg = None,
srcNodata = None, dstNodata = None, multithread = False,
tps = False, rpc = False, geoloc = False, polynomialOrder = None,
transformerOptions = None, cutlineDSName = None,
cutlineLayer = None, cutlineWhere = None, cutlineSQL = None, cutlineBlend = None, cropToCutline = False,
copyMetadata = True, metadataConflictValue = None,
setColorInterpretation = False,
callback = None, callback_data = None):
def Warp(destNameOrDestDS, srcDSOrSrcDSTab, options='', format='GTiff',
outputBounds=None,
outputBoundsSRS=None,
xRes=None, yRes=None, targetAlignedPixels=False,
width=0, height=0,
srcSRS=None, dstSRS=None,
srcAlpha=False, dstAlpha=False,
warpOptions=None, errorThreshold=None,
warpMemoryLimit=None, creationOptions=None, outputType=gdal.GDT_Unknown,
workingType=gdal.GDT_Unknown, resampleAlg=None,
srcNodata=None, dstNodata=None, multithread=False,
tps=False, rpc=False, geoloc=False, polynomialOrder=None,
transformerOptions=None, cutlineDSName=None,
cutlineLayer=None, cutlineWhere=None, cutlineSQL=None, cutlineBlend=None, cropToCutline=False,
copyMetadata=True, metadataConflictValue=None,
setColorInterpretation=False,
callback=None, callback_data=None):
""" This functions brings functionality of gdal.Warp() that is not available in GDAL versions below 2.1 (?).
Keyword arguments are :
......@@ -79,23 +75,23 @@ def Warp(destNameOrDestDS, srcDSOrSrcDSTab, options = '', format = 'GTiff',
"""
new_options = options
#new_options += ['-of', format]
# new_options += ['-of', format]
if outputType != gdal.GDT_Unknown:
new_options += ['-ot', gdal.GetDataTypeName(outputType) ]
new_options += ['-ot', gdal.GetDataTypeName(outputType)]
if workingType != gdal.GDT_Unknown:
new_options += ['-wt', gdal.GetDataTypeName(workingType) ]
new_options += ['-wt', gdal.GetDataTypeName(workingType)]
if outputBounds is not None:
new_options += ['-te', str(outputBounds[0]), str(outputBounds[1]), str(outputBounds[2]), str(outputBounds[3]) ]
new_options += ['-te', str(outputBounds[0]), str(outputBounds[1]), str(outputBounds[2]), str(outputBounds[3])]
if outputBoundsSRS is not None:
new_options += ['-te_srs', str(outputBoundsSRS) ]
new_options += ['-te_srs', str(outputBoundsSRS)]
if xRes is not None and yRes is not None:
new_options += ['-tr', str(xRes), str(yRes) ]
new_options += ['-tr', str(xRes), str(yRes)]
if width or height:
new_options += ['-ts', str(width), str(height)]
if srcSRS is not None:
new_options += ['-s_srs', str(srcSRS) ]
new_options += ['-s_srs', str(srcSRS)]
if dstSRS is not None:
new_options += ['-t_srs', str(dstSRS) ]
new_options += ['-t_srs', str(dstSRS)]
if targetAlignedPixels:
new_options += ['-tap']
if srcAlpha:
......@@ -122,17 +118,17 @@ def Warp(destNameOrDestDS, srcDSOrSrcDSTab, options = '', format = 'GTiff',
new_options += ['-r', 'average']
elif resampleAlg == gdal.GRA_Mode:
new_options += ['-r', 'mode']
else: # gdal.GRA_Gauss is missing
new_options += ['-r', str(resampleAlg) ]
else: # gdal.GRA_Gauss is missing
new_options += ['-r', str(resampleAlg)]
if warpMemoryLimit is not None:
new_options += ['-wm', str(warpMemoryLimit) ]
new_options += ['-wm', str(warpMemoryLimit)]
if creationOptions is not None:
for opt in creationOptions:
new_options += ['-co', opt ]
new_options += ['-co', opt]
if srcNodata is not None:
new_options += ['-srcnodata', str(srcNodata) ]
new_options += ['-srcnodata', str(srcNodata)]
if dstNodata is not None:
new_options += ['-dstnodata', str(dstNodata) ]
new_options += ['-dstnodata', str(dstNodata)]
if multithread:
new_options += ['-multi']
if tps:
......@@ -145,23 +141,23 @@ def Warp(destNameOrDestDS, srcDSOrSrcDSTab, options = '', format = 'GTiff',
new_options += ['-order', str(polynomialOrder)]
if transformerOptions is not None:
for opt in transformerOptions:
new_options += ['-to', opt ]
new_options += ['-to', opt]
if cutlineDSName is not None:
new_options += ['-cutline', str(cutlineDSName) ]
new_options += ['-cutline', str(cutlineDSName)]
if cutlineLayer is not None:
new_options += ['-cl', str(cutlineLayer) ]
new_options += ['-cl', str(cutlineLayer)]
if cutlineWhere is not None:
new_options += ['-cwhere', str(cutlineWhere) ]
new_options += ['-cwhere', str(cutlineWhere)]
if cutlineSQL is not None:
new_options += ['-csql', str(cutlineSQL) ]
new_options += ['-csql', str(cutlineSQL)]
if cutlineBlend is not None:
new_options += ['-cblend', str(cutlineBlend) ]
new_options += ['-cblend', str(cutlineBlend)]
if cropToCutline:
new_options += ['-crop_to_cutline']
if not copyMetadata:
new_options += ['-nomd']
if metadataConflictValue:
new_options += ['-cvmd', str(metadataConflictValue) ]
new_options += ['-cvmd', str(metadataConflictValue)]
if setColorInterpretation:
new_options += ['-setci']
......@@ -169,15 +165,15 @@ def Warp(destNameOrDestDS, srcDSOrSrcDSTab, options = '', format = 'GTiff',
drv = gdal.GetDriverByName('ENVI')
inPath = get_tempfile(prefix='warp_in_', ext='.bsq')
drv.CreateCopy(inPath, srcDSOrSrcDSTab)
srcDSOrSrcDSTab = None
elif isinstance(srcDSOrSrcDSTab,str):
del srcDSOrSrcDSTab
elif isinstance(srcDSOrSrcDSTab, str):
inPath = srcDSOrSrcDSTab
else:
raise ValueError
warpedPath = get_tempfile(prefix='warp_out_', ext='.bsq')
out, exitcode, err = subcall_with_output('gdalwarp %s %s -of ENVI -overwrite %s'
%(inPath, warpedPath, ' '.join(new_options)))
% (inPath, warpedPath, ' '.join(new_options)))
if exitcode:
raise Exception(err)
......@@ -189,24 +185,24 @@ def Warp(destNameOrDestDS, srcDSOrSrcDSTab, options = '', format = 'GTiff',
mem_ds = drv.CreateCopy(warpedPath, ds)
# cleanup
ds = None
[gdal.Unlink(p) for p in [inPath, os.path.splitext(inPath) [0] + '.hdr']]
del ds
[gdal.Unlink(p) for p in [inPath, os.path.splitext(inPath)[0] + '.hdr']]
[gdal.Unlink(p) for p in [warpedPath, os.path.splitext(warpedPath)[0] + '.hdr']]
return mem_ds
def Translate(destNameOrDestDS, srcDSOrSrcDSTab, options = '', format = 'GTiff',
outputType = gdal.GDT_Unknown, bandList = None, maskBand = None,
width = 0, height = 0, widthPct = 0.0, heightPct = 0.0,
xRes = 0.0, yRes = 0.0,
creationOptions = None, srcWin = None, projWin = None, projWinSRS = None, strict = False,
unscale = False, scaleParams = None, exponents = None,
outputBounds = None, metadataOptions = None,
outputSRS = None, GCPs = None,
noData = None, rgbExpand = None,
stats = False, rat = True, resampleAlg = None,
callback = None, callback_data = None):
def Translate(destNameOrDestDS, srcDSOrSrcDSTab, options='', format='GTiff',
outputType=gdal.GDT_Unknown, bandList=None, maskBand=None,
width=0, height=0, widthPct=0.0, heightPct=0.0,
xRes=0.0, yRes=0.0,
creationOptions=None, srcWin=None, projWin=None, projWinSRS=None, strict=False,
unscale=False, scaleParams=None, exponents=None,
outputBounds=None, metadataOptions=None,
outputSRS=None, GCPs=None,
noData=None, rgbExpand=None,
stats=False, rat=True, resampleAlg=None,
callback=None, callback_data=None):
""" This functions brings functionality of gdal.Translate() that is not available in GDAL versions below 2.1 (?).
Keyword arguments are :
......@@ -227,7 +223,8 @@ def Translate(destNameOrDestDS, srcDSOrSrcDSTab, options = '', format = 'GTiff',
projWinSRS --- SRS in which projWin is expressed
strict --- strict mode
unscale --- unscale values with scale and offset metadata
scaleParams --- list of scale parameters, each of the form [src_min,src_max] or [src_min,src_max,dst_min,dst_max]
scaleParams --- list of scale parameters, each of the form
[src_min,src_max] or [src_min,src_max,dst_min,dst_max]
exponents --- list of exponentiation parameters
outputBounds --- assigned output bounds: [ulx, uly, lrx, lry]
metadataOptions --- list of metadata options
......@@ -245,19 +242,19 @@ def Translate(destNameOrDestDS, srcDSOrSrcDSTab, options = '', format = 'GTiff',
new_options = options
new_options += ['-of', format]
if outputType != gdal.GDT_Unknown:
new_options += ['-ot', gdal.GetDataTypeName(outputType) ]
new_options += ['-ot', gdal.GetDataTypeName(outputType)]
if maskBand is not None:
new_options += ['-mask', str(maskBand) ]
new_options += ['-mask', str(maskBand)]
if bandList is not None:
for b in bandList:
new_options += ['-b', str(b) ]
new_options += ['-b', str(b)]
if width != 0 or height != 0:
new_options += ['-outsize', str(width), str(height)]
elif widthPct != 0 and heightPct != 0:
new_options += ['-outsize', str(widthPct) + '%%', str(heightPct) + '%%']
if creationOptions is not None:
for opt in creationOptions:
new_options += ['-co', opt ]
new_options += ['-co', opt]
if srcWin is not None:
new_options += ['-srcwin', str(srcWin[0]), str(srcWin[1]), str(srcWin[2]), str(srcWin[3])]
if strict:
......@@ -268,28 +265,29 @@ def Translate(destNameOrDestDS, srcDSOrSrcDSTab, options = '', format = 'GTiff',
for scaleParam in scaleParams:
new_options += ['-scale']
for v in scaleParam:
new_options += [ str(v) ]
new_options += [str(v)]
if exponents:
for exponent in exponents:
new_options += ['-exponent', str(exponent)]
if outputBounds is not None:
new_options += ['-a_ullr', str(outputBounds[0]), str(outputBounds[1]), str(outputBounds[2]), str(outputBounds[3])]
new_options += ['-a_ullr', str(outputBounds[0]), str(outputBounds[1]), str(outputBounds[2]),
str(outputBounds[3])]
if metadataOptions is not None:
for opt in metadataOptions:
new_options += ['-mo', opt ]
new_options += ['-mo', opt]
if outputSRS is not None:
new_options += ['-a_srs', str(outputSRS) ]
new_options += ['-a_srs', str(outputSRS)]
if GCPs is not None:
for gcp in GCPs:
new_options += ['-gcp', str(gcp.GCPPixel), str(gcp.GCPLine), str(gcp.GCPX), str(gcp.GCPY), str(gcp.GCPZ) ]
new_options += ['-gcp', str(gcp.GCPPixel), str(gcp.GCPLine), str(gcp.GCPX), str(gcp.GCPY), str(gcp.GCPZ)]
if projWin is not None:
new_options += ['-projwin', str(projWin[0]), str(projWin[1]), str(projWin[2]), str(projWin[3])]
if projWinSRS is not None:
new_options += ['-projwin_srs', str(projWinSRS) ]
new_options += ['-projwin_srs', str(projWinSRS)]
if noData is not None:
new_options += ['-a_nodata', str(noData) ]
new_options += ['-a_nodata', str(noData)]
if rgbExpand is not None:
new_options += ['-expand', str(rgbExpand) ]
new_options += ['-expand', str(rgbExpand)]
if stats:
new_options += ['-stats']
if not rat:
......@@ -310,23 +308,23 @@ def Translate(destNameOrDestDS, srcDSOrSrcDSTab, options = '', format = 'GTiff',
elif resampleAlg == gdal.GRA_Mode:
new_options += ['-r', 'mode']
else:
new_options += ['-r', str(resampleAlg) ]
new_options += ['-r', str(resampleAlg)]
if xRes != 0 and yRes != 0:
new_options += ['-tr', str(xRes), str(yRes) ]
new_options += ['-tr', str(xRes), str(yRes)]
if isinstance(srcDSOrSrcDSTab, gdal.Dataset):
drv = gdal.GetDriverByName('ENVI')
inPath = get_tempfile(prefix='translate_in_', ext='.bsq')
drv.CreateCopy(inPath, srcDSOrSrcDSTab)
srcDSOrSrcDSTab = None
elif isinstance(srcDSOrSrcDSTab,str):
del srcDSOrSrcDSTab
elif isinstance(srcDSOrSrcDSTab, str):
inPath = srcDSOrSrcDSTab
else:
raise ValueError
translatedPath = get_tempfile(prefix='translate_out_', ext='.bsq')
out, exitcode, err = subcall_with_output('gdal_translate %s %s -of ENVI %s'
%(inPath, translatedPath, ' '.join(new_options)))
% (inPath, translatedPath, ' '.join(new_options)))
if exitcode:
raise Exception(err)
......@@ -338,8 +336,8 @@ def Translate(destNameOrDestDS, srcDSOrSrcDSTab, options = '', format = 'GTiff',
mem_ds = drv.CreateCopy(translatedPath, ds)
# cleanup
ds = None
[gdal.Unlink(p) for p in [inPath, os.path.splitext(inPath) [0] + '.hdr']]
del ds
[gdal.Unlink(p) for p in [inPath, os.path.splitext(inPath)[0] + '.hdr']]
[gdal.Unlink(p) for p in [translatedPath, os.path.splitext(translatedPath)[0] + '.hdr']]
return mem_ds
......@@ -351,9 +349,9 @@ def ensure_GDAL_version_compatibility(func2run, funcName2ensure):
getattr(gdal, funcName2ensure)
except AttributeError:
if funcName2ensure in globals():
setattr(gdal,funcName2ensure, globals()[funcName2ensure])
setattr(gdal, funcName2ensure, globals()[funcName2ensure])
else:
raise AttributeError("'gdal' has no attribute '%s'." %funcName2ensure)
raise AttributeError("'gdal' has no attribute '%s'." % funcName2ensure)
@functools.wraps(func2run)
def inner(*args, **kwargs):
......@@ -362,7 +360,7 @@ def ensure_GDAL_version_compatibility(func2run, funcName2ensure):
return inner
def ensure_GDAL_version_compatibility(funcName2ensure):
def ensure_GDAL_version_compatibility_deco(funcName2ensure):
try:
getattr(gdal, funcName2ensure)
funcIsMissing = False
......@@ -371,29 +369,30 @@ def ensure_GDAL_version_compatibility(funcName2ensure):
print('drin1')
if funcName2ensure in globals():
print('drin2')
setattr(gdal,funcName2ensure, globals()[funcName2ensure])
setattr(gdal, funcName2ensure, globals()[funcName2ensure])
print('pre', globals()[funcName2ensure])
gdal.Warp = globals()[funcName2ensure]
from gdal import Warp
print('pre2', Warp)
#print(importlib.import_module('gdal.%s' % funcName2ensure))
# print(importlib.import_module('gdal.%s' % funcName2ensure))
else:
raise AttributeError("'gdal' has no attribute '%s'." %funcName2ensure)
raise AttributeError("'gdal' has no attribute '%s'." % funcName2ensure)
def real_decorator(function):
def wrapper(*args, **kwargs):
return function(*args, **kwargs)
return wrapper
if funcIsMissing:
delattr(gdal,funcName2ensure)
delattr(gdal, funcName2ensure)
return real_decorator
def get_gdal_func(funcName):
try:
return getattr(gdal,funcName)
return getattr(gdal, funcName)
except AttributeError:
if funcName in globals():
return globals()[funcName]
......
......@@ -4,11 +4,9 @@ __author__ = "Daniel Scheffler"
try:
from osgeo import gdal
from osgeo import gdalnumeric
from osgeo import gdalconst
except ImportError:
import gdal
import gdalnumeric # FIXME this will import this __module__
import gdalconst
def OpenNumPyArray(array):
......@@ -18,21 +16,21 @@ def OpenNumPyArray(array):
:param array: <numpy.ndarray> in the shape (bands, rows, columns)
:return:
"""
if array.ndim==2:
if array.ndim == 2:
rows, cols = array.shape
bands = 1
elif array.ndim==3:
bands,rows,cols=array.shape
bands = 1
elif array.ndim == 3:
bands, rows, cols = array.shape
else:
raise ValueError('OpenNumPyArray() currently only supports 2D and 3D arrays. Given array shape is %s.'
%str(array.shape))
% str(array.shape))
# get output datatype
gdal_dtype = gdalnumeric.NumericTypeCodeToGDALTypeCode(array.dtype) # FIXME not all datatypes can be translated
assert gdal_dtype is not None, 'Datatype %s is currently not supported by OpenNumPyArray().' %array.dtype
gdal_dtype = gdalnumeric.NumericTypeCodeToGDALTypeCode(array.dtype) # FIXME not all datatypes can be translated
assert gdal_dtype is not None, 'Datatype %s is currently not supported by OpenNumPyArray().' % array.dtype
mem_drv = gdal.GetDriverByName('MEM')
mem_ds = mem_drv.Create('/vsimem/tmp/memfile.mem', cols, rows, bands, gdal_dtype)
mem_ds = mem_drv.Create('/vsimem/tmp/memfile.mem', cols, rows, bands, gdal_dtype)
if mem_ds is None:
raise Exception(gdal.GetLastErrorMsg())
......@@ -40,9 +38,9 @@ def OpenNumPyArray(array):
for bandNr in range(bands):
band = mem_ds.GetRasterBand(bandNr + 1)
band.WriteArray(array[:, :, bandNr] if bands > 1 else array)
band = None
del band
mem_ds.FlushCache() # Write to disk.
mem_ds.FlushCache() # Write to disk.
return mem_ds
......@@ -53,4 +51,4 @@ def get_gdalnumeric_func(funcName):
if funcName in globals():
return globals()[funcName]
else:
raise AttributeError("'gdalnumeric' has no attribute '%s'." % funcName)
\ No newline at end of file
raise AttributeError("'gdalnumeric' has no attribute '%s'." % funcName)
__author__='Daniel Scheffler'
# -*- coding: utf-8 -*-
from . import exceptions
__all__=['exceptions']
\ No newline at end of file
__author__ = 'Daniel Scheffler'
__all__ = ['exceptions']
......@@ -2,11 +2,9 @@
__author__ = "Daniel Scheffler"
class TimeoutError(OSError):
""" Timeout expired. """
pass
class FileNotFoundError(OSError):
""" File not found. """
pass
# -*- coding: utf-8 -*-
__author__ = "Daniel Scheffler"
from . import object_oriented
__all__=['object_oriented']
\ No newline at end of file
__author__ = "Daniel Scheffler"
__all__ = ['object_oriented']
# -*- coding: utf-8 -*-
__author__='Daniel Scheffler'
__author__ = 'Daniel Scheffler'
def alias_property(key):
return property(
lambda self: getattr(self, key),
lambda self: getattr(self, key),
lambda self, val: setattr(self, key, val),
lambda self: delattr(self, key))
\ No newline at end of file
lambda self: delattr(self, key))
......@@ -3,4 +3,4 @@ __author__ = "Daniel Scheffler"
from . import conversion
__all__=['conversion']
\ No newline at end of file
__all__ = ['conversion']
# -*- coding: utf-8 -*-
__author__ = "Daniel Scheffler"
import datetime
import numpy as np
try:
from osgeo import gdal
except ImportError:
import gdal