Commit edbe653c authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Replaced GMS_object._numba_array_merger() with GMS_object._merge_arrays which...


Replaced GMS_object._numba_array_merger() with GMS_object._merge_arrays which does not use numba (and is much faster). Removed numba dependency. Updated HISTORY.rst.
Signed-off-by: Daniel Scheffler's avatarDaniel Scheffler <danschef@gfz-potsdam.de>
parent 13454d41
Pipeline #12596 failed with stage
in 7 minutes and 22 seconds
......@@ -2,6 +2,22 @@
History
=======
0.18.4 (coming soon)
--------------------
* Replaced calls of sorted_nicely with new dependency 'natsort'. Fixed circular imports.
* Fixed AssertionError caused by new version of pyproj.
* Fixed UnicodeEncodeError while writing ENVI headers.
* Moved scipy imports from module level to function level to avoid 'static TLS' ImportError.
* Fixed gdal_warp call within DEM_Creator class.
* Fixed issues while overwriting projections.
* Fixed logging issue.
* Pinned Python version to 3.7+.
* Fixed DeprecationsWarnings.
* Replaced GMS_object._numba_array_merger() with GMS_object._merge_arrays which does not use numba (and is much faster).
* Removed numba dependency.
0.18.3 (2020-04-06)
-------------------
......
......@@ -209,7 +209,7 @@ The development of the gms_preprocessing package was funded by the German Federa
The package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template.
Landsat-5/7/8 satellite data and SRTM/ASTER digital elevation models have been have been provided by the US Geological
Landsat-5/7/8 satellite data and SRTM/ASTER digital elevation models have been provided by the US Geological
Survey. Sentinel-2 data have been provided by ESA.
.. _Cookiecutter: https://github.com/audreyr/cookiecutter
......
......@@ -48,7 +48,6 @@ try:
from osgeo import ogr
except ImportError:
import ogr
from numba import jit
from multiprocessing import sharedctypes
from matplotlib import pyplot as plt
from subprocess import Popen, PIPE
......@@ -187,26 +186,6 @@ def convert_absPathArchive_to_GDALvsiPath(path_archive):
return os.path.join(gdal_prefix_dict[file_suffix], os.path.basename(path_archive))
@jit
def _numba_array_merger(GMS_obj, list_arraynames, list_GMS_tiles):
"""private function, e.g. called by merge_GMS_tiles_to_GMS_obj() in order to fasten array merging"""
for arrname in list_arraynames:
samplearray = getattr(list_GMS_tiles[0], arrname)
assert isinstance(samplearray, np.ndarray), \
'Received a %s object for attribute %s. Expected a numpy array.' % (type(samplearray), arrname)
is_3d = len(samplearray.shape) == 3
bands = (samplearray.shape[2],) if is_3d else () # dynamic -> works for arr, cld_arr,...
target_shape = tuple(GMS_obj.shape_fullArr[:2]) + bands
target_dtype = samplearray.dtype
setattr(GMS_obj, arrname, np.empty(target_shape, dtype=target_dtype))
for idx, tile in enumerate(list_GMS_tiles):
rowStart, rowEnd = tile.arr_pos[0]
colStart, colEnd = tile.arr_pos[1]
getattr(GMS_obj, arrname)[rowStart:rowEnd + 1, colStart:colEnd + 1] = getattr(tile, arrname)
return GMS_obj
class mp_SharedNdarray(object):
"""
wrapper class, which collect all neccessary instances to make a numpy ndarray
......
......@@ -45,7 +45,6 @@ from pkg_resources import parse_version
import numpy as np
import spectral
from spectral.io import envi
from numba import jit
from pandas import DataFrame, read_csv
from nested_dict import nested_dict
......@@ -1243,7 +1242,7 @@ class GMS_object(object):
bands = (samplearray.shape[2],) if is_3d else () # dynamic -> works for arr, cld_arr,...
target_shape = tuple(GMS_obj.shape_fullArr[:2]) + bands
target_dtype = samplearray.dtype
merged_array = GMS_obj._numba_array_merger(list_GMS_tiles, arrname, target_shape, target_dtype)
merged_array = GMS_obj._merge_arrays(list_GMS_tiles, arrname, target_shape, target_dtype)
setattr(GMS_obj, arrname if not arrname.startswith('_') else arrname[1:],
merged_array) # use setters if possible
......@@ -1421,11 +1420,9 @@ class GMS_object(object):
return sub_GMS_obj
@staticmethod
@jit
def _numba_array_merger(list_GMS_tiles, arrname2merge, target_shape, target_dtype):
def _merge_arrays(list_GMS_tiles, arrname2merge, target_shape, target_dtype):
# type: (list, str, tuple, np.dtype) -> np.ndarray
"""
private function, e.g. called by merge_GMS_tiles_to_GMS_obj() in order to fasten array merging
"""Merge multiple arrays into a single one.
:param list_GMS_tiles:
:param arrname2merge:
......@@ -1433,11 +1430,9 @@ class GMS_object(object):
:param target_dtype:
:return:
"""
out_arr = np.empty(target_shape, dtype=target_dtype)
for idx, tile in enumerate(list_GMS_tiles):
rowStart, rowEnd = tile.arr_pos[0]
colStart, colEnd = tile.arr_pos[1]
for tile in list_GMS_tiles:
(rowStart, rowEnd), (colStart, colEnd) = tile.arr_pos
out_arr[rowStart:rowEnd + 1, colStart:colEnd + 1] = getattr(tile, arrname2merge)
return out_arr
......
......@@ -16,7 +16,6 @@ pyorbital
dill
pytz
pandas
numba
spectral
# pyhdf # not pip installable
geopandas
......
......@@ -3,7 +3,6 @@ ephem
pyorbital
dill
pytz
numba
spectral
geopandas
iso8601
......
......@@ -41,10 +41,9 @@ with open("gms_preprocessing/version.py") as version_file:
requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz',
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
'pandas', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy', 'natsort',
'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.8.0', 'arosics>=0.8.7', 'six', 'tqdm', 'jsmin', 'cerberus',
'nested_dict', 'openpyxl', 'timeout_decorator', 'redis', 'retools', 'redis-semaphore', 'psutil', 'pyrsr>=0.3.1',
'natsort',
'spechomo @ git+https://gitext.gfz-potsdam.de/geomultisens/spechomo#egg=spechomo'
# spectral<0.16 has some problems with writing signed integer 8bit data
# fmask # conda install -c conda-forge python-fmask
......
......@@ -60,7 +60,6 @@ dependencies:
- pyorbital
- dill
- pytz
- numba
- pyinstrument
- geoalchemy2
- sqlalchemy
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment