Commit edbe653c authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Replaced GMS_object._numba_array_merger() with GMS_object._merge_arrays which...


Replaced GMS_object._numba_array_merger() with GMS_object._merge_arrays which does not use numba (and is much faster). Removed numba dependency. Updated HISTORY.rst.
Signed-off-by: Daniel Scheffler's avatarDaniel Scheffler <danschef@gfz-potsdam.de>
parent 13454d41
Pipeline #12596 failed with stage
in 7 minutes and 22 seconds
...@@ -2,6 +2,22 @@ ...@@ -2,6 +2,22 @@
History History
======= =======
0.18.4 (coming soon)
--------------------
* Replaced calls of sorted_nicely with new dependency 'natsort'. Fixed circular imports.
* Fixed AssertionError caused by new version of pyproj.
* Fixed UnicodeEncodeError while writing ENVI headers.
* Moved scipy imports from module level to function level to avoid 'static TLS' ImportError.
* Fixed gdal_warp call within DEM_Creator class.
* Fixed issues while overwriting projections.
* Fixed logging issue.
* Pinned Python version to 3.7+.
* Fixed DeprecationsWarnings.
* Replaced GMS_object._numba_array_merger() with GMS_object._merge_arrays which does not use numba (and is much faster).
* Removed numba dependency.
0.18.3 (2020-04-06) 0.18.3 (2020-04-06)
------------------- -------------------
......
...@@ -209,7 +209,7 @@ The development of the gms_preprocessing package was funded by the German Federa ...@@ -209,7 +209,7 @@ The development of the gms_preprocessing package was funded by the German Federa
The package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. The package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template.
Landsat-5/7/8 satellite data and SRTM/ASTER digital elevation models have been have been provided by the US Geological Landsat-5/7/8 satellite data and SRTM/ASTER digital elevation models have been provided by the US Geological
Survey. Sentinel-2 data have been provided by ESA. Survey. Sentinel-2 data have been provided by ESA.
.. _Cookiecutter: https://github.com/audreyr/cookiecutter .. _Cookiecutter: https://github.com/audreyr/cookiecutter
......
...@@ -48,7 +48,6 @@ try: ...@@ -48,7 +48,6 @@ try:
from osgeo import ogr from osgeo import ogr
except ImportError: except ImportError:
import ogr import ogr
from numba import jit
from multiprocessing import sharedctypes from multiprocessing import sharedctypes
from matplotlib import pyplot as plt from matplotlib import pyplot as plt
from subprocess import Popen, PIPE from subprocess import Popen, PIPE
...@@ -187,26 +186,6 @@ def convert_absPathArchive_to_GDALvsiPath(path_archive): ...@@ -187,26 +186,6 @@ def convert_absPathArchive_to_GDALvsiPath(path_archive):
return os.path.join(gdal_prefix_dict[file_suffix], os.path.basename(path_archive)) return os.path.join(gdal_prefix_dict[file_suffix], os.path.basename(path_archive))
@jit
def _numba_array_merger(GMS_obj, list_arraynames, list_GMS_tiles):
"""private function, e.g. called by merge_GMS_tiles_to_GMS_obj() in order to fasten array merging"""
for arrname in list_arraynames:
samplearray = getattr(list_GMS_tiles[0], arrname)
assert isinstance(samplearray, np.ndarray), \
'Received a %s object for attribute %s. Expected a numpy array.' % (type(samplearray), arrname)
is_3d = len(samplearray.shape) == 3
bands = (samplearray.shape[2],) if is_3d else () # dynamic -> works for arr, cld_arr,...
target_shape = tuple(GMS_obj.shape_fullArr[:2]) + bands
target_dtype = samplearray.dtype
setattr(GMS_obj, arrname, np.empty(target_shape, dtype=target_dtype))
for idx, tile in enumerate(list_GMS_tiles):
rowStart, rowEnd = tile.arr_pos[0]
colStart, colEnd = tile.arr_pos[1]
getattr(GMS_obj, arrname)[rowStart:rowEnd + 1, colStart:colEnd + 1] = getattr(tile, arrname)
return GMS_obj
class mp_SharedNdarray(object): class mp_SharedNdarray(object):
""" """
wrapper class, which collect all neccessary instances to make a numpy ndarray wrapper class, which collect all neccessary instances to make a numpy ndarray
......
...@@ -45,7 +45,6 @@ from pkg_resources import parse_version ...@@ -45,7 +45,6 @@ from pkg_resources import parse_version
import numpy as np import numpy as np
import spectral import spectral
from spectral.io import envi from spectral.io import envi
from numba import jit
from pandas import DataFrame, read_csv from pandas import DataFrame, read_csv
from nested_dict import nested_dict from nested_dict import nested_dict
...@@ -1243,7 +1242,7 @@ class GMS_object(object): ...@@ -1243,7 +1242,7 @@ class GMS_object(object):
bands = (samplearray.shape[2],) if is_3d else () # dynamic -> works for arr, cld_arr,... bands = (samplearray.shape[2],) if is_3d else () # dynamic -> works for arr, cld_arr,...
target_shape = tuple(GMS_obj.shape_fullArr[:2]) + bands target_shape = tuple(GMS_obj.shape_fullArr[:2]) + bands
target_dtype = samplearray.dtype target_dtype = samplearray.dtype
merged_array = GMS_obj._numba_array_merger(list_GMS_tiles, arrname, target_shape, target_dtype) merged_array = GMS_obj._merge_arrays(list_GMS_tiles, arrname, target_shape, target_dtype)
setattr(GMS_obj, arrname if not arrname.startswith('_') else arrname[1:], setattr(GMS_obj, arrname if not arrname.startswith('_') else arrname[1:],
merged_array) # use setters if possible merged_array) # use setters if possible
...@@ -1421,11 +1420,9 @@ class GMS_object(object): ...@@ -1421,11 +1420,9 @@ class GMS_object(object):
return sub_GMS_obj return sub_GMS_obj
@staticmethod @staticmethod
@jit def _merge_arrays(list_GMS_tiles, arrname2merge, target_shape, target_dtype):
def _numba_array_merger(list_GMS_tiles, arrname2merge, target_shape, target_dtype):
# type: (list, str, tuple, np.dtype) -> np.ndarray # type: (list, str, tuple, np.dtype) -> np.ndarray
""" """Merge multiple arrays into a single one.
private function, e.g. called by merge_GMS_tiles_to_GMS_obj() in order to fasten array merging
:param list_GMS_tiles: :param list_GMS_tiles:
:param arrname2merge: :param arrname2merge:
...@@ -1433,11 +1430,9 @@ class GMS_object(object): ...@@ -1433,11 +1430,9 @@ class GMS_object(object):
:param target_dtype: :param target_dtype:
:return: :return:
""" """
out_arr = np.empty(target_shape, dtype=target_dtype) out_arr = np.empty(target_shape, dtype=target_dtype)
for idx, tile in enumerate(list_GMS_tiles): for tile in list_GMS_tiles:
rowStart, rowEnd = tile.arr_pos[0] (rowStart, rowEnd), (colStart, colEnd) = tile.arr_pos
colStart, colEnd = tile.arr_pos[1]
out_arr[rowStart:rowEnd + 1, colStart:colEnd + 1] = getattr(tile, arrname2merge) out_arr[rowStart:rowEnd + 1, colStart:colEnd + 1] = getattr(tile, arrname2merge)
return out_arr return out_arr
......
...@@ -16,7 +16,6 @@ pyorbital ...@@ -16,7 +16,6 @@ pyorbital
dill dill
pytz pytz
pandas pandas
numba
spectral spectral
# pyhdf # not pip installable # pyhdf # not pip installable
geopandas geopandas
......
...@@ -3,7 +3,6 @@ ephem ...@@ -3,7 +3,6 @@ ephem
pyorbital pyorbital
dill dill
pytz pytz
numba
spectral spectral
geopandas geopandas
iso8601 iso8601
......
...@@ -41,10 +41,9 @@ with open("gms_preprocessing/version.py") as version_file: ...@@ -41,10 +41,9 @@ with open("gms_preprocessing/version.py") as version_file:
requirements = [ requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz', 'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz',
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy', 'pandas', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy', 'natsort',
'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.8.0', 'arosics>=0.8.7', 'six', 'tqdm', 'jsmin', 'cerberus', 'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.8.0', 'arosics>=0.8.7', 'six', 'tqdm', 'jsmin', 'cerberus',
'nested_dict', 'openpyxl', 'timeout_decorator', 'redis', 'retools', 'redis-semaphore', 'psutil', 'pyrsr>=0.3.1', 'nested_dict', 'openpyxl', 'timeout_decorator', 'redis', 'retools', 'redis-semaphore', 'psutil', 'pyrsr>=0.3.1',
'natsort',
'spechomo @ git+https://gitext.gfz-potsdam.de/geomultisens/spechomo#egg=spechomo' 'spechomo @ git+https://gitext.gfz-potsdam.de/geomultisens/spechomo#egg=spechomo'
# spectral<0.16 has some problems with writing signed integer 8bit data # spectral<0.16 has some problems with writing signed integer 8bit data
# fmask # conda install -c conda-forge python-fmask # fmask # conda install -c conda-forge python-fmask
......
...@@ -60,7 +60,6 @@ dependencies: ...@@ -60,7 +60,6 @@ dependencies:
- pyorbital - pyorbital
- dill - dill
- pytz - pytz
- numba
- pyinstrument - pyinstrument
- geoalchemy2 - geoalchemy2
- sqlalchemy - sqlalchemy
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment