Commit 51d8c6c2 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Replaced calls of sorted_nicely with new dependency 'natsort'. Fixed circular...


Replaced calls of sorted_nicely with new dependency 'natsort'. Fixed circular imports. Fixed AssertionError caused by new version of pyproj.
Signed-off-by: Daniel Scheffler's avatarDaniel Scheffler <danschef@gfz-potsdam.de>
parent d48b5e78
Pipeline #12555 failed with stage
in 5 minutes and 28 seconds
......@@ -33,6 +33,8 @@ test_gms_preprocessing:
# make tests
- pip install -U sicor
- pip install natsort # TODO remove as soon as docker container is rebuilt
# run tests
- make nosetests
- make docs
......
......@@ -32,7 +32,7 @@ import os
import matplotlib
matplotlib.use('Agg', warn=False) # switch matplotlib backend to 'Agg' and disable warning in case its already 'Agg'
matplotlib.use('Agg') # switch matplotlib backend to 'Agg' and disable warning in case its already 'Agg'
from gms_preprocessing import ProcessController, __version__ # noqa: E402
from gms_preprocessing.misc.database_tools import GMS_JOB # noqa: E402
......
......@@ -32,6 +32,7 @@ import warnings
import gdal
import gdalnumeric
import numpy as np
from natsort import natsorted
try:
from pyhdf import SD
......@@ -46,8 +47,6 @@ from py_tools_ds.geo.projection import EPSG2WKT, isProjectedOrGeographic
from ..options.config import GMS_config as CFG
from . import geoprocessing as GEOP
from ..io import output_writer as OUT_W
from ..misc import helper_functions as HLP_F
from ..misc.definition_dicts import get_outFillZeroSaturated, is_dataset_provided_as_fullScene
from ..misc.locks import IOLock
from ..model.gms_object import GMS_object
......@@ -115,6 +114,7 @@ class L1A_object(GMS_object):
self.shape_fullArr = self.arr.shape
def archive_to_rasObj(self, path_archive, path_output=None, subset=None):
from ..misc.helper_functions import convert_absPathArchive_to_GDALvsiPath
assert subset is None or isinstance(subset, list) and len(subset) == 2, \
"Subset argument has be a list with 2 elements."
......@@ -124,7 +124,7 @@ class L1A_object(GMS_object):
"subsetting." % subset[0]
self.logger.info('Reading %s %s %s image data...' % (self.satellite, self.sensor, self.subsystem))
gdal_path_archive = HLP_F.convert_absPathArchive_to_GDALvsiPath(path_archive)
gdal_path_archive = convert_absPathArchive_to_GDALvsiPath(path_archive)
project_dir = os.path.abspath(os.curdir)
os.chdir(os.path.dirname(path_archive))
files_in_archive = gdal.ReadDirRecursive(gdal_path_archive) # needs ~12sek for Landsat-8
......@@ -142,7 +142,7 @@ class L1A_object(GMS_object):
re.search(r'Sentinel-2', self.satellite)
n_files2search = len(full_LayerBandsAssignment) if is_ALOS_Landsat_S2 else 1
for File in HLP_F.sorted_nicely(files_in_archive):
for File in natsorted(files_in_archive):
search_res = \
re.search(r"IMG-0[0-9]-[\s\S]*", File) if re.search(r'ALOS', self.satellite) else \
re.search(r"[\S]*_B[1-9][0-9]?[\S]*.TIF", File) if re.search(r'Landsat', self.satellite) else \
......@@ -182,7 +182,7 @@ class L1A_object(GMS_object):
# NOTE: image_files is a SORTED list of image filenames; self.LayerBandsAssignment may be sorted by CWL
filtered_files = []
for bN in self.LayerBandsAssignment: # unsorted, e.g., ['1', '2', '3', '4', '5', '9', '6', '7']
for fN, b in zip(image_files, HLP_F.sorted_nicely(full_LayerBandsAssignment)): # both sorted nicely
for fN, b in zip(image_files, natsorted(full_LayerBandsAssignment)): # both sorted nicely
if b == bN:
filtered_files.append(fN)
......@@ -492,6 +492,8 @@ class L1A_object(GMS_object):
Method is skipped if self.GeoAlign_ok and self.GeoTransProj_ok evaluate to 'True'. All attributes connected
with the georeference of self.arr are automatically updated."""
from ..io.output_writer import add_attributes_to_ENVIhdr
if False in [self.GeoAlign_ok, self.GeoTransProj_ok]:
previous_dataname = self.MetaObj.Dataname
if hasattr(self, 'arr') and isinstance(self.arr, (GeoArray, np.ndarray)) and \
......@@ -516,7 +518,7 @@ class L1A_object(GMS_object):
gResolution=self.MetaObj.gResolution,
shape_fullArr=self.shape_fullArr)
self.add_rasterInfo_to_MetaObj(rasObj)
OUT_W.add_attributes_to_ENVIhdr(
add_attributes_to_ENVIhdr(
{'map info': self.MetaObj.map_info, 'coordinate system string': self.MetaObj.projection},
os.path.splitext(self.MetaObj.Dataname)[0] + '.hdr')
self.arr = self.MetaObj.Dataname
......@@ -664,7 +666,9 @@ class L1A_object(GMS_object):
self.MetaObj.bands = 1 if len(self.arr.shape) == 2 else self.arr.shape[2]
self.arr.gt = mapinfo2geotransform(self.MetaObj.map_info)
if not self.arr.prj:
self.arr.prj = self.MetaObj.projection
# must be set here because nodata mask has been computed from self.arr without geoinfos:
self.mask_nodata.gt = self.arr.gt
self.mask_nodata.prj = self.arr.prj
......
......@@ -32,15 +32,17 @@ import sys
import traceback
import warnings
from logging import getLogger
from typing import Union, List # noqa F401 # flake8 issue
from typing import Union, List, TYPE_CHECKING # noqa F401 # flake8 issue
from ..model.gms_object import GMS_object # noqa F401 # flake8 issue
from ..model.gms_object import failed_GMS_object
from ..options.config import GMS_config as CFG
from ..misc import database_tools as DB_T
from ..misc.helper_functions import is_proc_level_lower
from .definition_dicts import db_jobs_statistics_def, proc_chain
if TYPE_CHECKING:
from ..model.gms_object import GMS_object # noqa F401 # flake8 issue
from ..model.gms_object import failed_GMS_object
__author__ = 'Daniel Scheffler'
......@@ -163,6 +165,7 @@ class ExceptionHandler(object):
@staticmethod
def is_failed(GMS_objs):
from ..model.gms_object import failed_GMS_object
return isinstance(GMS_objs, failed_GMS_object) or \
(isinstance(GMS_objs, list) and isinstance(GMS_objs[0], failed_GMS_object))
......@@ -244,6 +247,8 @@ class ExceptionHandler(object):
timeout=30000)
def handle_failed(self):
from ..model.gms_object import failed_GMS_object
try:
_, exc_val, exc_tb = self.exc_details
......@@ -281,6 +286,8 @@ class ExceptionHandler(object):
# been updated by the first subsystem (that earlier reached L1A)
# FIXME proc_status_all_GMSobjs is not available if other subsystems are processed by another
# FIXME multiprocessing worker or on another machine (cluster node)
from ..model.gms_object import GMS_object
procstatus_other_ss = {k: v for k, v in GMS_object.proc_status_all_GMSobjs[failed_Obj.scene_ID].items()
if k != failed_Obj.subsystem}
for ss, statusentry in procstatus_other_ss.items():
......
......@@ -56,12 +56,6 @@ from xml.etree.ElementTree import QName
from ..options.config import GMS_config as CFG
from . import database_tools as DB_T
from ..algorithms.L1A_P import L1A_object
from ..algorithms.L1B_P import L1B_object
from ..algorithms.L1C_P import L1C_object
from ..algorithms.L2A_P import L2A_object
from ..algorithms.L2B_P import L2B_object
from ..algorithms.L2C_P import L2C_object
from ..misc.definition_dicts import proc_chain
from py_tools_ds.geo.coord_trafo import mapXY2imXY, reproject_shapelyGeometry
......@@ -69,8 +63,23 @@ from py_tools_ds.geo.coord_calc import corner_coord_to_minmax
__author__ = 'Daniel Scheffler'
parentObjDict = {'L1A': L1A_object, 'L1B': L1B_object, 'L1C': L1C_object,
'L2A': L2A_object, 'L2B': L2B_object, 'L2C': L2C_object}
def get_parentObjDict():
from ..algorithms.L1A_P import L1A_object
from ..algorithms.L1B_P import L1B_object
from ..algorithms.L1C_P import L1C_object
from ..algorithms.L2A_P import L2A_object
from ..algorithms.L2B_P import L2B_object
from ..algorithms.L2C_P import L2C_object
return dict(L1A=L1A_object,
L1B=L1B_object,
L1C=L1C_object,
L2A=L2A_object,
L2B=L2B_object,
L2C=L2C_object)
initArgsDict = {'L1A': (None,), 'L1B': (None,), 'L1C': (None,),
'L2A': (None,), 'L2B': (None,), 'L2C': (None,)}
......
......@@ -444,7 +444,8 @@ class GMS_object(object):
if hasattr(self, 'MetaObj') and self.MetaObj:
self._arr.nodata = self.MetaObj.spec_vals['fill']
self._arr.gt = mapinfo2geotransform(self.MetaObj.map_info) if self.MetaObj.map_info else [0, 1, 0, 0, 0, -1]
self._arr.prj = self.MetaObj.projection if self.MetaObj.projection else self._arr.projection
if not self._arr.projection:
self._arr.prj = self.MetaObj.projection
else:
self._arr.nodata = DEF_D.get_outFillZeroSaturated(self._arr.dtype)[0]
......@@ -1298,7 +1299,9 @@ class GMS_object(object):
self.logger.info(logmsg)
# copy object
sub_GMS_obj = HLP_F.parentObjDict[self.proc_level](*HLP_F.initArgsDict[self.proc_level]) # init
from ..misc.helper_functions import get_parentObjDict
parentObjDict = get_parentObjDict()
sub_GMS_obj = parentObjDict[self.proc_level](*HLP_F.initArgsDict[self.proc_level]) # init
sub_GMS_obj.__dict__.update(
{k: getattr(self, k) for k in self.__dict__.keys()
if not isinstance(getattr(self, k), (GeoArray, np.ndarray))}.copy())
......
......@@ -44,6 +44,7 @@ import numpy as np
import pyproj
from matplotlib import dates as mdates
from pyorbital import astronomy
from natsort import natsorted
from py_tools_ds.geo.map_info import geotransform2mapinfo
from py_tools_ds.geo.projection import WKT2EPSG
......@@ -333,7 +334,7 @@ class METADATA(object):
# band specific metadata #
##########################
LBA_full_sorted = HLP_F.sorted_nicely(self.LayerBandsAssignment_full)
LBA_full_sorted = natsorted(self.LayerBandsAssignment_full)
# Gains and Offsets
h9 = re.search(r"<Image_Interpretation>[\s\S]*</Image_Interpretation>", dim_, re.I)
......@@ -606,7 +607,7 @@ class METADATA(object):
##########################
# band specific metadata #
##########################
LBA_full_sorted = HLP_F.sorted_nicely(self.LayerBandsAssignment_full)
LBA_full_sorted = natsorted(self.LayerBandsAssignment_full)
# Gains and Offsets
h4 = re.search(r"GROUP = MIN_MAX_RADIANCE[\s\S]*END_GROUP = MIN_MAX_PIXEL_VALUE", mtl_, re.I)
......@@ -967,7 +968,7 @@ class METADATA(object):
# band specific metadata #
##########################
LBA_full_sorted = HLP_F.sorted_nicely(self.LayerBandsAssignment_full)
LBA_full_sorted = natsorted(self.LayerBandsAssignment_full)
# Gains + Offsets
h9 = re.findall(r"<re:radiometricScaleFactor>([^<]*)</re:radiometricScaleFactor>",
......@@ -1300,7 +1301,7 @@ class METADATA(object):
# band specific metadata #
##########################
LBA_full_sorted = HLP_F.sorted_nicely(self.LayerBandsAssignment_full)
LBA_full_sorted = natsorted(self.LayerBandsAssignment_full)
# Gains/Offsets
h4 = re.findall(r"GROUP[\s]*=[\s]*"
......@@ -1482,7 +1483,7 @@ class METADATA(object):
# band specific metadata #
##########################
LBA_full_sorted = HLP_F.sorted_nicely(self.LayerBandsAssignment_full)
LBA_full_sorted = natsorted(self.LayerBandsAssignment_full)
# GainMode with corresponding coefficients + Offsets
gains_AVNIR = {'1': ['N/A', 'N/A', 'N/A', 'N/A'], '2': [0.5880, 0.5730, 0.5020, 0.5570],
......@@ -1548,7 +1549,7 @@ class METADATA(object):
open_specific_file_within_archive(self.FolderOrArchive, '*/*data*/LED-*', read_mode='rb')
# Gains & offsets
LBA_full_sorted = HLP_F.sorted_nicely(self.LayerBandsAssignment_full)
LBA_full_sorted = natsorted(self.LayerBandsAssignment_full)
self.Gains = dict(zip(LBA_full_sorted, [float(lead_[4680 * 3 + 2703:4680 * 3 + 2710]),
float(lead_[4680 * 3 + 2719:4680 * 3 + 2726]),
float(lead_[4680 * 3 + 2735:4680 * 3 + 2742]),
......@@ -1730,7 +1731,7 @@ class METADATA(object):
# band specific metadata #
##########################
LBA_full_sorted = HLP_F.sorted_nicely(self.LayerBandsAssignment_full)
LBA_full_sorted = natsorted(self.LayerBandsAssignment_full)
# ATTENTION Gains are only provided for 12 bands! I don't know why?
Gains = [float(ele.text) for ele in xml_Scene_root.findall(".//PHYSICAL_GAINS")]
......
......@@ -37,6 +37,7 @@ import re
from typing import TYPE_CHECKING
import shutil
import sys
from natsort import natsorted
from ..io import output_writer as OUT_W
from ..io import input_reader as INP_R
......@@ -188,7 +189,7 @@ class ProcessController(object):
self.logger.info('%s: According to logfile no completely processed data exist at any '
'processing level. Dataset has to be reprocessed.' % dataset['entity_ID'])
else:
AllWrittenProcL_dueLog = HLP_F.sorted_nicely(list(set(AllWrittenProcL_dueLog)))
AllWrittenProcL_dueLog = natsorted(list(set(AllWrittenProcL_dueLog)))
return AllWrittenProcL_dueLog
# check if there are not multiple database records for this dataset
......@@ -374,7 +375,9 @@ class ProcessController(object):
# NOTE: DON'T multiprocess that with MAP(GMS_object(*initargs).from_disk, work)
# in case of multiple subsystems GMS_object(*initargs) would always point to the same object in memory
# -> subsystem attribute will be overwritten each time
DB_objs = [HLP_F.parentObjDict[prevLvl].from_disk(tuple_GMS_subset=w) for w in work]
from ..misc.helper_functions import get_parentObjDict
parentObjDict = get_parentObjDict()
DB_objs = [parentObjDict[prevLvl].from_disk(tuple_GMS_subset=w) for w in work]
if DB_objs:
DB_objs = list(chain.from_iterable(DB_objs)) if list in [type(i) for i in DB_objs] else list(DB_objs)
......
......@@ -22,3 +22,4 @@ retools
redis-semaphore
psutil
pyrsr>=0.3.1
natsort
......@@ -44,6 +44,7 @@ requirements = [
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.8.0', 'arosics>=0.8.7', 'six', 'tqdm', 'jsmin', 'cerberus',
'nested_dict', 'openpyxl', 'timeout_decorator', 'redis', 'retools', 'redis-semaphore', 'psutil', 'pyrsr>=0.3.1',
'natsort',
'spechomo @ git+https://gitext.gfz-potsdam.de/geomultisens/spechomo#egg=spechomo'
# spectral<0.16 has some problems with writing signed integer 8bit data
# fmask # conda install -c conda-forge python-fmask
......
......@@ -76,6 +76,7 @@ dependencies:
- redis-semaphore
- psutil
- ecmwf-api-client
- natsort
- py_tools_ds>=0.12.4
- geoarray>=0.8.0
- arosics>=0.8.7
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment