Commit c5515d7c authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Fixed some bugs and added docstrings within L1B_P.Scene_finder(). PEP8 editing for run_gms.py.

parent b702f2b6
Pipeline #1242 failed with stage
in 9 minutes and 10 seconds
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
__author__='Daniel Scheffler' __author__ = 'Daniel Scheffler'
import argparse import argparse
import sys
import warnings import warnings
import matplotlib import matplotlib
matplotlib.use('Agg', warn=False) # switch matplotlib backend to 'Agg' and disable warning in case its already 'Agg' matplotlib.use('Agg', warn=False) # switch matplotlib backend to 'Agg' and disable warning in case its already 'Agg'
from gms_preprocessing import process_controller, __version__ from gms_preprocessing import process_controller, __version__ # noqa: E402
from gms_preprocessing.misc.database_tools import GMS_JOB from gms_preprocessing.misc.database_tools import GMS_JOB # noqa: E402
def run_from_jobid(args): def run_from_jobid(args):
...@@ -21,8 +20,8 @@ def run_from_jobid(args): ...@@ -21,8 +20,8 @@ def run_from_jobid(args):
# set up process controller instance # set up process controller instance
PC = process_controller(args.jobid, parallelization_level='scenes') PC = process_controller(args.jobid, parallelization_level='scenes')
#PC.job.path_procdata_scenes = '/geoms/data/processed_scenes_dev' # PC.job.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
#PC.job.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev' # PC.job.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
# run the job # run the job
PC.run_all_processors() PC.run_all_processors()
...@@ -30,14 +29,14 @@ def run_from_jobid(args): ...@@ -30,14 +29,14 @@ def run_from_jobid(args):
def run_from_sceneids(args): def run_from_sceneids(args):
# create and run a download job # create and run a download job
warnings.warn('Currently the console argument parser expects the given scenes as already downloaded.') # TODO warnings.warn('Currently the console argument parser expects the given scenes as already downloaded.') # TODO
# create a new processing job from scene IDs # create a new processing job from scene IDs
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection) dbJob = GMS_JOB(db_connection)
...@@ -55,11 +54,11 @@ def run_from_entityids(args): ...@@ -55,11 +54,11 @@ def run_from_entityids(args):
:return: :return:
""" """
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection) dbJob = GMS_JOB(db_connection)
...@@ -77,11 +76,11 @@ def run_from_filenames(args): ...@@ -77,11 +76,11 @@ def run_from_filenames(args):
:return: :return:
""" """
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection) dbJob = GMS_JOB(db_connection)
...@@ -92,7 +91,6 @@ def run_from_filenames(args): ...@@ -92,7 +91,6 @@ def run_from_filenames(args):
_run_job(dbJob) _run_job(dbJob)
def run_from_constraints(args): def run_from_constraints(args):
# create a new job from constraints # create a new job from constraints
# TODO # TODO
...@@ -124,50 +122,51 @@ def _run_job(dbJob, parallelization_level='scenes'): ...@@ -124,50 +122,51 @@ def _run_job(dbJob, parallelization_level='scenes'):
def get_gms_argparser(): def get_gms_argparser():
"""Return argument parser for run_gms.py program.""" """Return argument parser for run_gms.py program."""
### CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN # CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog='run_gms.py', prog='run_gms.py',
description='='*70+'\n'+'GeoMultiSens preprocessing console argument parser. ' \ description='=' * 70 + '\n' + 'GeoMultiSens preprocessing console argument parser. '
'Python implementation by Daniel Scheffler (daniel.scheffler@gfz-potsdam.de)', 'Python implementation by Daniel Scheffler (daniel.scheffler@gfz-potsdam.de)',
epilog="The argument parser offers multiple sub-argument parsers (jobid, sceneids, ...) for starting GMS jobs. " epilog="The argument parser offers multiple sub-argument parsers (jobid, sceneids, ...) for starting GMS jobs. "
"use '>>> python /path/to/gms_preprocessing/run_gms.py <sub-parser> -h' for detailed documentation and usage " "use '>>> python /path/to/gms_preprocessing/run_gms.py <sub-parser> -h' for detailed documentation and "
"hints.") "usage hints.")
parser.add_argument('--version', action='version', version=__version__) parser.add_argument('--version', action='version', version=__version__)
subparsers = parser.add_subparsers() subparsers = parser.add_subparsers()
# CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN
parser_jobid = subparsers\
.add_parser('jobid', description='Run a GeoMultiSens preprocessing job using an already existing job ID.',
help="Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser).")
### CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN parser_sceneids = subparsers\
parser_jobid = subparsers.add_parser('jobid', .add_parser('sceneids', description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
description= 'Run a GeoMultiSens preprocessing job using an already existing job ID.', help="Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser).")
help="Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser).")
parser_sceneids = subparsers.add_parser('sceneids', parser_entityids = subparsers\
description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.', .add_parser('entityids', description='Run a GeoMultiSens preprocessing job for a given list of entity IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser).") help="Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser).")
parser_entityids = subparsers.add_parser('entityids', parser_filenames = subparsers\
description='Run a GeoMultiSens preprocessing job for a given list of entity IDs.', .add_parser('filenames', description='Run a GeoMultiSens preprocessing job for a given list of filenames of '
help="Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser).") 'downloaded satellite image archives!',
help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
"image archives! (Sub-Parser).")
parser_filenames = subparsers.add_parser('filenames', parser_constraints = subparsers\
description='Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite image archives!', .add_parser('constraints', description='Run a GeoMultiSens preprocessing job matching the given constraints.',
help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite image archives! (Sub-Parser).") help="Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser).")
parser_constraints = subparsers.add_parser('constraints', # parse_from_sceneids = subparsers.add_parser('from_sceneids',
description='Run a GeoMultiSens preprocessing job matching the given constraints.',
help="Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser).")
#parse_from_sceneids = subparsers.add_parser('from_sceneids',
# description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.', # description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
# help="use '>>> python /path/to/GeMultiSens/run_gms.py from_sceneids -h' for documentation and usage hints") # help="use '>>> python /path/to/GeMultiSens/run_gms.py from_sceneids -h' for documentation and usage hints")
# ADD ARGUMENTS
## ADD ARGUMENTS
# add arguments to parser_jobid # add arguments to parser_jobid
jid_p = parser_jobid.add_argument jid_p = parser_jobid.add_argument
jid_p('jobid', type=int, help='job ID of an already created GeoMultiSens preprocessing job (must be present in the jobs table of the database)') jid_p('jobid', type=int, help='job ID of an already created GeoMultiSens preprocessing job (must be present in the '
'jobs table of the database)')
# add arguments to parser_sceneids # add arguments to parser_sceneids
sid_p = parser_sceneids.add_argument sid_p = parser_sceneids.add_argument
...@@ -189,32 +188,30 @@ def get_gms_argparser(): ...@@ -189,32 +188,30 @@ def get_gms_argparser():
# add arguments to parse_constraints # add arguments to parse_constraints
con_p = parser_constraints.add_argument con_p = parser_constraints.add_argument
# TODO # TODO
#con_p('constraints', nargs='+', type=str, help="list of entity IDs corresponding to valid records within the " # con_p('constraints', nargs='+', type=str, help="list of entity IDs corresponding to valid records within the "
# "'scenes' table of the database") # "'scenes' table of the database")
# add general arguments # TODO add these configs to each subparser # add general arguments # TODO add these configs to each subparser
general_opts = { general_opts = {
'-db_host':dict(), '-db_host': dict(),
'-exec_mode':dict(nargs=3, type=bool, help="L1A Processor configuration", '-exec_mode': dict(nargs=3, type=bool, help="L1A Processor configuration",
metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]), metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]),
'-exec_L1AP':dict(), '-exec_L1AP': dict(),
'-exec_L1BP':dict(), '-exec_L1BP': dict(),
'-exec_L1CP':dict(), '-exec_L1CP': dict(),
'-exec_L2AP':dict(), '-exec_L2AP': dict(),
'-exec_L2BP':dict(), '-exec_L2BP': dict(),
'-exec_L2CP':dict(), '-exec_L2CP': dict(),
'-sub_multiProc':dict(), '-sub_multiProc': dict(),
'-exc_handler':dict(), '-exc_handler': dict(),
'-blocksize':dict(), '-blocksize': dict(),
'-profiling':dict(), '-profiling': dict(),
'-bench_all':dict(), '-bench_all': dict(),
'-bench_cloudMask':dict(), '-bench_cloudMask': dict(),
} }
# LINK PARSERS TO RUN FUNCTIONS
## LINK PARSERS TO RUN FUNCTIONS
parser_jobid.set_defaults(func=run_from_jobid) parser_jobid.set_defaults(func=run_from_jobid)
parser_sceneids.set_defaults(func=run_from_sceneids) parser_sceneids.set_defaults(func=run_from_sceneids)
parser_entityids.set_defaults(func=run_from_entityids) parser_entityids.set_defaults(func=run_from_entityids)
...@@ -224,8 +221,7 @@ def get_gms_argparser(): ...@@ -224,8 +221,7 @@ def get_gms_argparser():
return parser return parser
if __name__ == '__main__':
if __name__=='__main__':
# if len(sys.argv)<2: # if len(sys.argv)<2:
# # a job ID has not been given # # a job ID has not been given
# #
...@@ -242,7 +238,8 @@ if __name__=='__main__': ...@@ -242,7 +238,8 @@ if __name__=='__main__':
# # ID = 26185252 # 1x L8, Zielsensor L8, spat.ref L8 # # ID = 26185252 # 1x L8, Zielsensor L8, spat.ref L8
# # ID = 26185253 # 25x L8, Zielsensor L8, spat.ref L8 # # ID = 26185253 # 25x L8, Zielsensor L8, spat.ref L8
# # ID = 26185254 # 10x L8, Zielsensor L8, spat.ref L8 # # ID = 26185254 # 10x L8, Zielsensor L8, spat.ref L8
# # ID = 26185255 # 1x L8 Bug 5 corners found -> Grund=Schreibfehler L1A im tiled Python-mode bei mehr als 1 Szene im Job # # Grund=Schreibfehler L1A im tiled Python-mode bei mehr als 1 Szene im Job:
# # ID = 26185255 # 1x L8 Bug 5 corners found
# # ID = 26185256 # 1x L7 SLC off, Zielsensor L8, spat.ref L8 # # ID = 26185256 # 1x L7 SLC off, Zielsensor L8, spat.ref L8
# # ID = 26185257 # Beta-Job - 219 x L8, 172 x L7, 111 x S2, spatref L8 # # ID = 26185257 # Beta-Job - 219 x L8, 172 x L7, 111 x S2, spatref L8
# # ID = 26185258 # Beta-Job - 219 x L8, spatref L8 # # ID = 26185258 # Beta-Job - 219 x L8, spatref L8
...@@ -261,12 +258,15 @@ if __name__=='__main__': ...@@ -261,12 +258,15 @@ if __name__=='__main__':
# # ID = 26185275 # "1x L7, target L8, spat.ref L8 L1B Matching failed" # # ID = 26185275 # "1x L7, target L8, spat.ref L8 L1B Matching failed"
# # ID = 26185276 # "1x L7, target L8, spat.ref L8 L1B Matching window became too small." # # ID = 26185276 # "1x L7, target L8, spat.ref L8 L1B Matching window became too small."
# # ID = 26185279 # "GEOMS: 25x L7, target L8, spat.ref L8" # # ID = 26185279 # "GEOMS: 25x L7, target L8, spat.ref L8"
# # ID = 26185280 # "GEOMS: 1x L7, target L8, spat.ref L8, debugging NoneType object is not subscriptable within mapinfo2geotransform" # # "GEOMS: 1x L7, target L8, spat.ref L8, debugging NoneType object is not subscriptable within
# # mapinfo2geotransform":
# # ID = 26185280
# # ID = 26185281 # "GEOMS: 4x L7, target L8, spat.ref L8, freeze of pool.map" # # ID = 26185281 # "GEOMS: 4x L7, target L8, spat.ref L8, freeze of pool.map"
# # ID = 26185283 # "GEOMS: 10x L7, target L8, spat.ref L8, freeze of pool.map" # # ID = 26185283 # "GEOMS: 10x L7, target L8, spat.ref L8, freeze of pool.map"
# # ID = 26185284 # "GEOMS: 11x L7, target L8, spat.ref L8, freeze of pool.map" # # ID = 26185284 # "GEOMS: 11x L7, target L8, spat.ref L8, freeze of pool.map"
# # ID = 26185321 # "GEOMS: 1x L7, target L8, spat.ref L8, debugging L1B_P" # # ID = 26185321 # "GEOMS: 1x L7, target L8, spat.ref L8, debugging L1B_P"
# # ID = 26185322 # "GEOMS: 1x L7, target L8, spat.ref L8, Bug calc_shifted_cross_power_spectrum: NoneType object not iterable" # # "GEOMS: 1x L7, target L8, spat.ref L8, Bug calc_shifted_cross_power_spectrum: NoneType object not iterable":
# # ID = 26185322
# # ID = 26185277 # "GMS41: 10x L7, target L8, spat.ref L8, Permission errors during logging" # # ID = 26185277 # "GMS41: 10x L7, target L8, spat.ref L8, Permission errors during logging"
# # ID = 26185278 # "Beta-Job - 172 x L7, spatref L8" # # ID = 26185278 # "Beta-Job - 172 x L7, spatref L8"
# # ID = 26185284 # "GMS41: "all beta-L8 with cloud cover <30% (74 scenes)" # # ID = 26185284 # "GMS41: "all beta-L8 with cloud cover <30% (74 scenes)"
...@@ -277,11 +277,8 @@ if __name__=='__main__': ...@@ -277,11 +277,8 @@ if __name__=='__main__':
# else: # else:
# ID = int(sys.argv[1]) # ID = int(sys.argv[1])
# RUN! # RUN!
parsed_args = get_gms_argparser().parse_args() parsed_args = get_gms_argparser().parse_args()
parsed_args.func(parsed_args) parsed_args.func(parsed_args)
print('\nready.') print('\nready.')
...@@ -17,6 +17,7 @@ from datetime import datetime, timedelta ...@@ -17,6 +17,7 @@ from datetime import datetime, timedelta
import numpy as np import numpy as np
from geopandas import GeoDataFrame from geopandas import GeoDataFrame
from shapely.geometry import box from shapely.geometry import box
import pytz
from arosics import COREG, DESHIFTER from arosics import COREG, DESHIFTER
from geoarray import GeoArray from geoarray import GeoArray
...@@ -56,13 +57,18 @@ class Scene_finder(object): ...@@ -56,13 +57,18 @@ class Scene_finder(object):
if not (dt.month == 2 and dt.day == 29) else dt.replace(dt.year + years, 3, 1) if not (dt.month == 2 and dt.day == 29) else dt.replace(dt.year + years, 3, 1)
self.timeStart = add_years(self.src_AcqDate, -plusminus_years) self.timeStart = add_years(self.src_AcqDate, -plusminus_years)
timeEnd = add_years(self.src_AcqDate, +plusminus_years) timeEnd = add_years(self.src_AcqDate, +plusminus_years)
self.timeEnd = timeEnd if timeEnd <= datetime.now() else datetime.now() timeNow = datetime.utcnow().replace(tzinfo=pytz.UTC)
self.timeEnd = timeEnd if timeEnd <= timeNow else timeNow
self.possib_ref_scenes = None # set by self.spatial_query() self.possib_ref_scenes = None # set by self.spatial_query()
self.GDF_ref_scenes = GeoDataFrame() # set by self.spatial_query() self.GDF_ref_scenes = GeoDataFrame() # set by self.spatial_query()
self.ref_scene = None self.ref_scene = None
def spatial_query(self, timeout=5): def spatial_query(self, timeout=5):
"""Query the postgreSQL database to find possible reference scenes matching the specified criteria.
:param timeout: maximum query duration allowed (seconds)
"""
for i in range(10): for i in range(10):
try: try:
SpIM = SpatialIndexMediator(timeout=timeout) SpIM = SpatialIndexMediator(timeout=timeout)
...@@ -87,21 +93,22 @@ class Scene_finder(object): ...@@ -87,21 +93,22 @@ class Scene_finder(object):
GDF['cloudcover'] = list(GDF['object'].map(lambda scene: scene.cloudcover)) GDF['cloudcover'] = list(GDF['object'].map(lambda scene: scene.cloudcover))
GDF['polyLonLat'] = list(GDF['object'].map(lambda scene: scene.polyLonLat)) GDF['polyLonLat'] = list(GDF['object'].map(lambda scene: scene.polyLonLat))
def LonLat2UTM(polyLL): return reproject_shapelyGeometry(polyLL, 4326, self.src_prj) def LonLat2UTM(polyLL):
return reproject_shapelyGeometry(polyLL, 4326, self.src_prj)
GDF['polyUTM'] = list(GDF['polyLonLat'].map(LonLat2UTM)) GDF['polyUTM'] = list(GDF['polyLonLat'].map(LonLat2UTM))
self.GDF_ref_scenes = GDF self.GDF_ref_scenes = GDF
def filter_possib_ref_scenes(self): def filter_possib_ref_scenes(self):
"""Filter possible scenes by running all filter functions."""
self._filter_by_overlap() self._filter_by_overlap()
self._filter_by_proc_status() # self._filter_by_proc_status()
self._filter_by_dataset_existance() # self._filter_by_dataset_existance()
self._filter_by_entity_ID_availability() self._filter_by_entity_ID_availability()
self._filter_by_projection() self._filter_by_projection()
def choose_ref_scene(self): def choose_ref_scene(self):
"""choose refence scene with minimum cloud cover and maximum overlap""" """Choose reference scene with minimum cloud cover and maximum overlap."""
if not self.GDF_ref_scenes.empty: if not self.GDF_ref_scenes.empty:
GDF = self.GDF_ref_scenes GDF = self.GDF_ref_scenes
GDF = GDF[GDF['cloudcover'] == GDF['cloudcover'].min()] GDF = GDF[GDF['cloudcover'] == GDF['cloudcover'].min()]
...@@ -114,6 +121,7 @@ class Scene_finder(object): ...@@ -114,6 +121,7 @@ class Scene_finder(object):
return None return None
def _filter_by_overlap(self): def _filter_by_overlap(self):
"""Filter all scenes with less spatial overlap than self.min_overlap."""
GDF = self.GDF_ref_scenes GDF = self.GDF_ref_scenes
if not GDF.empty: if not GDF.empty:
# get overlap parameter # get overlap parameter
...@@ -128,6 +136,7 @@ class Scene_finder(object): ...@@ -128,6 +136,7 @@ class Scene_finder(object):
self.GDF_ref_scenes = GDF.loc[GDF['overlap percentage'] >= self.min_overlap] self.GDF_ref_scenes = GDF.loc[GDF['overlap percentage'] >= self.min_overlap]
def _filter_by_proc_status(self): def _filter_by_proc_status(self):
"""Filter all scenes that have not been processed before according to proc. status (at least L1A is needed)."""
GDF = self.GDF_ref_scenes GDF = self.GDF_ref_scenes
if not GDF.empty: if not GDF.empty:
# get processing level of refernce scenes # get processing level of refernce scenes
...@@ -142,6 +151,7 @@ class Scene_finder(object): ...@@ -142,6 +151,7 @@ class Scene_finder(object):
self.GDF_ref_scenes = GDF[GDF['proc_level'].notnull()] self.GDF_ref_scenes = GDF[GDF['proc_level'].notnull()]
def _filter_by_dataset_existance(self): def _filter_by_dataset_existance(self):
"""Filter all scenes where no processed data can be found on disk."""
GDF = self.GDF_ref_scenes GDF = self.GDF_ref_scenes
if not GDF.empty: if not GDF.empty:
# get path of binary file and check if the corresponding dataset exists # get path of binary file and check if the corresponding dataset exists
...@@ -159,6 +169,7 @@ class Scene_finder(object): ...@@ -159,6 +169,7 @@ class Scene_finder(object):
self.GDF_ref_scenes = GDF[GDF['refDs_exists']] self.GDF_ref_scenes = GDF[GDF['refDs_exists']]
def _filter_by_entity_ID_availability(self): def _filter_by_entity_ID_availability(self):
"""Filter all scenes where no proper entity ID can be found in the database."""
GDF = self.GDF_ref_scenes GDF = self.GDF_ref_scenes
if not GDF.empty: if not GDF.empty:
# check if a proper entity ID can be gathered from database # check if a proper entity ID can be gathered from database
...@@ -170,9 +181,10 @@ class Scene_finder(object): ...@@ -170,9 +181,10 @@ class Scene_finder(object):
GDF.drop('temp_queryRes', axis=1, inplace=True) GDF.drop('temp_queryRes', axis=1, inplace=True)
# filter scenes out that have no entity ID (database errors) # filter scenes out that have no entity ID (database errors)
self.GDF_ref_scenes = GDF[GDF['refDs_exists']] self.GDF_ref_scenes = GDF[GDF['entityid'].notnull()]
def _filter_by_projection(self): def _filter_by_projection(self):
"""Filter all scenes that have a different projection than the target image."""
GDF = self.GDF_ref_scenes GDF = self.GDF_ref_scenes
if not GDF.empty: if not GDF.empty:
# compare projections of target and reference image # compare projections of target and reference image
...@@ -219,7 +231,7 @@ class L1B_object(L1A_object): ...@@ -219,7 +231,7 @@ class L1B_object(L1A_object):
@property @property
def spatRef_available(self): def spatRef_available(self):
if self._spatRef_available is None: if self._spatRef_available is not None:
return self._spatRef_available return self._spatRef_available
else: else:
self.get_spatial_reference_scene() self.get_spatial_reference_scene()
......
...@@ -934,8 +934,8 @@ class GMS_JOB(object): ...@@ -934,8 +934,8 @@ class GMS_JOB(object):
self.sceneids = list(self.dataframe['sceneid']) self.sceneids = list(self.dataframe['sceneid'])
self.statistics = [len(self.sceneids)] + [0] * 8 self.statistics = [len(self.sceneids)] + [0] * 8
self.bounds = box(*MultiPolygon(list(self.dataframe['polygons'])).bounds) self.bounds = box(*MultiPolygon(list(self.dataframe['polygons'])).bounds)
self.timerange_start = self.dataframe.acquisitiondate.min().to_datetime() self.timerange_start = self.dataframe.acquisitiondate.min().to_pydatetime()
self.timerange_end = self.dataframe.acquisitiondate.max().to_datetime() self.timerange_end = self.dataframe.acquisitiondate.max().to_pydatetime()
def from_job_ID(self, job_ID): def from_job_ID(self, job_ID):
# type: (int) -> object # type: (int) -> object
......
...@@ -7,6 +7,7 @@ import re ...@@ -7,6 +7,7 @@ import re
import warnings import warnings
from datetime import datetime, timedelta from datetime import datetime, timedelta
from shapely.geometry import Polygon from shapely.geometry import Polygon
import pytz
class SpatialIndexMediatorServer: class SpatialIndexMediatorServer:
...@@ -227,9 +228,9 @@ class SpatialIndexMediator: ...@@ -227,9 +228,9 @@ class SpatialIndexMediator:
if filterTimerange: if filterTimerange:
if timestamp.month == 2 and timestamp.day == 29: if timestamp.month == 2 and timestamp.day == 29:
# deal with feb.29th # deal with feb.29th
timestampInRefYear = timestamp.replace(refDate.year, 3, 1) timestampInRefYear = timestamp.replace(refDate.year, 3, 1).replace(tzinfo=pytz.UTC)
else: else:
timestampInRefYear = timestamp.replace(refDate.year) timestampInRefYear = timestamp.replace(refDate.year).replace(tzinfo=pytz.UTC)
if abs(refDate - timestampInRefYear).days > maxDaysDelta: if abs(refDate - timestampInRefYear).days > maxDaysDelta:
# skip scene # skip scene
......
...@@ -12,6 +12,7 @@ shapely ...@@ -12,6 +12,7 @@ shapely
ephem ephem
pyorbital pyorbital
dill dill
pytz
pandas pandas
numba numba
spectral spectral
......
...@@ -11,9 +11,9 @@ with open('HISTORY.rst') as history_file: ...@@ -11,9 +11,9 @@ with open('HISTORY.rst') as history_file:
history = history_file.read() history = history_file.read()
requirements = [ requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz',
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy', 'psycopg2', 'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
'py_tools_ds', 'geoarray', 'arosics' 'psycopg2', 'py_tools_ds', 'geoarray', 'arosics'
# spectral<0.16 has some problems with writing signed integer 8bit data # spectral<0.16 has some problems with writing signed integer 8bit data
# fmask # conda install -c conda-forge python-fmask # fmask # conda install -c conda-forge python-fmask
# 'pyhdf', # conda install --yes -c conda-forge pyhdf # 'pyhdf', # conda install --yes -c conda-forge pyhdf
......
...@@ -13,11 +13,11 @@ RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \ ...@@ -13,11 +13,11 @@ RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
conda install --yes pyqt numba; \ conda install --yes pyqt numba; \
conda install --yes -c conda-forge pyfftw=0.10.4 ; \ conda install --yes -c conda-forge pyfftw=0.10.4 ; \
conda install --yes -c ioam holoviews bokeh ; \ conda install --yes -c ioam holoviews bokeh ; \
conda install --yes -c conda-forge numpy gdal scikit-image matplotlib pyproj rasterio shapely basemap pykrige \ conda install --yes -c conda-forge numpy gdal scikit-image scikit-learn matplotlib pyproj rasterio shapely basemap \
glymur pygrib pyproj cachetools pyhdf ephem python-fmask ; \ pykrige glymur pygrib pyproj cachetools pyhdf ephem python-fmask scipy ; \
conda install --yes -c conda-forge 'icu=58.*' lxml ; \ conda install --yes -c conda-forge 'icu=58.*' lxml ; \
pip install geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse \ pip install pandas geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse dill pytz \
spectral psycopg2 pyorbital pyinstrument geoalchemy2 py_tools_ds geoarray arosics \ spectral>0.16 psycopg2 pyorbital pyinstrument geoalchemy2 sqlalchemy py_tools_ds geoarray arosics \
flake8 pycodestyle pylint pydocstyle nose nose2 nose-htmloutput coverage rednose" # must include all the requirements needed to build the docs! flake8 pycodestyle pylint pydocstyle nose nose2 nose-htmloutput coverage rednose" # must include all the requirements needed to build the docs!
# copy some needed stuff to /root # copy some needed stuff to /root
......