Commit 2c2146d5 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Fixed some bugs and added docstrings within L1B_P.Scene_finder(). PEP8 editing for run_gms.py.


Former-commit-id: c5515d7c
parent 99907e9e
# -*- coding: utf-8 -*-
__author__='Daniel Scheffler'
# -*- coding: utf-8 -*-
__author__ = 'Daniel Scheffler'
import argparse
import sys
import warnings
import matplotlib
matplotlib.use('Agg', warn=False) # switch matplotlib backend to 'Agg' and disable warning in case its already 'Agg'
matplotlib.use('Agg', warn=False) # switch matplotlib backend to 'Agg' and disable warning in case its already 'Agg'
from gms_preprocessing import process_controller, __version__
from gms_preprocessing.misc.database_tools import GMS_JOB
from gms_preprocessing import process_controller, __version__ # noqa: E402
from gms_preprocessing.misc.database_tools import GMS_JOB # noqa: E402
def run_from_jobid(args):
......@@ -21,8 +20,8 @@ def run_from_jobid(args):
# set up process controller instance
PC = process_controller(args.jobid, parallelization_level='scenes')
#PC.job.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
#PC.job.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
# PC.job.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
# PC.job.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
# run the job
PC.run_all_processors()
......@@ -30,14 +29,14 @@ def run_from_jobid(args):
def run_from_sceneids(args):
# create and run a download job
warnings.warn('Currently the console argument parser expects the given scenes as already downloaded.') # TODO
warnings.warn('Currently the console argument parser expects the given scenes as already downloaded.') # TODO
# create a new processing job from scene IDs
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection)
......@@ -55,11 +54,11 @@ def run_from_entityids(args):
:return:
"""
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection)
......@@ -77,11 +76,11 @@ def run_from_filenames(args):
:return:
"""
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection)
......@@ -92,7 +91,6 @@ def run_from_filenames(args):
_run_job(dbJob)
def run_from_constraints(args):
# create a new job from constraints
# TODO
......@@ -124,50 +122,51 @@ def _run_job(dbJob, parallelization_level='scenes'):
def get_gms_argparser():
"""Return argument parser for run_gms.py program."""
### CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN
# CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN
parser = argparse.ArgumentParser(
prog='run_gms.py',
description='='*70+'\n'+'GeoMultiSens preprocessing console argument parser. ' \
'Python implementation by Daniel Scheffler (daniel.scheffler@gfz-potsdam.de)',
description='=' * 70 + '\n' + 'GeoMultiSens preprocessing console argument parser. '
'Python implementation by Daniel Scheffler (daniel.scheffler@gfz-potsdam.de)',
epilog="The argument parser offers multiple sub-argument parsers (jobid, sceneids, ...) for starting GMS jobs. "
"use '>>> python /path/to/gms_preprocessing/run_gms.py <sub-parser> -h' for detailed documentation and usage "
"hints.")
"use '>>> python /path/to/gms_preprocessing/run_gms.py <sub-parser> -h' for detailed documentation and "
"usage hints.")
parser.add_argument('--version', action='version', version=__version__)
subparsers = parser.add_subparsers()
# CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN
parser_jobid = subparsers\
.add_parser('jobid', description='Run a GeoMultiSens preprocessing job using an already existing job ID.',
help="Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser).")
### CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN
parser_jobid = subparsers.add_parser('jobid',
description= 'Run a GeoMultiSens preprocessing job using an already existing job ID.',
help="Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser).")
parser_sceneids = subparsers\
.add_parser('sceneids', description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser).")
parser_sceneids = subparsers.add_parser('sceneids',
description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser).")
parser_entityids = subparsers\
.add_parser('entityids', description='Run a GeoMultiSens preprocessing job for a given list of entity IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser).")
parser_entityids = subparsers.add_parser('entityids',
description='Run a GeoMultiSens preprocessing job for a given list of entity IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser).")
parser_filenames = subparsers\
.add_parser('filenames', description='Run a GeoMultiSens preprocessing job for a given list of filenames of '
'downloaded satellite image archives!',
help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
"image archives! (Sub-Parser).")
parser_filenames = subparsers.add_parser('filenames',
description='Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite image archives!',
help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite image archives! (Sub-Parser).")
parser_constraints = subparsers\
.add_parser('constraints', description='Run a GeoMultiSens preprocessing job matching the given constraints.',
help="Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser).")
parser_constraints = subparsers.add_parser('constraints',
description='Run a GeoMultiSens preprocessing job matching the given constraints.',
help="Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser).")
#parse_from_sceneids = subparsers.add_parser('from_sceneids',
# parse_from_sceneids = subparsers.add_parser('from_sceneids',
# description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
# help="use '>>> python /path/to/GeMultiSens/run_gms.py from_sceneids -h' for documentation and usage hints")
## ADD ARGUMENTS
# ADD ARGUMENTS
# add arguments to parser_jobid
jid_p = parser_jobid.add_argument
jid_p('jobid', type=int, help='job ID of an already created GeoMultiSens preprocessing job (must be present in the jobs table of the database)')
jid_p('jobid', type=int, help='job ID of an already created GeoMultiSens preprocessing job (must be present in the '
'jobs table of the database)')
# add arguments to parser_sceneids
sid_p = parser_sceneids.add_argument
......@@ -189,32 +188,30 @@ def get_gms_argparser():
# add arguments to parse_constraints
con_p = parser_constraints.add_argument
# TODO
#con_p('constraints', nargs='+', type=str, help="list of entity IDs corresponding to valid records within the "
# con_p('constraints', nargs='+', type=str, help="list of entity IDs corresponding to valid records within the "
# "'scenes' table of the database")
# add general arguments # TODO add these configs to each subparser
general_opts = {
'-db_host':dict(),
'-exec_mode':dict(nargs=3, type=bool, help="L1A Processor configuration",
metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]),
'-exec_L1AP':dict(),
'-exec_L1BP':dict(),
'-exec_L1CP':dict(),
'-exec_L2AP':dict(),
'-exec_L2BP':dict(),
'-exec_L2CP':dict(),
'-sub_multiProc':dict(),
'-exc_handler':dict(),
'-blocksize':dict(),
'-profiling':dict(),
'-bench_all':dict(),
'-bench_cloudMask':dict(),
'-db_host': dict(),
'-exec_mode': dict(nargs=3, type=bool, help="L1A Processor configuration",
metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]),
'-exec_L1AP': dict(),
'-exec_L1BP': dict(),
'-exec_L1CP': dict(),
'-exec_L2AP': dict(),
'-exec_L2BP': dict(),
'-exec_L2CP': dict(),
'-sub_multiProc': dict(),
'-exc_handler': dict(),
'-blocksize': dict(),
'-profiling': dict(),
'-bench_all': dict(),
'-bench_cloudMask': dict(),
}
## LINK PARSERS TO RUN FUNCTIONS
# LINK PARSERS TO RUN FUNCTIONS
parser_jobid.set_defaults(func=run_from_jobid)
parser_sceneids.set_defaults(func=run_from_sceneids)
parser_entityids.set_defaults(func=run_from_entityids)
......@@ -224,8 +221,7 @@ def get_gms_argparser():
return parser
if __name__=='__main__':
if __name__ == '__main__':
# if len(sys.argv)<2:
# # a job ID has not been given
#
......@@ -242,7 +238,8 @@ if __name__=='__main__':
# # ID = 26185252 # 1x L8, Zielsensor L8, spat.ref L8
# # ID = 26185253 # 25x L8, Zielsensor L8, spat.ref L8
# # ID = 26185254 # 10x L8, Zielsensor L8, spat.ref L8
# # ID = 26185255 # 1x L8 Bug 5 corners found -> Grund=Schreibfehler L1A im tiled Python-mode bei mehr als 1 Szene im Job
# # Grund=Schreibfehler L1A im tiled Python-mode bei mehr als 1 Szene im Job:
# # ID = 26185255 # 1x L8 Bug 5 corners found
# # ID = 26185256 # 1x L7 SLC off, Zielsensor L8, spat.ref L8
# # ID = 26185257 # Beta-Job - 219 x L8, 172 x L7, 111 x S2, spatref L8
# # ID = 26185258 # Beta-Job - 219 x L8, spatref L8
......@@ -261,12 +258,15 @@ if __name__=='__main__':
# # ID = 26185275 # "1x L7, target L8, spat.ref L8 L1B Matching failed"
# # ID = 26185276 # "1x L7, target L8, spat.ref L8 L1B Matching window became too small."
# # ID = 26185279 # "GEOMS: 25x L7, target L8, spat.ref L8"
# # ID = 26185280 # "GEOMS: 1x L7, target L8, spat.ref L8, debugging NoneType object is not subscriptable within mapinfo2geotransform"
# # "GEOMS: 1x L7, target L8, spat.ref L8, debugging NoneType object is not subscriptable within
# # mapinfo2geotransform":
# # ID = 26185280
# # ID = 26185281 # "GEOMS: 4x L7, target L8, spat.ref L8, freeze of pool.map"
# # ID = 26185283 # "GEOMS: 10x L7, target L8, spat.ref L8, freeze of pool.map"
# # ID = 26185284 # "GEOMS: 11x L7, target L8, spat.ref L8, freeze of pool.map"
# # ID = 26185321 # "GEOMS: 1x L7, target L8, spat.ref L8, debugging L1B_P"
# # ID = 26185322 # "GEOMS: 1x L7, target L8, spat.ref L8, Bug calc_shifted_cross_power_spectrum: NoneType object not iterable"
# # "GEOMS: 1x L7, target L8, spat.ref L8, Bug calc_shifted_cross_power_spectrum: NoneType object not iterable":
# # ID = 26185322
# # ID = 26185277 # "GMS41: 10x L7, target L8, spat.ref L8, Permission errors during logging"
# # ID = 26185278 # "Beta-Job - 172 x L7, spatref L8"
# # ID = 26185284 # "GMS41: "all beta-L8 with cloud cover <30% (74 scenes)"
......@@ -277,11 +277,8 @@ if __name__=='__main__':
# else:
# ID = int(sys.argv[1])
# RUN!
parsed_args = get_gms_argparser().parse_args()
parsed_args.func(parsed_args)
print('\nready.')
......@@ -17,6 +17,7 @@ from datetime import datetime, timedelta
import numpy as np
from geopandas import GeoDataFrame
from shapely.geometry import box
import pytz
from arosics import COREG, DESHIFTER
from geoarray import GeoArray
......@@ -56,13 +57,18 @@ class Scene_finder(object):
if not (dt.month == 2 and dt.day == 29) else dt.replace(dt.year + years, 3, 1)
self.timeStart = add_years(self.src_AcqDate, -plusminus_years)
timeEnd = add_years(self.src_AcqDate, +plusminus_years)
self.timeEnd = timeEnd if timeEnd <= datetime.now() else datetime.now()
timeNow = datetime.utcnow().replace(tzinfo=pytz.UTC)
self.timeEnd = timeEnd if timeEnd <= timeNow else timeNow
self.possib_ref_scenes = None # set by self.spatial_query()
self.GDF_ref_scenes = GeoDataFrame() # set by self.spatial_query()
self.ref_scene = None
def spatial_query(self, timeout=5):
"""Query the postgreSQL database to find possible reference scenes matching the specified criteria.
:param timeout: maximum query duration allowed (seconds)
"""
for i in range(10):
try:
SpIM = SpatialIndexMediator(timeout=timeout)
......@@ -87,21 +93,22 @@ class Scene_finder(object):
GDF['cloudcover'] = list(GDF['object'].map(lambda scene: scene.cloudcover))
GDF['polyLonLat'] = list(GDF['object'].map(lambda scene: scene.polyLonLat))
def LonLat2UTM(polyLL): return reproject_shapelyGeometry(polyLL, 4326, self.src_prj)
def LonLat2UTM(polyLL):
return reproject_shapelyGeometry(polyLL, 4326, self.src_prj)
GDF['polyUTM'] = list(GDF['polyLonLat'].map(LonLat2UTM))
self.GDF_ref_scenes = GDF
def filter_possib_ref_scenes(self):
"""Filter possible scenes by running all filter functions."""
self._filter_by_overlap()
self._filter_by_proc_status()
self._filter_by_dataset_existance()
# self._filter_by_proc_status()
# self._filter_by_dataset_existance()
self._filter_by_entity_ID_availability()
self._filter_by_projection()
def choose_ref_scene(self):
"""choose refence scene with minimum cloud cover and maximum overlap"""
"""Choose reference scene with minimum cloud cover and maximum overlap."""
if not self.GDF_ref_scenes.empty:
GDF = self.GDF_ref_scenes
GDF = GDF[GDF['cloudcover'] == GDF['cloudcover'].min()]
......@@ -114,6 +121,7 @@ class Scene_finder(object):
return None
def _filter_by_overlap(self):
"""Filter all scenes with less spatial overlap than self.min_overlap."""
GDF = self.GDF_ref_scenes
if not GDF.empty:
# get overlap parameter
......@@ -128,6 +136,7 @@ class Scene_finder(object):
self.GDF_ref_scenes = GDF.loc[GDF['overlap percentage'] >= self.min_overlap]
def _filter_by_proc_status(self):
"""Filter all scenes that have not been processed before according to proc. status (at least L1A is needed)."""
GDF = self.GDF_ref_scenes
if not GDF.empty:
# get processing level of refernce scenes
......@@ -142,6 +151,7 @@ class Scene_finder(object):
self.GDF_ref_scenes = GDF[GDF['proc_level'].notnull()]
def _filter_by_dataset_existance(self):
"""Filter all scenes where no processed data can be found on disk."""
GDF = self.GDF_ref_scenes
if not GDF.empty:
# get path of binary file and check if the corresponding dataset exists
......@@ -159,6 +169,7 @@ class Scene_finder(object):
self.GDF_ref_scenes = GDF[GDF['refDs_exists']]
def _filter_by_entity_ID_availability(self):
"""Filter all scenes where no proper entity ID can be found in the database."""
GDF = self.GDF_ref_scenes
if not GDF.empty:
# check if a proper entity ID can be gathered from database
......@@ -170,9 +181,10 @@ class Scene_finder(object):
GDF.drop('temp_queryRes', axis=1, inplace=True)
# filter scenes out that have no entity ID (database errors)
self.GDF_ref_scenes = GDF[GDF['refDs_exists']]
self.GDF_ref_scenes = GDF[GDF['entityid'].notnull()]
def _filter_by_projection(self):
"""Filter all scenes that have a different projection than the target image."""
GDF = self.GDF_ref_scenes
if not GDF.empty:
# compare projections of target and reference image
......@@ -219,7 +231,7 @@ class L1B_object(L1A_object):
@property
def spatRef_available(self):
if self._spatRef_available is None:
if self._spatRef_available is not None:
return self._spatRef_available
else:
self.get_spatial_reference_scene()
......
......@@ -934,8 +934,8 @@ class GMS_JOB(object):
self.sceneids = list(self.dataframe['sceneid'])
self.statistics = [len(self.sceneids)] + [0] * 8
self.bounds = box(*MultiPolygon(list(self.dataframe['polygons'])).bounds)
self.timerange_start = self.dataframe.acquisitiondate.min().to_datetime()
self.timerange_end = self.dataframe.acquisitiondate.max().to_datetime()
self.timerange_start = self.dataframe.acquisitiondate.min().to_pydatetime()
self.timerange_end = self.dataframe.acquisitiondate.max().to_pydatetime()
def from_job_ID(self, job_ID):
# type: (int) -> object
......
......@@ -7,6 +7,7 @@ import re
import warnings
from datetime import datetime, timedelta
from shapely.geometry import Polygon
import pytz
class SpatialIndexMediatorServer:
......@@ -227,9 +228,9 @@ class SpatialIndexMediator:
if filterTimerange:
if timestamp.month == 2 and timestamp.day == 29:
# deal with feb.29th
timestampInRefYear = timestamp.replace(refDate.year, 3, 1)
timestampInRefYear = timestamp.replace(refDate.year, 3, 1).replace(tzinfo=pytz.UTC)
else:
timestampInRefYear = timestamp.replace(refDate.year)
timestampInRefYear = timestamp.replace(refDate.year).replace(tzinfo=pytz.UTC)
if abs(refDate - timestampInRefYear).days > maxDaysDelta:
# skip scene
......
......@@ -11,9 +11,9 @@ with open('HISTORY.rst') as history_file:
history = history_file.read()
requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill',
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy', 'psycopg2',
'py_tools_ds', 'geoarray', 'arosics'
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz',
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
'psycopg2', 'py_tools_ds', 'geoarray', 'arosics'
# spectral<0.16 has some problems with writing signed integer 8bit data
# fmask # conda install -c conda-forge python-fmask
# 'pyhdf', # conda install --yes -c conda-forge pyhdf
......
......@@ -13,11 +13,11 @@ RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
conda install --yes pyqt numba; \
conda install --yes -c conda-forge pyfftw=0.10.4 ; \
conda install --yes -c ioam holoviews bokeh ; \
conda install --yes -c conda-forge numpy gdal scikit-image matplotlib pyproj rasterio shapely basemap pykrige \
glymur pygrib pyproj cachetools pyhdf ephem python-fmask ; \
conda install --yes -c conda-forge numpy gdal scikit-image scikit-learn matplotlib pyproj rasterio shapely basemap \
pykrige glymur pygrib pyproj cachetools pyhdf ephem python-fmask scipy ; \
conda install --yes -c conda-forge 'icu=58.*' lxml ; \
pip install geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse \
spectral psycopg2 pyorbital pyinstrument geoalchemy2 py_tools_ds geoarray arosics \
pip install pandas geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse dill pytz \
spectral>0.16 psycopg2 pyorbital pyinstrument geoalchemy2 sqlalchemy py_tools_ds geoarray arosics \
flake8 pycodestyle pylint pydocstyle nose nose2 nose-htmloutput coverage rednose" # must include all the requirements needed to build the docs!
# copy some needed stuff to /root
......
......@@ -192,7 +192,7 @@ class Test_Landsat7_SLC_off_PreCollectionData(BaseTestCases.TestAll):
# def setUpClass(cls):
# cls.create_job(26186267, job_config_kwargs) # FIXME job_ID!
#
class Test_Landsat8_PreCollectionData(BaseTestCases.TestAll):
"""
Parametrized testclass. Tests the level-processes on a Landsat-8 OLI_TIRS scene (pre-collection data).
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment