Commit 7e498f39 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Added console parser functionality to run GMS job from a list of archive filenames.

bin.run_gms:
- updated run_from_sceneids()
- revised run_from_entityids(): now working
- added run_from_filenames()
- added _run_job()
- added new argument subparser 'filenames'

geomultisens.misc.database_tools:
- GMS_JOB:
    - added from_entityIDlist()
    - added from_filenames()

updated __version__ and __versionalias__
parent 5acce159
......@@ -45,64 +45,75 @@ def run_from_sceneids(args):
virtual_sensor_id=virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref,
comment='')
dbJob.create()
jobid = dbJob.id
# set up process controller instance
parallelization_level = 'scenes'
warnings.warn("Currently the console argument parser sets the parallelization level to 'scenes'.") # TODO
PC = process_controller(jobid, parallelization_level=parallelization_level)
#PC.job.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
#PC.job.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
# run the job
PC.run_all_processors()
_run_job(dbJob)
def run_from_entityids(args):
# create a new job from entity IDs
# TODO
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
dbJob = GMS_JOB(db_connection)
dbJob.from_entityIDlist() # TODO
"""Create a new job from entity IDs.
:param args:
:return:
"""
# set up process controller instance
PC = process_controller(args.ID, parallelization_level='scenes')
# PC.job.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
# PC.job.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
# run the job
PC.run_all_processors()
dbJob = GMS_JOB(db_connection)
dbJob.from_entityIDlist(list_entityids=args.entityids,
virtual_sensor_id=virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref,
comment='')
_run_job(dbJob)
def run_from_filenames(args):
# find out sceneIDs belonging to filenames
# create a new job from entity IDs
# TODO
"""Create a new GMS job from filenames of downloaded archives and run it!
:param args:
:return:
"""
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
# set up process controller instance
PC = process_controller(args.ID, parallelization_level='scenes')
# PC.job.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
# PC.job.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
dbJob = GMS_JOB(db_connection)
dbJob.from_filenames(list_filenames=args.filenames,
virtual_sensor_id=virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref,
comment='')
_run_job(dbJob)
# run the job
PC.run_all_processors()
def run_from_constraints(args):
# create a new job from constraints
# TODO
raise NotImplementedError
def _run_job(dbJob, parallelization_level='scenes'):
# type: (GMS_JOB) -> None
"""
:param dbJob:
:return:
"""
dbJob.create()
jobid = dbJob.id
# set up process controller instance
PC = process_controller(args.ID, parallelization_level='scenes')
warnings.warn("Currently the console argument parser sets the parallelization level to 'scenes'.") # TODO
PC = process_controller(jobid, parallelization_level=parallelization_level)
# PC.job.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
# PC.job.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
......@@ -191,6 +202,10 @@ if __name__=='__main__':
description='Run a GeoMultiSens job for a given list of entity IDs.',
help="Run a GeoMultiSens job for a given list of entity IDs (Sub-Parser).")
parser_filenames = subparsers.add_parser('filenames',
description='Run a GeoMultiSens job for a given list of filenames of downloaded satellite image archives!',
help="Run a GeoMultiSens job for a given list of filenames of downloaded satellite image archives! (Sub-Parser).")
parser_constraints = subparsers.add_parser('constraints',
description='Run a GeoMultiSens job matching the given constraints.',
help="Run a GeoMultiSens job matching the given constraints (Sub-Parser).")
......@@ -202,24 +217,30 @@ if __name__=='__main__':
## ADD ARGUMENTS
if sys.argv[1]=='jobid':
# add arguments to parse_jobid
# add arguments to parser_jobid
jid_p = parser_jobid.add_argument
jid_p('jobid', type=int,
help='job ID of an already created GeoMultiSens job (must be present in the jobs table of the database)')
elif sys.argv[1]=='sceneids':
# add arguments to parse_sceneids
# add arguments to parser_sceneids
sid_p = parser_sceneids.add_argument
sid_p('sceneids', nargs='+', type=int,
help="list of scene IDs corresponding to valid records within the 'scenes' table of the database")
elif sys.argv[1]=='entityids':
# add arguments to parse_entityids
# add arguments to parser_entityids
eid_p = parser_entityids.add_argument
eid_p('entityids', nargs='+', type=str,
help="list of entity IDs corresponding to valid records within the 'scenes' table of the database")
# FIXME satellite and sensor are required
elif sys.argv[1]=='filenames':
# add arguments to parser_filenames
eid_p = parser_filenames.add_argument
eid_p('filenames', nargs='+', type=str,
help="list of filenames of satellite image archives corresponding to valid records within the 'scenes' "
"table of the database")
elif sys.argv[1]=='constraints':
# add arguments to parse_constraints
......@@ -254,6 +275,7 @@ if __name__=='__main__':
parser_jobid.set_defaults(func=run_from_jobid)
parser_sceneids.set_defaults(func=run_from_sceneids)
parser_entityids.set_defaults(func=run_from_entityids)
parser_filenames.set_defaults(func=run_from_filenames)
parser_constraints.set_defaults(func=run_from_constraints)
......
......@@ -2,8 +2,8 @@
__author__ = """Daniel Scheffler"""
__email__ = 'daniel.scheffler@gfz-potsdam.de'
__version__ = '0.2.1'
__versionalias__ = '20170531.01'
__version__ = '0.3.0'
__versionalias__ = '20170601.01'
from . import algorithms
......
......@@ -765,14 +765,52 @@ class GMS_JOB(object):
return self
def from_entityIDlist(self):
def from_entityIDlist(self, list_entityids, virtual_sensor_id, datasetid_spatial_ref=249, comment=None):
"""Create a GMS_JOB instance based on the given list of entity IDs.
:param list_entityids:
:param virtual_sensor_id:
:param datasetid_spatial_ref:
:param comment:
:return:
"""
Create a GMS_JOB instance based on the given list of entity IDs.
res_sceneIDs = get_info_from_postgreSQLdb(self.conn, 'scenes', ['id',], {'entityid': list_entityids})
if not res_sceneIDs:
raise ValueError('No matching database entries found for the given entity IDs.')
list_sceneIDs = np.array(res_sceneIDs)[:,0].tolist()
count_no_match = len(list_entityids) - len(list_sceneIDs)
if count_no_match:
warnings.warn('%s datasets could not be found the database. They cannot be processed.')
return self.from_sceneIDlist(list_sceneIDs, virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref, comment=comment)
def from_filenames(self, list_filenames, virtual_sensor_id, datasetid_spatial_ref=249, comment=None):
"""Create a GMS_JOB instance based on the given list of provider archive filenames.
:param list_filenames:
:param virtual_sensor_id:
:param datasetid_spatial_ref:
:param comment:
:return:
"""
raise NotImplementedError # TODO
return self
res_sceneIDs = get_info_from_postgreSQLdb(self.conn, 'scenes', ['id',], {'filename': list_filenames})
if not res_sceneIDs:
raise ValueError('No matching database entries found for the given filenames.')
list_sceneIDs = np.array(res_sceneIDs)[:,0].tolist()
count_no_match = len(list_filenames) - len(list_sceneIDs)
if count_no_match:
warnings.warn('%s datasets could not be found the database. They cannot be processed.')
return self.from_sceneIDlist(list_sceneIDs, virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref, comment=comment)
def _get_validated_sceneInfoGDFs(self, GDF_SatSenFname):
......
# -*- coding: utf-8 -*-
from __future__ import (division, print_function, unicode_literals,absolute_import)
__author__='Daniel Scheffler'
import numpy as np
from pandas import DataFrame
__author__='Daniel Scheffler'
import datetime
import os
import time
......
......@@ -11,7 +11,7 @@ with open('HISTORY.rst') as history_file:
requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'pyephem', 'pyorbital', 'dill',
'pandas', 'numba', 'spectral', 'pyhdf', 'geopandas', 'iso8601',
'pandas', 'numba', 'spectral', 'pyhdf', 'geopandas', 'iso8601', 'pyinstrument',
]
test_requirements = [
......@@ -20,7 +20,7 @@ test_requirements = [
setup(
name='geomultisens',
version='0.2.1',
version='0.3.0',
description="GeoMultiSens - Scalable Multi-Sensor Analysis of Remote Sensing Data",
long_description=readme + '\n\n' + history,
author="Daniel Scheffler",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment