Commit 944b84d1 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

- L1B_P: L1B-shift calculation now working but input data is still read from disk

- L1B_P: implemented dynamic postgresql-query for getting a matching reference image for an image to be shifted
- all database related functions moved to new module called database_tools
- all path related functions moved to a new module called path_generator
- added  some docstrings
- GEOP: added pixelToMapYX()
- GEOP Bugfix: get_overlap_polygon() now returns an overlap percentage of 0 if no overlap exists (instead of an error)
- L0A_P: derivation of scene basic metadata needed for job creation moved to DB_T.get_scene_and_dataset_infos_from_postgreSQLdb(sceneid)
parent 891590f2
......@@ -41,9 +41,9 @@ except:
# custom
# import sysenvironment as s
import pyproj
from pyorbital import astronomy
from pyorbital import astronomy
import ephem
import spectral.io.envi as envi
from spectral.io import envi
from shapely.geometry import Polygon
from shapely.geometry import shape
......@@ -1522,9 +1522,9 @@ class GEOPROCESSING(object):
if 'GDAL_DATA' not in os.environ:
self.logger.warning("GDAL_DATA variable not set. Setting it to '%s'." % sys.executable.split('anaconda')[
0] + 'anaconda/share/gdal') # Prevents "Unable to open EPSG support file gcs.csv".
anaconda_path = re.search('([\w+\S\s\w+]*anaconda[\w+\S\s\w+]*?)/',sys.executable,re.I).group(1)
anaconda_path = re.search('([\w+\S\s]*anaconda[\w+\S\s]*?)/',sys.executable,re.I).group(1)
os.environ['GDAL_DATA'] = os.path.join(anaconda_path,'share/gdal')
if not re.search('/anaconda[\w+\S\s\w+]*?/share/gdal', os.environ['GDAL_DATA'],re.I):
if not re.search('/anaconda[\w+\S\s]*?/share/gdal', os.environ['GDAL_DATA'],re.I):
self.logger.warning(
'GDAL_DATA variable seems to be incorrectly set since Anaconda´s Python is executed and GDAL_DATA = %s' %
os.environ['GDAL_DATA'])
......@@ -1720,10 +1720,12 @@ class GEOPROCESSING(object):
return mask_1bit
def calc_FullDataset_corner_positions(self, mask_data_nodata=None):
'''
"""
Calculates the image coordinates of the true data corners for rotated datasets.
ONLY usable for entire images - no tiles!
'''
:param mask_data_nodata: 2D-numpy array 1bit
:return: [UL, UR, LL, LR] as [(ULrow,ULcol),(URrow,URcol),...],[(ULLon,ULLat),(URLon,URLat),..]
"""
if self.subset is not None: self.logger.critical(
'Unable to calculate trueDataCornerPos for a subset of an image. calc_FullDataset_corner_positions can \
only be called if GEOP object represents a full image.'); sys.exit()
......@@ -2571,7 +2573,10 @@ def defBandNo(hdrPath, searchW, v=0):
if type(searchW) != list:
searchW = [float(searchW)]
wL = re.search("wavelength[\s]*=[\s]*{([0-9.,\s]*)}", hdr_, re.I)
wL = re.search("wavelength[\s]*=[\s]*([0-9.,\s]*)", hdr_, re.I)
print('regexp had to be changed WITHOUT test due to pycharm error: dangling metacharacter')
# originally: re.search("wavelength[\s]*=[\s]*{([0-9.,\s]*)}", hdr_, re.I)
if wL == None:
print("No wavelength specified")
......@@ -3461,8 +3466,8 @@ def transform_utm_to_wgs84(easting, northing, zone):
UTM = pyproj.Proj(proj='utm', zone=zone, ellps='WGS84')
return UTM(easting, northing, inverse=True)
def transform_wgs84_to_utm(lon, lat):
''' returns easting, northing, altitude '''
def transform_wgs84_to_utm(lon, lat, zone=None):
''' returns easting, northing, altitude. If zone is not set it is derived automatically. '''
if not lon >= -180. or not lon <= 180. or not lat >= -90. or not lat <= 90.:
print('''GEOPROCESSING error: Input coordinates for transform_wgs84_to_utm() out of range.
Got %s/%s.''' %(lon,lat))
......@@ -3487,7 +3492,7 @@ def transform_wgs84_to_utm(lon, lat):
# # create transform component
# wgs84_to_utm_geo_transform = osr.CoordinateTransformation(wgs84_coordinate_system, utm_coordinate_system) # (, )
# return wgs84_to_utm_geo_transform.TransformPoint(lon, lat, 0) # [RW,HW,meters above sea level]
UTM = pyproj.Proj(proj='utm', zone=get_utm_zone(lon), ellps='WGS84')
UTM = pyproj.Proj(proj='utm', zone=get_utm_zone(lon) if zone is None else zone, ellps='WGS84')
return UTM(lon, lat)
def get_point_bounds(points):
......@@ -3647,18 +3652,65 @@ def pixelToLatLon(geotifAddr, pixelPairs):
return latLonPairs
def pixelToMapYX(path_im, pixelCoords):
"""Translates given pixel locations into map coordinates of the given image.
INPUTS:
geotransform - GDAL geotransform of the image to be shifted
projection - GDAL projection of the image to be shifted
pixelCoords - The pixel coordinates to be translated in the form [x,y]. A list of coordinate pairs is possible: [[x1,y1],[x2,y2]].
OUTPUT:
The lat/lon or UTM translation result of the pixel coordinates in the form [lat,lon] or [[lat1,lon1],[lat2,lon2]].
"""
ds = gdal.Open(path_im)
gt = ds.GetGeoTransform()
srs = osr.SpatialReference()
srs.ImportFromWkt(ds.GetProjection())
# Set up the coordinate transformation object
ct = osr.CoordinateTransformation(srs, srs)
mapYmapXPairs = []
pixelCoords = [pixelCoords] if not isinstance(pixelCoords[0],list) else pixelCoords
for point in pixelCoords:
# Translate the pixel pairs into untranslated points
u_mapX = point[0] * gt[1] + gt[0]
u_mapY = point[1] * gt[5] + gt[3]
# Transform the points to the space
(mapX, mapY, holder) = ct.TransformPoint(u_mapX, u_mapY)
# Add the point to our return array
mapYmapXPairs.append([mapY, mapX])
return mapYmapXPairs
def EPSG2Proj4(EPSG_code):
srs = osr.SpatialReference()
srs.ImportFromEPSG(EPSG_code)
return srs.ExportToProj4()
def get_footprint_polygon(CornerLonLat): ## NOT tested!
return Polygon(CornerLonLat)
def get_footprint_polygon(CornerLonLat):
""" Converts a list of coordinates into a shapely polygon object.
:param CornerLonLat: a list of coordinate tuples like [[lon,lat], [lon. lat], ..]
in clockwise or counter clockwise order
:return: a shapyly.Polygon() object
"""
outpoly = Polygon(CornerLonLat)
assert outpoly. is_valid, 'The given coordinates result in an invalid polygon. Check coordinate order.'
return outpoly
def get_overlap_polygon(poly1, poly2):
""" Returns the overlap of two shapely.Polygon() objects.
:param poly1: first shapely.Polygon() object
:param poly2: second shapely.Polygon() object
:return: overlap polygon as shapely.Polygon() object
:return: overlap percentage as float value [%]
:return: area of overlap polygon
"""
overlap_poly = poly1.intersection(poly2)
overlap_percentage = 100 * shape(overlap_poly).area / shape(poly2).area
return overlap_poly, overlap_percentage
if not overlap_poly.is_empty:
overlap_percentage = 100 * shape(overlap_poly).area / shape(poly2).area
return {'overlap poly':overlap_poly, 'overlap percentage':overlap_percentage, 'overlap area':overlap_poly.area}
else:
return {'overlap poly':None, 'overlap percentage':0, 'overlap area':0}
# def get_lonlat_coord_array(rasObj_or_ENVI_file): ### doch wieder in object? -> object muss eh instanziert werden, um aufnahmegeometrin zu berechnen
# '''Returns numpy array containing longitude pixel coordinates (band 0) and latitude pixel coordinates (band 1).'''
......
......@@ -26,6 +26,8 @@ from gms_io import Input_reader as INP_R
from gms_io import Output_writer as OUT_W
from misc import helper_functions as HLP_F
from algorithms import L0B_P as L0B_P
from misc import database_tools as DB_T
from misc import path_generator as PG
from algorithms.METADATA_BD import get_LayerBandsAssignment
########################### core functions ####################################
......@@ -80,7 +82,7 @@ def get_entity_IDs_within_AOI(): # called in console mode
def get_data_list_of_current_jobID(): # called in webapp mode
query = lambda tablename,vals2return,cond_dict,records2fetch=0:\
INP_R.get_info_from_postgreSQLdb(job.conn_database,tablename,vals2return,cond_dict,records2fetch)
DB_T.get_info_from_postgreSQLdb(job.conn_database,tablename,vals2return,cond_dict,records2fetch)
resultset = query('scenes_jobs','sceneids',{'id':job.ID},3)
assert len(resultset) != 0, "Invalid jobID given - no corresponding job with the ID=%s found in database.\n" %job.ID
assert len(resultset) == 1, "Error in database. The jobID %s exists more than once. \n" %job.ID
......@@ -88,24 +90,7 @@ def get_data_list_of_current_jobID(): # called in webapp mode
data_list =[]
for sceneid in sceneids:
# get scene and dataset infos from db
resultset = query('scenes',['datasetid','satelliteid','sensorid','subsystemid', 'acquisitiondate', 'entityid',
'filename'],{'id':sceneid})
if len(resultset) == 0: sys.stderr.write("Scene with id %s not found. Skipping.." % sceneid); continue
scenedata = resultset[0]
ds = collections.OrderedDict()
ds.update({'proc_level' :'L0A'})
ds.update({'scene_ID' :sceneid})
ds.update({'datasetid' :scenedata[0]})
ds.update({'image_type' :query('datasets' ,'image_type',{'id':scenedata[0]})[0][0]})
ds.update({'satellite' :query('satellites','name' ,{'id':scenedata[1]})[0][0]})
ds.update({'sensor' :query('sensors' ,'name' ,{'id':scenedata[2]})[0][0]})
ds.update({'subsystem' :query('subsystems','name' ,{'id':scenedata[3]})[0][0] \
if scenedata[3] is not None else None})
ds.update({'acquisition_date':scenedata[4]})
ds.update({'entity_ID' :scenedata[5]})
ds.update({'filename' :scenedata[6]})
ds = DB_T.get_scene_and_dataset_infos_from_postgreSQLdb(sceneid)
ds['sensor'] = 'ETM+' if re.search('ETM+', ds['sensor']) else ds['sensor']
if usecase.skip_thermal and ds['subsystem']=='TIR': continue # removes ASTER TIR in case of skip_thermal
......@@ -133,7 +118,7 @@ def LandsatID2dataset(ID_list):
def get_sensormode(dataset):
if re.search('SPOT',dataset['satellite']):
path_archive = HLP_F.path_generator(dataset).get_local_archive_path_baseN()
path_archive = PG.path_generator(dataset).get_local_archive_path_baseN()
dim_ = HLP_F.open_specific_file_within_archive(path_archive,'*/scene01/metadata.dim')[0]
SPOT_mode = re.search("<SENSOR_CODE>([a-zA-Z0-9]*)</SENSOR_CODE>",dim_, re.I).group(1)
assert SPOT_mode in ['J','X','XS','A','P','M'], 'Unknown SPOT sensor mode: %s' %SPOT_mode
......@@ -143,13 +128,13 @@ def get_sensormode(dataset):
def add_local_availability(dataset):
if job.call_type == 'webapp':
DB_match = INP_R.get_info_from_postgreSQLdb \
DB_match = DB_T.get_info_from_postgreSQLdb \
(job.conn_database,'scenes_proc',['proc_level','layer_bands_assignment'], {'sceneid':dataset['scene_ID']})
else: ## job.call_type == 'console'
DB_match = INP_R.get_info_from_SQLdb(job.path_database,'processed_data',['proc_level','LayerBandsAssignment'],\
DB_match = DB_T.get_info_from_SQLdb(job.path_database,'processed_data',['proc_level','LayerBandsAssignment'],\
{'image_type':dataset['image_type'],'satellite':dataset['satellite'], 'sensor':dataset['sensor'],\
'subsystem':dataset['subsystem'], 'sensormode':dataset['sensormode'], 'entity_ID':dataset['entity_ID']})
path_logfile = HLP_F.path_generator(dataset).get_path_logfile()
path_logfile = PG.path_generator(dataset).get_path_logfile()
def get_HighestProcL_dueLog(path_logfile):
if os.path.exists(path_logfile):
......@@ -181,9 +166,9 @@ def add_local_availability(dataset):
print ('The dataset %s is not included in the database of processed data but according to ' \
'logfile %s has been written successfully. Recreating missing database entry.' \
%(dataset['entity_ID'],HighestProcL_dueLog))
OUT_W.data_DB_updater(GMS_file_dict)
DB_T.data_DB_updater(GMS_file_dict)
if job.call_type == 'console':
OUT_W.data_DB_to_csv()
DB_T.data_DB_to_csv()
dataset['proc_level'] = HighestProcL_dueLog
elif len(DB_match) == 1:
print('Found a matching %s dataset for %s. Processing skipped until %s.' \
......
......@@ -21,6 +21,7 @@
import os, re,sys
import misc.helper_functions as HLP_F
import misc.path_generator as PG
job = GMS_config.job # read from builtins (set by process_controller)
########################### core functions ####################################
......@@ -38,13 +39,13 @@ class L0B_object(object):
self.acquisition_date = data_list_posX['acquisition_date']
self.entity_ID = data_list_posX['entity_ID']
self.scene_ID = data_list_posX['scene_ID']
PG = HLP_F.path_generator(self.__dict__)
self.baseN = PG.get_baseN()
self.path_procdata = PG.get_path_procdata()
self.path_logfile = PG.get_path_logfile()
PathGen = PG.path_generator(self.__dict__)
self.baseN = PathGen.get_baseN()
self.path_procdata = PathGen.get_path_procdata()
self.path_logfile = PathGen.get_path_logfile()
self.logger = HLP_F.setup_logger('log__'+self.baseN, self.path_logfile,self.job_CPUs, append=0)
PG = HLP_F.path_generator(self.__dict__) # passes a logger in addition to previous attributes
self.path_archive = PG.get_local_archive_path_baseN()
PathGen = PG.path_generator(self.__dict__) # passes a logger in addition to previous attributes
self.path_archive = PathGen.get_local_archive_path_baseN()
if not os.path.isfile(self.path_archive) and not os.path.isdir(self.path_archive):
self.logger.info("The %s dataset '%s' has not been processed earlier and no corresponding raw data archive"
......
......@@ -17,8 +17,22 @@
###############################################################################
########################### Library import ####################################
import numpy as np, os, spectral, spectral.io.envi as envi, re, sys, glob, subprocess, shutil, collections, \
gdal, gdalconst, gdalnumeric, zipfile, tarfile
import numpy as np
import os
import spectral
import spectral.io.envi
import re
import sys
import glob
import subprocess
import shutil
import collections
import gdal
import gdalconst
import gdalnumeric
import zipfile
import tarfile
import datetime
from pyhdf import SD
from numba import jit, autojit
......@@ -30,6 +44,7 @@ import gms_io.envifilehandling_BD as ef
import misc.helper_functions as HLP_F
import gms_io.Input_reader as INP_R
import gms_io.Output_writer as OUT_W
import misc.path_generator as PG
job, usecase = GMS_config.job, GMS_config.usecase # read from builtins (set by process_controller)
sys.path.append('./algorithms')
......@@ -70,7 +85,7 @@ class L1A_object(object):
{'image_type': self.image_type, 'Satellite': self.satellite, 'Sensor': self.sensor,
'Subsystem': self.subsystem, 'logger': self.logger})
# self.path_cloud_class_obj = INP_R.get_path_cloud_class_obj(self.GMS_identifier)
self.path_cloud_class_obj = INP_R.get_path_cloud_class_obj(self.GMS_identifier, \
self.path_cloud_class_obj = PG.get_path_cloud_class_obj(self.GMS_identifier, \
get_all=True if job.bench_CLD_class else False)
# self.CLD_obj = CLD_P.GmsCloudClassifier(classifier=self.path_cloud_class_obj)
assert self.path_archive != None and isinstance(self.path_archive, str), \
......@@ -159,13 +174,14 @@ class L1A_object(object):
del self.logger, self.GMS_identifier['logger'], self.MetaObj.logger
def fill_from_disk(self,tuple_GMS_subset):
path_GMS_file = tuple_GMS_subset[0]
GMSfileDict = INP_R.GMSfile2dict(path_GMS_file)
path_GMS_file = tuple_GMS_subset[0]
GMSfileDict = INP_R.GMSfile2dict(path_GMS_file)
for key,value in zip(GMSfileDict.keys(), GMSfileDict.values()):
setattr(self, key, value)
self.arr_shape = tuple_GMS_subset[1][0]
self.arr_pos = tuple_GMS_subset[1][1]
self.logger = HLP_F.setup_logger('log__'+self.baseN, self.path_logfile, self.job_CPUs,append=1)
self.acquisition_date = datetime.datetime.strptime(self.acquisition_date,'%Y-%m-%d')
self.arr_shape = tuple_GMS_subset[1][0]
self.arr_pos = tuple_GMS_subset[1][1]
self.logger = HLP_F.setup_logger('log__'+self.baseN, self.path_logfile, self.job_CPUs,append=1)
# if self.arr_pos is None:
# self.logger.info('Reading file: %s' % (self.baseN))
# else:
......@@ -671,7 +687,7 @@ class L1A_object(object):
self.arr = rasObj.tondarray(direction=3)
self.GMS_identifier['logger'] = temp_logger
if not job.bench_CLD_class:
self.path_cloud_class_obj = INP_R.get_path_cloud_class_obj(self.GMS_identifier)
self.path_cloud_class_obj = PG.get_path_cloud_class_obj(self.GMS_identifier)
CLD_obj = CLD_P.GmsCloudClassifier(classifier=self.path_cloud_class_obj)
if self.MetaObj.bands == CLD_obj.classifier.n_channels:
data = CLD_obj(self)
......@@ -683,7 +699,7 @@ class L1A_object(object):
%(sensorcode,self.MetaObj.bands,CLD_obj.classifier.n_channels))
data = None
else:
pathlist_cloud_class_obj = INP_R.get_path_cloud_class_obj(self.GMS_identifier, get_all=True)
pathlist_cloud_class_obj = PG.get_path_cloud_class_obj(self.GMS_identifier, get_all=True)
import time
# load_time,proc_time,classifier_name = [],[],[]
categories_timinggroup_timing = np.empty((2*len(pathlist_cloud_class_obj),3),dtype='<U7')
......
This diff is collapsed.
......@@ -31,9 +31,11 @@ try: from osgeo import osr
except: import osr
import gms_io.envifilehandling_BD as ef
import algorithms.GEOPROCESSING_BD as GEOP
import misc.helper_functions as HLP_F
from gms_io import envifilehandling_BD as ef
from algorithms import GEOPROCESSING_BD as GEOP
from misc import helper_functions as HLP_F
from misc import database_tools as DB_T
job, usecase = GMS_config.job, GMS_config.usecase # read from builtins (set by process_controller)
# + Input Reader (has to be left out here in order to avoid circular dependencies)
......@@ -508,14 +510,14 @@ class METADATA(object):
self.logger.info('Scene-ID could not be extracted and has to be retrieved from %s metadata database...' %self.Satellite)
import gms_io.Input_reader as INP_R
if job.call_type == 'console':
HLP_F.update_metaDB_if_needed(self.Satellite,self.Sensor,self.Subsystem,self.AcqDate)
DB_T.update_metaDB_if_needed(self.Satellite,self.Sensor,self.Subsystem,self.AcqDate)
tablename = '%s_%s_%s' %(self.Satellite.replace('-',''),self.Sensor.replace('+',''),self.Subsystem) \
if self.Subsystem != '' else '%s_%s' %(self.Satellite.replace('-',''),self.Sensor.replace('+',''))
tablename = tablename if tablename not in ['Landsat4_TM','Landsat5_TM'] else 'Landsat45_TM'
result = INP_R.get_info_from_SQLdb(job.path_db_meta,tablename,['sceneID','sensor'], {'acquisitionDate':self.AcqDate, \
result = DB_T.get_info_from_SQLdb(job.path_db_meta,tablename,['sceneID','sensor'], {'acquisitionDate':self.AcqDate, \
'path':self.WRS_path,'row':self.WRS_row}, records2fetch = 1)
else:
result= INP_R.get_info_from_postgreSQLdb(job.conn_db_meta,'scenes','entityID',{'id':self.SceneID})
result= DB_T.get_info_from_postgreSQLdb(job.conn_db_meta,'scenes','entityID',{'id':self.SceneID})
if len(result) == 1: # e.g. [('LE71950282003121EDC00',)]
self.EntityID = result[0][0]
......
......@@ -29,11 +29,14 @@ import dill
import psycopg2
from io import StringIO as StringIO
import scipy.interpolate
import misc.helper_functions as HLP_F
#import misc.helper_functions as HLP_F
import gms_io.Output_writer as OUT_W
import algorithms.GEOPROCESSING_BD as GEOP
import algorithms.METADATA_BD as META
import misc.database_tools as DB_T
import misc.path_generator as PG
job = GMS_config.job # read from builtins (set by process_controller)
# + misc.helper_functions.setup_logger (left out here in order to avoid circular dependencies)
########################### core functions ####################################
class out_object(object):
......@@ -49,7 +52,8 @@ class out_object(object):
print('Reading file: %s' % (self.baseN))
else:
print('Reading file: %s @ position %s' %(self.baseN, self.arr_pos))
self.logger = HLP_F.setup_logger('log__'+self.baseN, self.path_logfile, self.job_CPUs,append=1)
from misc.helper_functions import setup_logger
self.logger = setup_logger('log__'+self.baseN, self.path_logfile, self.job_CPUs,append=1)
self.GMS_identifier = collections.OrderedDict({'image_type':self.image_type, 'Satellite':self.satellite,'Sensor':self.sensor,'Subsystem': self.subsystem, 'logger':self.logger})
self.meta = read_ENVIhdr_to_dict(os.path.splitext(self.path_Outfile_L1A)[0]+'.hdr', self.logger)
......@@ -145,36 +149,6 @@ def unify_envi_header_keys(header_dict):
del unified_header_dict[key]
return unified_header_dict
def get_info_from_SQLdb(path_db,tablename,vals2return,cond_dict,records2fetch=0):
if not isinstance(vals2return,list): vals2return = [vals2return]
assert isinstance(records2fetch,int), \
"get_info_from_SQLdb: Expected an integer for the argument 'records2return'. Got %s" %type(records2fetch)
if not os.path.isfile(path_db): return 'database connection fault'
connection = sqlite3.connect(path_db)
cursor = connection.cursor()
condition = "WHERE " + " AND ".join(["%s=?" %(list(cond_dict.keys())[i]) for i in range(len(cond_dict))])
cursor.execute("SELECT " +','.join(vals2return)+ " FROM " +tablename+ " " + condition, list(cond_dict.values()))
records2return = cursor.fetchall() if records2fetch == 0 else [cursor.fetchone()] if records2fetch == 1 else \
cursor.fetchmany(size = records2fetch) # e.g. [('LE71950282003121EDC00',), ('LE71950282003105ASN00',)]
cursor.close()
connection.close()
return records2return
def get_info_from_postgreSQLdb(conn_params,tablename,vals2return,cond_dict,records2fetch=0):
if not isinstance(vals2return,list): vals2return = [vals2return]
assert isinstance(records2fetch,int), \
"get_info_from_postgreSQLdb: Expected an integer for the argument 'records2return'. Got %s" %type(records2fetch)
connection = psycopg2.connect(conn_params)
if connection is None: return 'database connection fault'
cursor = connection.cursor()
condition = "WHERE " + " AND ".join(["%s=%s" %(k,v) for k,v in cond_dict.items()])
cursor.execute("SELECT " +','.join(vals2return)+ " FROM " +tablename+ " " + condition)
records2return = cursor.fetchall() if records2fetch == 0 else [cursor.fetchone()] if records2fetch == 1 else \
cursor.fetchmany(size = records2fetch) # e.g. [('LE71950282003121EDC00',), ('LE71950282003105ASN00',)]
cursor.close()
connection.close()
return records2return
def get_list_GMSfiles(dataset_list__target__tuple):
"""Returns a list of absolute paths linking to gms-files of truely written datasets that fullfill certain criteria.
Input:
......@@ -187,16 +161,16 @@ def get_list_GMSfiles(dataset_list__target__tuple):
if GMS_call_type == 'webapp':
GMS_list = []
for dataset in dataset_list:
path_gms_file = HLP_F.path_generator(dataset).get_path_gmsfile()
path_gms_file = PG.path_generator(dataset).get_path_gmsfile()
if os.path.exists(path_gms_file):
GMS_list.append(path_gms_file)
else: # GMS_call_type == 'console'
returned_tuples=[]
for dataset in dataset_list:
# returned_tuples = get_info_from_SQLdb(job.path_database,'processed_data',['path_procdata','baseN'],{'job_ID':job_ID,'image_type':'RSD','georef':'Slave','proc_level':'L1A'})
returned_tuples = returned_tuples + get_info_from_SQLdb(job.path_database,'processed_data',['path_procdata','baseN'],{'image_type':dataset['image_type'],'entity_ID':dataset['entity_ID'],'subsystem':dataset['subsystem'],'proc_level':target})
# returned_tuples = returned_tuples + get_info_from_SQLdb(job.path_database,'processed_data',['path_procdata','baseN'],{'image_type':dataset['image_type'],'entity_ID':'LE71510322000093SGS00','subsystem':dataset['subsystem'],'proc_level':target})
# returned_tuples = returned_tuples + get_info_from_SQLdb(job.path_database,'processed_data',['path_procdata','baseN'],{'image_type':dataset['image_type'],'entity_ID':'LC81510322013184LGN00','subsystem':dataset['subsystem'],'proc_level':target})
# returned_tuples = DB_T.get_info_from_SQLdb(job.path_database,'processed_data',['path_procdata','baseN'],{'job_ID':job_ID,'image_type':'RSD','georef':'Slave','proc_level':'L1A'})
returned_tuples = returned_tuples + DB_T.get_info_from_SQLdb(job.path_database,'processed_data',['path_procdata','baseN'],{'image_type':dataset['image_type'],'entity_ID':dataset['entity_ID'],'subsystem':dataset['subsystem'],'proc_level':target})
# returned_tuples = returned_tuples + DB_T.get_info_from_SQLdb(job.path_database,'processed_data',['path_procdata','baseN'],{'image_type':dataset['image_type'],'entity_ID':'LE71510322000093SGS00','subsystem':dataset['subsystem'],'proc_level':target})
# returned_tuples = returned_tuples + DB_T.get_info_from_SQLdb(job.path_database,'processed_data',['path_procdata','baseN'],{'image_type':dataset['image_type'],'entity_ID':'LC81510322013184LGN00','subsystem':dataset['subsystem'],'proc_level':target})
GMS_list = [os.path.join(returned_tuples[i][0],returned_tuples[i][1]+'_%s.gms' %target) for i in range(len(returned_tuples)) if os.path.isfile(os.path.join(returned_tuples[i][0],returned_tuples[i][1]+'_%s.gms' %target))]
return GMS_list
......@@ -225,9 +199,10 @@ def SRF_reader(GMS_identifier):
return SRF_dict
def pickle_SRF_DB(L1A_Instances):
from misc.helper_functions import setup_logger
list_GMS_identifiers = [i.GMS_identifier for i in L1A_Instances]
out_dict = collections.OrderedDict()
logger = HLP_F.setup_logger('log__SRF2PKL', os.path.join(job.path_testing,'out/log__SRF2PKL.log'),1,append=0)
logger = setup_logger('log__SRF2PKL', os.path.join(job.path_testing,'out/log__SRF2PKL.log'),1,append=0)
for Id,Inst in zip(list_GMS_identifiers,L1A_Instances):
Id['logger'] = logger
out_dict[Inst.satellite+'_'+Inst.sensor+(('_'+Inst.subsystem) if Inst.subsystem not in ['',None] else '')] = SRF_reader(Id)
......@@ -247,58 +222,3 @@ def Solar_Irradiance_reader(resol_nm = None, wvl_min_nm = None, wvl_max_nm = Non
wvl_rsp = np.arange(wvl_min,wvl_max,resol_nm)
sol_irr = scipy.interpolate.interp1d(sol_irr[:,0],sol_irr[:,1],kind='linear')(wvl_rsp)
return sol_irr
def get_path_cloud_class_obj(GMS_identifier, get_all=False):
"""Returns the absolute path of the the training data used by cloud classifier."""
GMS_sensorcode = HLP_F.get_GMS_sensorcode(GMS_identifier)
satellite,sensor,logger = (GMS_identifier['Satellite'],GMS_identifier['Sensor'],GMS_identifier['logger'])
path_cloud_classifier_objects = os.path.join(job.path_cloud_classif,satellite,sensor)
obj_name_dic = {
'AVNIR-2': None,
'TM4' : None,
# 'TM5' : '9fe4af95-a16e-45e6-9397-7085c74a30d8.dill', # 16.9. class. bayesian
# 'TM5' : 'xval_0.96_classicalBayesian_c723159c.dill', # 28.9. class. bayesian
'TM5' : 'xval_0.96_classicalBayesian_b84d087e.dill', # 28.9. fastest classifier
# 'TM7' : '1c8560d9-8436-43e7-b170-416c15e732a7.dill', # ganz einfach
# 'TM7' : '38bc1428-2775-4d0c-a551-dcea99ff9046.dill',
# 'TM7' : '9fe4af95-a16e-45e6-9397-7085c74a30d8.dill', # 16.9. class. bayesian
# 'TM7' : 'xval_0.96_classicalBayesian_c723159c.dill', # 28.9. class. bayesian
'TM7' : 'xval_0.96_classicalBayesian_b84d087e.dill', # 28.9. fastest classifier
# 'LDCM' : 'xval_0.97_classicalBayesian_3ac27853.dill', # 28.9. class. bayesian
'LDCM' : 'xval_0.97_classicalBayesian_305e7da1.dill', # 28.9. fastest class. bayesian
# 'LDCM' : glob.glob(os.path.join(path_cloud_classifier_objects,'*.dill'))[0] if glob.glob(os.path.join(path_cloud_classifier_objects,'*.dill')) != [] else None,
'S1a' : None,
'S1b' : None,
'S2a' : None,
'S2b' : None,
'S3a' : None,
'S3b' : None,
'S4a' : None,
'S4b' : None,
'S5a' : None,
'S5b' : None,
'RE5' : None,
'AST_V1' : None,
'AST_V2' : None,
'AST_S' : None,
'AST_T' : None }
if get_all: #returns a list
classifier_names = glob.glob(os.path.join(path_cloud_classifier_objects,'*.dill')) if glob.glob(os.path.join(path_cloud_classifier_objects,'*.dill')) != [] else None
classifier_path = [os.path.join(path_cloud_classifier_objects,str(i)) for i in classifier_names]
else:
classifier_path = os.path.join(path_cloud_classifier_objects,str(obj_name_dic[GMS_sensorcode]))
try:
path4pathcheck = classifier_path if isinstance(classifier_path,str) else \
classifier_path[0] if isinstance(classifier_path,list) else None
if os.path.isfile(os.path.join(path_cloud_classifier_objects,path4pathcheck)):
return classifier_path
else:
logger.warning('Cloud masking is not yet implemented for %s %s.' % (satellite,sensor))
return None
except KeyError:
logger.warning("Sensorcode '%s' is not included in sensorcode dictionary and can not be converted into GMS \
sensorcode." %(meta_sensorcode))
return None
except: raise
......@@ -18,7 +18,6 @@ import pip
import numpy as np
import os
import sys
import sqlite3
import csv
import json
import collections
......@@ -28,9 +27,11 @@ import inspect
import errno
import dill
import datetime
import psycopg2
from misc.helper_functions import get_mask_classdefinition, get_mask_colormap, setup_logger
import misc.helper_functions as HLP_F
import misc.database_tools as DB_T
import misc.path_generator as PG
job, usecase = GMS_config.job, GMS_config.usecase # read from builtins (set by process_controller)
......@@ -230,8 +231,8 @@ def Obj2ENVI(InObj, write_masks_as_ENVI_classification = True):
# write GMS-file
# IMPORTANT: DO NOT pass the complete object but only a copy of the dictionary in order to prevent ASCII_writer and
# data_DB_updater from modifying the attributes of the object!!
ASCII_writer(InObj.__dict__.copy(), HLP_F.path_generator(InObj.__dict__.copy()).get_path_gmsfile(),InObj.logger)
data_DB_updater(InObj.__dict__.copy())
ASCII_writer(InObj.__dict__.copy(), PG.path_generator(InObj.__dict__.copy()).get_path_gmsfile(),InObj.logger)
DB_T.data_DB_updater(InObj.__dict__.copy())
InObj.logger.info('%s data successfully saved.' %InObj.proc_level)
del InObj.logger
......@@ -241,88 +242,6 @@ def add_attributes_to_ENVIhdr(attr2add_dict, hdr_path, logger=None):
attr_dict.update(attr2add_dict)
HDR_writer(attr_dict, hdr_path)
def data_DB_updater(obj_dict):
assert isinstance(obj_dict,dict), 'The input for data_DB_updater() has to be a dictionary.'
list2str = lambda list2convert: ''.join([str(val) for val in list2convert])
if job.call_type == 'console':
if not os.path.isfile(job.path_database):
print('No internal database found. Creating a new one...')
connection = sqlite3.connect(job.path_database)
cursor = connection.cursor()
fullColumnList = ['job_ID','job_CPUs','image_type','satellite', 'sensor','subsystem','sensormode',\
'acquisition_date', 'entity_ID', 'georef', 'proc_level', 'LayerBandsAssignment', 'path_procdata']
cursor.execute('''CREATE TABLE IF NOT EXISTS processed_data (%s)''' %(', ').join(fullColumnList))
currentColumnList = [i[1] for i in cursor.execute("PRAGMA table_info('processed_data')").fetchall()]
missingColumns = [col for col in fullColumnList if col not in currentColumnList]
if missingColumns != []: # automatic adding of missing columns
cursor.execute('''CREATE TABLE IF NOT EXISTS processed_data_temp (%s)''' %(', ').join(fullColumnList))
cursor.execute("SELECT "+','.join(currentColumnList)+" FROM processed_data")
[cursor.execute("INSERT INTO processed_data_temp (%(cols)s) VALUES (%(vals)s)" %{'cols':','.join(\
currentColumnList),'vals':','.join(['?']*len(currentColumnList))}, row) for row in cursor.fetchall()]
cursor.execute("DROP TABLE processed_data")
cursor.execute("ALTER TABLE processed_data_temp RENAME TO processed_data")
cursor.execute("SELECT EXISTS(SELECT 1 FROM processed_data WHERE entity_ID=? AND sensor=? AND subsystem=?)", \
[obj_dict['entity_ID'],obj_dict['sensor'],obj_dict['subsystem']])
if cursor.fetchone()[0] == 0: # create new entry
new_record = [obj_dict[key] for key in fullColumnList]
new_record = [(''.join([str(val[li]) for li in range(len(val))])) if isinstance(val,list) else val \
for val in new_record] # e.g. converts list of LayerBandsAssignment to string
cursor.execute("INSERT INTO processed_data VALUES (%s)" %','.join(['?']*len(new_record)), new_record)
else: # udate existing entry
values2update = [obj_dict[key] for key in ['job_ID','job_CPUs','proc_level','path_procdata','LayerBandsAssignment']]
values2update = [(''.join([