Commit 75a3cf12 authored by Daniel Scheffler's avatar Daniel Scheffler Committed by Mathias Peters
Browse files

Added code to check proper activation of GDAL. Removed hardcoded database host...

Added code to check proper activation of GDAL. Removed hardcoded database host from tests. Bugfix for test_spectral_resampler.
Former-commit-id: 4fda9e47
Former-commit-id: 8547efe6
parent d7a11d3b
......@@ -310,17 +310,6 @@ class GEOPROCESSING(object):
if dst_CS == 'LonLat' and dst_CS_datum == 'WGS84':
dst_EPSG_code = 4326
if 'anaconda' in sys.executable: # GDAL environment check # TODO move to config
if 'GDAL_DATA' not in os.environ:
# Prevents "Unable to open EPSG support file gcs.csv".
self.logger.warning("GDAL_DATA variable not set. Setting it to '%s'."
% sys.executable.split('anaconda')[0] + 'anaconda/share/gdal')
anaconda_path = re.search('([\w+\S\s]*anaconda[\w+\S\s]*?)/', sys.executable, re.I).group(1)
os.environ['GDAL_DATA'] = os.path.join(anaconda_path, 'share/gdal')
if not re.search('/anaconda[\w+\S\s]*?/share/gdal', os.environ['GDAL_DATA'], re.I):
self.logger.warning('GDAL_DATA variable seems to be incorrectly set since Anaconda´s Python is '
'executed and GDAL_DATA = %s' % os.environ['GDAL_DATA'])
assert mode in ['rasterio', 'GDAL'], "The 'mode' argument must be set to 'rasterio' or 'GDAL'."
if mode == 'rasterio':
......
......@@ -1298,7 +1298,7 @@ def import_shapefile_into_postgreSQL_database(path_shp, tablename, cols2import=N
# import geoalchemy2
# dtype_dic.update({'geom': geoalchemy2.types.Geometry(geometry_type='POLYGON',srid=4326)})
print('Adding shapefile geometries to postgreSQL table %s...' % tablename)
engine = create_engine('postgresql://gmsdb:gmsdb@geoms/geomultisens') # FIXME hardcoded
engine = create_engine('postgresql://gmsdb:gmsdb@%s/geomultisens' % CFG.db_host)
pdDataFrame_to_sql_k(engine, subGDF, tablename, index_label=index_label,
keys=primarykey, if_exists=if_exists, dtype=dtype_dic)
# set SRID
......
# -*- coding: utf-8 -*-
import os
import sys
import subprocess
import warnings
try:
from osgeo import gdal
......@@ -106,3 +109,26 @@ class GMSEnvironment(object):
if not os.path.exists(CFG.path_archive):
raise GMSEnvironmentError("The given provider archive path '%s' does not exist!" % CFG.path_archive)
@staticmethod
def ensure_properly_activated_GDAL():
if 'conda' in sys.version:
conda_rootdir = subprocess.check_output('conda info --root', shell=True).decode('utf-8')[:-1]
gdal_data_dir = os.path.join(conda_rootdir, 'share', 'gdal')
# GDAL_DATA
if 'GDAL_DATA' not in os.environ:
# Prevents "Unable to open EPSG support file gcs.csv".
warnings.warn("GDAL_DATA variable not set. Setting it to '%s'." % gdal_data_dir)
os.environ['GDAL_DATA'] = gdal_data_dir
if os.path.abspath(os.environ['GDAL_DATA']) != gdal_data_dir:
warnings.warn("GDAL_DATA variable seems to be incorrectly set since Anaconda´s Python is "
"executed from %s and GDAL_DATA = %s. Setting it to '%s'."
% (conda_rootdir, os.environ['GDAL_DATA'], gdal_data_dir))
os.environ['GDAL_DATA'] = gdal_data_dir
# CPL_ZIP_ENCODING
if 'CPL_ZIP_ENCODING' not in os.environ:
# Prevents "Warning 1: Recode from CP437 to UTF-8 failed with the error: "Invalid argument"."
# during gdal.Open().
os.environ['CPL_ZIP_ENCODING'] = 'UTF-8'
......@@ -14,16 +14,18 @@ from ..misc.exceptions import GMSEnvironmentError
class SpatialIndexMediatorServer:
controller = 'index-mediator-server.sh'
def __init__(self, rootDir, logger=Logger(__name__)):
self.rootDir = rootDir
self.path_idxMedSrv = os.path.join(rootDir, self.controller)
self.path_idxMedSrv = os.path.join(rootDir, 'index-mediator-server.sh')
self.logger = logger
# validate
if not os.path.isfile(self.path_idxMedSrv):
raise GMSEnvironmentError('File path of index mediator server does not exist at %s.' % self.path_idxMedSrv)
self.path_idxMedSrv = os.path.join(rootDir, 'index-mediator-server')
if not os.path.isfile(self.path_idxMedSrv):
raise GMSEnvironmentError('File path of index mediator server does not exist at %s.'
% self.path_idxMedSrv)
@property
def is_running(self):
......@@ -110,7 +112,7 @@ class SpatialIndexMediator:
"""
Establishes a connection to the spatial index mediator server.
:param host: host address of the index mediator server (default "geoms.gfz-potsdam.de")
:param host: host address of the index mediator server (default "localhost")
:param port: port number of the index mediator server (default 8654)
:param timeout timeout as float in seconds (default 5.0 sec)
"""
......
......@@ -97,6 +97,9 @@ def set_config(job_ID, json_config='', exec_mode='Python', parallelization_level
:param virtual_sensor_id: 1: Landsat-8, 10: Sentinel-2A 10m
:param datasetid_spatial_ref: 249 Sentinel-2A
"""
#################################
# set GMS_JobConfig in builtins #
#################################
# FIXME virtual_sensor_id and datasetid_spatial_ref are not respected by JobConfig.
if not hasattr(builtins, 'GMS_JobConfig') or reset_status:
kwargs = dict([x for x in locals().items() if x[0] != "self" and not x[0].startswith('__')])
......
......@@ -75,6 +75,7 @@ class process_controller(object):
self.GMSEnv = ENV.GMSEnvironment(self.logger)
self.GMSEnv.check_dependencies()
self.GMSEnv.check_read_write_permissions()
self.GMSEnv.ensure_properly_activated_GDAL()
# check if process_controller is executed by debugger
# isdebugging = 1 if True in [frame[1].endswith("pydevd.py") for frame in inspect.stack()] else False
......
# -*- coding: utf-8 -*-
"""Unit test package for testproject."""
import os
# set database host during tests
db_host = 'localhost' if 'GMS_db_host' not in os.environ else os.environ['GMS_db_host']
......@@ -17,6 +17,8 @@ import warnings
from gms_preprocessing import __path__
from gms_preprocessing.misc.database_tools import delete_record_in_postgreSQLdb
from . import db_host
path_run_gms = os.path.abspath(os.path.join(__path__[0], '..', 'bin', 'run_gms.py'))
......@@ -49,9 +51,9 @@ class Base_CLITester:
def test_hostname_custom(self):
parsed_args = self.parser_run.parse_args(self.baseargs +
['--db_host', 'geoms'])
['--db_host', db_host])
parsed_args.func(parsed_args)
self.assertEqual(self.current_CFG.db_host, 'geoms')
self.assertEqual(self.current_CFG.db_host, db_host)
class Test_run_jobid(Base_CLITester.Base_CLITestCase):
......
......@@ -18,6 +18,8 @@ from gms_preprocessing.options.config import path_options_default
from gms_preprocessing.options.config import GMSValidator
from gms_preprocessing.options.options_schema import gms_schema_config_output
from . import db_host
class Test_get_options(TestCase):
def test_target_is_file_no_validation(self):
......@@ -32,7 +34,7 @@ class Test_get_options(TestCase):
class Test_JobConfig(TestCase):
def setUp(self):
self.jobID = 26186662
self.db_host = 'geoms'
self.db_host = db_host
def test_plain_args(self):
cfg = JobConfig(self.jobID, db_host=self.db_host)
......
......@@ -15,6 +15,8 @@ from gms_preprocessing.misc.exception_handler import log_uncaught_exceptions
from gms_preprocessing.algorithms.L1A_P import L1A_object
from gms_preprocessing.misc.database_tools import get_info_from_postgreSQLdb
from . import db_host
class BaseTest_ExceptionHandler:
"""Test class for gms_preprocessing.misc.exception_handler.ExceptionHandler."""
......@@ -22,7 +24,7 @@ class BaseTest_ExceptionHandler:
PC = None # default
def get_process_controller(self, jobID):
self.PC = process_controller(jobID, parallelization_level='scenes', db_host='geoms',
self.PC = process_controller(jobID, parallelization_level='scenes', db_host=db_host,
is_test=True, log_level='DEBUG')
self.PC.config.disable_exception_handler = False
......
......@@ -53,15 +53,16 @@ from gms_preprocessing.algorithms.L2B_P import L2B_object
from gms_preprocessing.algorithms.L2C_P import L2C_object
from gms_preprocessing.misc.database_tools import get_info_from_postgreSQLdb
from . import db_host
__author__ = 'Daniel Scheffler' # edited by Jessica Palka.
# Rootpath of the gms_preprocessing-repository.
gmsRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
# Defining the configurations needed to start a job containing the different dataset scenes.
# TODO Change the job-configurations for selected datasets.
job_config_kwargs = dict(parallelization_level='scenes', db_host='geoms', delete_old_output=True, is_test=True)
job_config_kwargs = dict(parallelization_level='scenes', db_host=db_host, delete_old_output=True, is_test=True)
##########################
......@@ -299,7 +300,7 @@ class Test_in_normal_mode(unittest.TestCase):
# self.job_id = 26186906 # Bug Input Validator
self.job_id = 26186925 # 1 Sentinel-2A, Bug NoneType' object has no attribute 'find'
self.PC = process_controller(self.job_id, **dict(is_test=False, parallelization_level='scenes', db_host='geoms',
self.PC = process_controller(self.job_id, **dict(is_test=False, parallelization_level='scenes', db_host=db_host,
delete_old_output=True, disable_exception_handler=True))
def test(self):
......
......@@ -16,6 +16,7 @@ from gms_preprocessing.io.input_reader import DEM_Creator
from gms_preprocessing import set_config
from py_tools_ds.geo.projection import EPSG2WKT, WKT2EPSG
from .import db_host
gmsRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
......@@ -23,7 +24,7 @@ gmsRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')
class Test_DEM_Creator(unittest.TestCase):
def setUp(self):
# Testjob Landsat-8
set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset_status=True, is_test=True,
set_config(exec_mode='Python', job_ID=26186196, db_host=db_host, reset_status=True, is_test=True,
path_archive=os.path.join(gmsRepo_rootpath, 'tests', 'data', 'archive_data'))
self.boxMapXY = ((277365.0, 5546625.0), (292365.0, 5546625.0), (292365.0, 5531625.0), (277365.0, 5531625.0))
......
......@@ -19,6 +19,7 @@ from gms_preprocessing import __file__ # noqa E402 module level import not at t
from gms_preprocessing.options.config import set_config # noqa E402 module level import not at top of file
from gms_preprocessing.algorithms.L2B_P import KMeansRSImage # noqa E402 module level import not at top of file
from . import db_host
testdata = os.path.join(os.path.dirname(__file__),
'../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq')
......@@ -30,7 +31,7 @@ class Test_KMeansRSImage(unittest.TestCase):
@classmethod
def setUpClass(cls):
# Testjob Landsat-8
set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset_status=True)
set_config(exec_mode='Python', job_ID=26186196, db_host=db_host, reset_status=True)
cls.geoArr = GeoArray(testdata)
cls.geoArr.to_mem()
cls.kmeans = KMeansRSImage(cls.geoArr, n_clusters=10)
......
......@@ -21,6 +21,7 @@ from gms_preprocessing.algorithms.L2B_P import ReferenceCube_Generator # noqa E
from gms_preprocessing.algorithms.L2B_P import RefCube # noqa E402 module level import not at top of file
from gms_preprocessing.algorithms.L2B_P import SpectralHomogenizer # noqa E402 module level import not at top of file
from . import db_host
testdata = os.path.join(__path__[0], '../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq')
......@@ -31,7 +32,7 @@ testdata = os.path.join(__path__[0], '../tests/data/hy_spec_data/Bavaria_farmlan
# @classmethod
# def setUpClass(cls):
# # Testjob Landsat-8
# set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset_status=True, is_test=True)
# set_config(exec_mode='Python', job_ID=26186196, db_host=db_host, reset_status=True, is_test=True)
# cls.SHC = ReferenceCube_Generator_OLD([testdata, testdata, ], v=False)
#
# def test_generate_reference_cube_L8(self):
......@@ -66,7 +67,7 @@ class Test_ReferenceCube_Generator(unittest.TestCase):
@classmethod
def setUpClass(cls):
# Testjob Landsat-8
set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset_status=True, is_test=True)
set_config(exec_mode='Python', job_ID=26186196, db_host=db_host, reset_status=True, is_test=True)
cls.tmpOutdir = tempfile.TemporaryDirectory()
cls.testIms = [testdata, testdata, ]
cls.tgt_sat_sen_list = [
......@@ -135,7 +136,7 @@ class Test_SpectralHomogenizer(unittest.TestCase):
@classmethod
def setUpClass(cls):
# Testjob Landsat-8
cfg = set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset_status=True, is_test=True)
cfg = set_config(exec_mode='Python', job_ID=26186196, db_host=db_host, reset_status=True, is_test=True)
cls.SpH = SpectralHomogenizer(classifier_rootDir=cfg.path_spechomo_classif)
cls.testArr_L8 = GeoArray(np.random.randint(1, 10000, (50, 50, 7), dtype=np.int16)) # no band 9, no pan
......
......@@ -19,6 +19,8 @@ from gms_preprocessing.io.input_reader import SRF
from gms_preprocessing.options.config import set_config
from gms_preprocessing.algorithms.L2B_P import SpectralResampler as SR
from . import db_host
testdata = os.path.join(os.path.dirname(__file__),
'../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq')
......@@ -29,7 +31,7 @@ class Test_SpectralResampler(unittest.TestCase):
@classmethod
def setUpClass(cls):
# Testjob Landsat-8
set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset_status=True)
set_config(exec_mode='Python', job_ID=26186196, db_host=db_host, reset_status=True, is_test=True)
cls.geoArr = GeoArray(testdata)
cls.geoArr.to_mem()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment