Commit 4fda9e47 authored by Daniel Scheffler's avatar Daniel Scheffler Committed by Mathias Peters
Browse files

Added code to check proper activation of GDAL. Removed hardcoded database host...

Added code to check proper activation of GDAL. Removed hardcoded database host from tests. Bugfix for test_spectral_resampler.
parent ba135fdd
Pipeline #1768 failed with stage
in 5 minutes and 18 seconds
...@@ -310,17 +310,6 @@ class GEOPROCESSING(object): ...@@ -310,17 +310,6 @@ class GEOPROCESSING(object):
if dst_CS == 'LonLat' and dst_CS_datum == 'WGS84': if dst_CS == 'LonLat' and dst_CS_datum == 'WGS84':
dst_EPSG_code = 4326 dst_EPSG_code = 4326
if 'anaconda' in sys.executable: # GDAL environment check # TODO move to config
if 'GDAL_DATA' not in os.environ:
# Prevents "Unable to open EPSG support file gcs.csv".
self.logger.warning("GDAL_DATA variable not set. Setting it to '%s'."
% sys.executable.split('anaconda')[0] + 'anaconda/share/gdal')
anaconda_path = re.search('([\w+\S\s]*anaconda[\w+\S\s]*?)/', sys.executable, re.I).group(1)
os.environ['GDAL_DATA'] = os.path.join(anaconda_path, 'share/gdal')
if not re.search('/anaconda[\w+\S\s]*?/share/gdal', os.environ['GDAL_DATA'], re.I):
self.logger.warning('GDAL_DATA variable seems to be incorrectly set since Anaconda´s Python is '
'executed and GDAL_DATA = %s' % os.environ['GDAL_DATA'])
assert mode in ['rasterio', 'GDAL'], "The 'mode' argument must be set to 'rasterio' or 'GDAL'." assert mode in ['rasterio', 'GDAL'], "The 'mode' argument must be set to 'rasterio' or 'GDAL'."
if mode == 'rasterio': if mode == 'rasterio':
......
...@@ -1298,7 +1298,7 @@ def import_shapefile_into_postgreSQL_database(path_shp, tablename, cols2import=N ...@@ -1298,7 +1298,7 @@ def import_shapefile_into_postgreSQL_database(path_shp, tablename, cols2import=N
# import geoalchemy2 # import geoalchemy2
# dtype_dic.update({'geom': geoalchemy2.types.Geometry(geometry_type='POLYGON',srid=4326)}) # dtype_dic.update({'geom': geoalchemy2.types.Geometry(geometry_type='POLYGON',srid=4326)})
print('Adding shapefile geometries to postgreSQL table %s...' % tablename) print('Adding shapefile geometries to postgreSQL table %s...' % tablename)
engine = create_engine('postgresql://gmsdb:gmsdb@geoms/geomultisens') # FIXME hardcoded engine = create_engine('postgresql://gmsdb:gmsdb@%s/geomultisens' % CFG.db_host)
pdDataFrame_to_sql_k(engine, subGDF, tablename, index_label=index_label, pdDataFrame_to_sql_k(engine, subGDF, tablename, index_label=index_label,
keys=primarykey, if_exists=if_exists, dtype=dtype_dic) keys=primarykey, if_exists=if_exists, dtype=dtype_dic)
# set SRID # set SRID
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os import os
import sys
import subprocess
import warnings
try: try:
from osgeo import gdal from osgeo import gdal
...@@ -106,3 +109,26 @@ class GMSEnvironment(object): ...@@ -106,3 +109,26 @@ class GMSEnvironment(object):
if not os.path.exists(CFG.path_archive): if not os.path.exists(CFG.path_archive):
raise GMSEnvironmentError("The given provider archive path '%s' does not exist!" % CFG.path_archive) raise GMSEnvironmentError("The given provider archive path '%s' does not exist!" % CFG.path_archive)
@staticmethod
def ensure_properly_activated_GDAL():
if 'conda' in sys.version:
conda_rootdir = subprocess.check_output('conda info --root', shell=True).decode('utf-8')[:-1]
gdal_data_dir = os.path.join(conda_rootdir, 'share', 'gdal')
# GDAL_DATA
if 'GDAL_DATA' not in os.environ:
# Prevents "Unable to open EPSG support file gcs.csv".
warnings.warn("GDAL_DATA variable not set. Setting it to '%s'." % gdal_data_dir)
os.environ['GDAL_DATA'] = gdal_data_dir
if os.path.abspath(os.environ['GDAL_DATA']) != gdal_data_dir:
warnings.warn("GDAL_DATA variable seems to be incorrectly set since Anaconda´s Python is "
"executed from %s and GDAL_DATA = %s. Setting it to '%s'."
% (conda_rootdir, os.environ['GDAL_DATA'], gdal_data_dir))
os.environ['GDAL_DATA'] = gdal_data_dir
# CPL_ZIP_ENCODING
if 'CPL_ZIP_ENCODING' not in os.environ:
# Prevents "Warning 1: Recode from CP437 to UTF-8 failed with the error: "Invalid argument"."
# during gdal.Open().
os.environ['CPL_ZIP_ENCODING'] = 'UTF-8'
...@@ -14,16 +14,18 @@ from ..misc.exceptions import GMSEnvironmentError ...@@ -14,16 +14,18 @@ from ..misc.exceptions import GMSEnvironmentError
class SpatialIndexMediatorServer: class SpatialIndexMediatorServer:
controller = 'index-mediator-server.sh'
def __init__(self, rootDir, logger=Logger(__name__)): def __init__(self, rootDir, logger=Logger(__name__)):
self.rootDir = rootDir self.rootDir = rootDir
self.path_idxMedSrv = os.path.join(rootDir, self.controller) self.path_idxMedSrv = os.path.join(rootDir, 'index-mediator-server.sh')
self.logger = logger self.logger = logger
# validate # validate
if not os.path.isfile(self.path_idxMedSrv): if not os.path.isfile(self.path_idxMedSrv):
raise GMSEnvironmentError('File path of index mediator server does not exist at %s.' % self.path_idxMedSrv) self.path_idxMedSrv = os.path.join(rootDir, 'index-mediator-server')
if not os.path.isfile(self.path_idxMedSrv):
raise GMSEnvironmentError('File path of index mediator server does not exist at %s.'
% self.path_idxMedSrv)
@property @property
def is_running(self): def is_running(self):
...@@ -110,7 +112,7 @@ class SpatialIndexMediator: ...@@ -110,7 +112,7 @@ class SpatialIndexMediator:
""" """
Establishes a connection to the spatial index mediator server. Establishes a connection to the spatial index mediator server.
:param host: host address of the index mediator server (default "geoms.gfz-potsdam.de") :param host: host address of the index mediator server (default "localhost")
:param port: port number of the index mediator server (default 8654) :param port: port number of the index mediator server (default 8654)
:param timeout timeout as float in seconds (default 5.0 sec) :param timeout timeout as float in seconds (default 5.0 sec)
""" """
......
...@@ -97,6 +97,9 @@ def set_config(job_ID, json_config='', exec_mode='Python', parallelization_level ...@@ -97,6 +97,9 @@ def set_config(job_ID, json_config='', exec_mode='Python', parallelization_level
:param virtual_sensor_id: 1: Landsat-8, 10: Sentinel-2A 10m :param virtual_sensor_id: 1: Landsat-8, 10: Sentinel-2A 10m
:param datasetid_spatial_ref: 249 Sentinel-2A :param datasetid_spatial_ref: 249 Sentinel-2A
""" """
#################################
# set GMS_JobConfig in builtins #
#################################
# FIXME virtual_sensor_id and datasetid_spatial_ref are not respected by JobConfig. # FIXME virtual_sensor_id and datasetid_spatial_ref are not respected by JobConfig.
if not hasattr(builtins, 'GMS_JobConfig') or reset_status: if not hasattr(builtins, 'GMS_JobConfig') or reset_status:
kwargs = dict([x for x in locals().items() if x[0] != "self" and not x[0].startswith('__')]) kwargs = dict([x for x in locals().items() if x[0] != "self" and not x[0].startswith('__')])
......
...@@ -75,6 +75,7 @@ class process_controller(object): ...@@ -75,6 +75,7 @@ class process_controller(object):
self.GMSEnv = ENV.GMSEnvironment(self.logger) self.GMSEnv = ENV.GMSEnvironment(self.logger)
self.GMSEnv.check_dependencies() self.GMSEnv.check_dependencies()
self.GMSEnv.check_read_write_permissions() self.GMSEnv.check_read_write_permissions()
self.GMSEnv.ensure_properly_activated_GDAL()
# check if process_controller is executed by debugger # check if process_controller is executed by debugger
# isdebugging = 1 if True in [frame[1].endswith("pydevd.py") for frame in inspect.stack()] else False # isdebugging = 1 if True in [frame[1].endswith("pydevd.py") for frame in inspect.stack()] else False
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""Unit test package for testproject.""" """Unit test package for testproject."""
import os
# set database host during tests
db_host = 'localhost' if 'GMS_db_host' not in os.environ else os.environ['GMS_db_host']
...@@ -17,6 +17,8 @@ import warnings ...@@ -17,6 +17,8 @@ import warnings
from gms_preprocessing import __path__ from gms_preprocessing import __path__
from gms_preprocessing.misc.database_tools import delete_record_in_postgreSQLdb from gms_preprocessing.misc.database_tools import delete_record_in_postgreSQLdb
from . import db_host
path_run_gms = os.path.abspath(os.path.join(__path__[0], '..', 'bin', 'run_gms.py')) path_run_gms = os.path.abspath(os.path.join(__path__[0], '..', 'bin', 'run_gms.py'))
...@@ -49,9 +51,9 @@ class Base_CLITester: ...@@ -49,9 +51,9 @@ class Base_CLITester:
def test_hostname_custom(self): def test_hostname_custom(self):
parsed_args = self.parser_run.parse_args(self.baseargs + parsed_args = self.parser_run.parse_args(self.baseargs +
['--db_host', 'geoms']) ['--db_host', db_host])
parsed_args.func(parsed_args) parsed_args.func(parsed_args)
self.assertEqual(self.current_CFG.db_host, 'geoms') self.assertEqual(self.current_CFG.db_host, db_host)
class Test_run_jobid(Base_CLITester.Base_CLITestCase): class Test_run_jobid(Base_CLITester.Base_CLITestCase):
......
...@@ -18,6 +18,8 @@ from gms_preprocessing.options.config import path_options_default ...@@ -18,6 +18,8 @@ from gms_preprocessing.options.config import path_options_default
from gms_preprocessing.options.config import GMSValidator from gms_preprocessing.options.config import GMSValidator
from gms_preprocessing.options.options_schema import gms_schema_config_output from gms_preprocessing.options.options_schema import gms_schema_config_output
from . import db_host
class Test_get_options(TestCase): class Test_get_options(TestCase):
def test_target_is_file_no_validation(self): def test_target_is_file_no_validation(self):
...@@ -32,7 +34,7 @@ class Test_get_options(TestCase): ...@@ -32,7 +34,7 @@ class Test_get_options(TestCase):
class Test_JobConfig(TestCase): class Test_JobConfig(TestCase):
def setUp(self): def setUp(self):
self.jobID = 26186662 self.jobID = 26186662
self.db_host = 'geoms' self.db_host = db_host
def test_plain_args(self): def test_plain_args(self):
cfg = JobConfig(self.jobID, db_host=self.db_host) cfg = JobConfig(self.jobID, db_host=self.db_host)
......
...@@ -15,6 +15,8 @@ from gms_preprocessing.misc.exception_handler import log_uncaught_exceptions ...@@ -15,6 +15,8 @@ from gms_preprocessing.misc.exception_handler import log_uncaught_exceptions
from gms_preprocessing.algorithms.L1A_P import L1A_object from gms_preprocessing.algorithms.L1A_P import L1A_object
from gms_preprocessing.misc.database_tools import get_info_from_postgreSQLdb from gms_preprocessing.misc.database_tools import get_info_from_postgreSQLdb
from . import db_host
class BaseTest_ExceptionHandler: class BaseTest_ExceptionHandler:
"""Test class for gms_preprocessing.misc.exception_handler.ExceptionHandler.""" """Test class for gms_preprocessing.misc.exception_handler.ExceptionHandler."""
...@@ -22,7 +24,7 @@ class BaseTest_ExceptionHandler: ...@@ -22,7 +24,7 @@ class BaseTest_ExceptionHandler:
PC = None # default PC = None # default
def get_process_controller(self, jobID): def get_process_controller(self, jobID):
self.PC = process_controller(jobID, parallelization_level='scenes', db_host='geoms', self.PC = process_controller(jobID, parallelization_level='scenes', db_host=db_host,
is_test=True, log_level='DEBUG') is_test=True, log_level='DEBUG')
self.PC.config.disable_exception_handler = False self.PC.config.disable_exception_handler = False
......
...@@ -53,15 +53,16 @@ from gms_preprocessing.algorithms.L2B_P import L2B_object ...@@ -53,15 +53,16 @@ from gms_preprocessing.algorithms.L2B_P import L2B_object
from gms_preprocessing.algorithms.L2C_P import L2C_object from gms_preprocessing.algorithms.L2C_P import L2C_object
from gms_preprocessing.misc.database_tools import get_info_from_postgreSQLdb from gms_preprocessing.misc.database_tools import get_info_from_postgreSQLdb
from . import db_host
__author__ = 'Daniel Scheffler' # edited by Jessica Palka. __author__ = 'Daniel Scheffler' # edited by Jessica Palka.
# Rootpath of the gms_preprocessing-repository. # Rootpath of the gms_preprocessing-repository.
gmsRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) gmsRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
# Defining the configurations needed to start a job containing the different dataset scenes. # Defining the configurations needed to start a job containing the different dataset scenes.
# TODO Change the job-configurations for selected datasets. # TODO Change the job-configurations for selected datasets.
job_config_kwargs = dict(parallelization_level='scenes', db_host='geoms', delete_old_output=True, is_test=True) job_config_kwargs = dict(parallelization_level='scenes', db_host=db_host, delete_old_output=True, is_test=True)
########################## ##########################
...@@ -299,7 +300,7 @@ class Test_in_normal_mode(unittest.TestCase): ...@@ -299,7 +300,7 @@ class Test_in_normal_mode(unittest.TestCase):
# self.job_id = 26186906 # Bug Input Validator # self.job_id = 26186906 # Bug Input Validator
self.job_id = 26186925 # 1 Sentinel-2A, Bug NoneType' object has no attribute 'find' self.job_id = 26186925 # 1 Sentinel-2A, Bug NoneType' object has no attribute 'find'
self.PC = process_controller(self.job_id, **dict(is_test=False, parallelization_level='scenes', db_host='geoms', self.PC = process_controller(self.job_id, **dict(is_test=False, parallelization_level='scenes', db_host=db_host,
delete_old_output=True, disable_exception_handler=True)) delete_old_output=True, disable_exception_handler=True))
def test(self): def test(self):
......
...@@ -16,6 +16,7 @@ from gms_preprocessing.io.input_reader import DEM_Creator ...@@ -16,6 +16,7 @@ from gms_preprocessing.io.input_reader import DEM_Creator
from gms_preprocessing import set_config from gms_preprocessing import set_config
from py_tools_ds.geo.projection import EPSG2WKT, WKT2EPSG from py_tools_ds.geo.projection import EPSG2WKT, WKT2EPSG
from .import db_host
gmsRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) gmsRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
...@@ -23,7 +24,7 @@ gmsRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..') ...@@ -23,7 +24,7 @@ gmsRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')
class Test_DEM_Creator(unittest.TestCase): class Test_DEM_Creator(unittest.TestCase):
def setUp(self): def setUp(self):
# Testjob Landsat-8 # Testjob Landsat-8
set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset_status=True, is_test=True, set_config(exec_mode='Python', job_ID=26186196, db_host=db_host, reset_status=True, is_test=True,
path_archive=os.path.join(gmsRepo_rootpath, 'tests', 'data', 'archive_data')) path_archive=os.path.join(gmsRepo_rootpath, 'tests', 'data', 'archive_data'))
self.boxMapXY = ((277365.0, 5546625.0), (292365.0, 5546625.0), (292365.0, 5531625.0), (277365.0, 5531625.0)) self.boxMapXY = ((277365.0, 5546625.0), (292365.0, 5546625.0), (292365.0, 5531625.0), (277365.0, 5531625.0))
......
...@@ -19,6 +19,7 @@ from gms_preprocessing import __file__ # noqa E402 module level import not at t ...@@ -19,6 +19,7 @@ from gms_preprocessing import __file__ # noqa E402 module level import not at t
from gms_preprocessing.options.config import set_config # noqa E402 module level import not at top of file from gms_preprocessing.options.config import set_config # noqa E402 module level import not at top of file
from gms_preprocessing.algorithms.L2B_P import KMeansRSImage # noqa E402 module level import not at top of file from gms_preprocessing.algorithms.L2B_P import KMeansRSImage # noqa E402 module level import not at top of file
from . import db_host
testdata = os.path.join(os.path.dirname(__file__), testdata = os.path.join(os.path.dirname(__file__),
'../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq') '../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq')
...@@ -30,7 +31,7 @@ class Test_KMeansRSImage(unittest.TestCase): ...@@ -30,7 +31,7 @@ class Test_KMeansRSImage(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
# Testjob Landsat-8 # Testjob Landsat-8
set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset_status=True) set_config(exec_mode='Python', job_ID=26186196, db_host=db_host, reset_status=True)
cls.geoArr = GeoArray(testdata) cls.geoArr = GeoArray(testdata)
cls.geoArr.to_mem() cls.geoArr.to_mem()
cls.kmeans = KMeansRSImage(cls.geoArr, n_clusters=10) cls.kmeans = KMeansRSImage(cls.geoArr, n_clusters=10)
......
...@@ -21,6 +21,7 @@ from gms_preprocessing.algorithms.L2B_P import ReferenceCube_Generator # noqa E ...@@ -21,6 +21,7 @@ from gms_preprocessing.algorithms.L2B_P import ReferenceCube_Generator # noqa E
from gms_preprocessing.algorithms.L2B_P import RefCube # noqa E402 module level import not at top of file from gms_preprocessing.algorithms.L2B_P import RefCube # noqa E402 module level import not at top of file
from gms_preprocessing.algorithms.L2B_P import SpectralHomogenizer # noqa E402 module level import not at top of file from gms_preprocessing.algorithms.L2B_P import SpectralHomogenizer # noqa E402 module level import not at top of file
from . import db_host
testdata = os.path.join(__path__[0], '../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq') testdata = os.path.join(__path__[0], '../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq')
...@@ -31,7 +32,7 @@ testdata = os.path.join(__path__[0], '../tests/data/hy_spec_data/Bavaria_farmlan ...@@ -31,7 +32,7 @@ testdata = os.path.join(__path__[0], '../tests/data/hy_spec_data/Bavaria_farmlan
# @classmethod # @classmethod
# def setUpClass(cls): # def setUpClass(cls):
# # Testjob Landsat-8 # # Testjob Landsat-8
# set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset_status=True, is_test=True) # set_config(exec_mode='Python', job_ID=26186196, db_host=db_host, reset_status=True, is_test=True)
# cls.SHC = ReferenceCube_Generator_OLD([testdata, testdata, ], v=False) # cls.SHC = ReferenceCube_Generator_OLD([testdata, testdata, ], v=False)
# #
# def test_generate_reference_cube_L8(self): # def test_generate_reference_cube_L8(self):
...@@ -66,7 +67,7 @@ class Test_ReferenceCube_Generator(unittest.TestCase): ...@@ -66,7 +67,7 @@ class Test_ReferenceCube_Generator(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
# Testjob Landsat-8 # Testjob Landsat-8
set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset_status=True, is_test=True) set_config(exec_mode='Python', job_ID=26186196, db_host=db_host, reset_status=True, is_test=True)
cls.tmpOutdir = tempfile.TemporaryDirectory() cls.tmpOutdir = tempfile.TemporaryDirectory()
cls.testIms = [testdata, testdata, ] cls.testIms = [testdata, testdata, ]
cls.tgt_sat_sen_list = [ cls.tgt_sat_sen_list = [
...@@ -135,7 +136,7 @@ class Test_SpectralHomogenizer(unittest.TestCase): ...@@ -135,7 +136,7 @@ class Test_SpectralHomogenizer(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
# Testjob Landsat-8 # Testjob Landsat-8
cfg = set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset_status=True, is_test=True) cfg = set_config(exec_mode='Python', job_ID=26186196, db_host=db_host, reset_status=True, is_test=True)
cls.SpH = SpectralHomogenizer(classifier_rootDir=cfg.path_spechomo_classif) cls.SpH = SpectralHomogenizer(classifier_rootDir=cfg.path_spechomo_classif)
cls.testArr_L8 = GeoArray(np.random.randint(1, 10000, (50, 50, 7), dtype=np.int16)) # no band 9, no pan cls.testArr_L8 = GeoArray(np.random.randint(1, 10000, (50, 50, 7), dtype=np.int16)) # no band 9, no pan
......
...@@ -19,6 +19,8 @@ from gms_preprocessing.io.input_reader import SRF ...@@ -19,6 +19,8 @@ from gms_preprocessing.io.input_reader import SRF
from gms_preprocessing.options.config import set_config from gms_preprocessing.options.config import set_config
from gms_preprocessing.algorithms.L2B_P import SpectralResampler as SR from gms_preprocessing.algorithms.L2B_P import SpectralResampler as SR
from . import db_host
testdata = os.path.join(os.path.dirname(__file__), testdata = os.path.join(os.path.dirname(__file__),
'../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq') '../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq')
...@@ -29,7 +31,7 @@ class Test_SpectralResampler(unittest.TestCase): ...@@ -29,7 +31,7 @@ class Test_SpectralResampler(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
# Testjob Landsat-8 # Testjob Landsat-8
set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset_status=True) set_config(exec_mode='Python', job_ID=26186196, db_host=db_host, reset_status=True, is_test=True)
cls.geoArr = GeoArray(testdata) cls.geoArr = GeoArray(testdata)
cls.geoArr.to_mem() cls.geoArr.to_mem()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment