Commit 9a8abdbb authored by Jessica Palka's avatar Jessica Palka
Browse files

Merge remote-tracking branch 'origin/dev' into dev

Conflicts:
	.gitignore

Former-commit-id: c796ea9e
parents f12be999 fb8fb3f7
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
*.log
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# pyenv python configuration file
.python-version
/GeoMultiSens.py
.idea/
BAK/
OLD/
database/cloud_classifier/
database/sampledata/
database/metadata/
database/processed_data/
.pyc
/geomultisens/database/cloud_classifier/
/tests/data/sampledata/
/geomultisens/database/metadata/
/geomultisens/database/tiling_system/
/geomultisens/database/processed_data/
/geomultisens/database/earth_sun_distance/horizon_web_interface.cgi_files/
/geomultisens/database/earth_sun_distance/horizon_web_interface.cgi.html
/geomultisens/database/earth_sun_distance/README
/geomultisens/database/solar_irradiance/Solar_irradiance_Thuillier_2002.xls
/geomultisens/database/solar_irradiance/Thuillier_2003_solar_irradiance_400_2400.pdf
/geomultisens/database/solar_irradiance/SUNp1fontenla.asc
/geomultisens/database/solar_irradiance/SOLAR.txt
/geomultisens/logs/
.ipynb_checkpoints/
__pycache__/
io/envi_io.py
testing/ENVIcompressed.bsq
testing/ENVIcompressed.hdr
testing/ENVIcompressed.worked.bsq
testing/ENVIcompressed.worked.hdr
testing/ENVIselfcompressed.bsq
testing/Landsat-8__OLI_TIRS__LC81930242016103LGN00__33UVT_L2C_selfcompressed.bsq
testing/Landsat-8__OLI_TIRS__LC81930242016103LGN00__33UVT_L2C_selfcompressed.hdr
testing/Landsat-8__OLI_TIRS__LC81930242016103LGN00__33UVT_L2C_selfcompressed.hdr.BAK
testing/Landsat-8__OLI_TIRS__LC81930242016119LGN00__32UPB_masks_L2C.bsq
testing/Landsat-8__OLI_TIRS__LC81930242016119LGN00__32UPB_masks_L2C.hdr
testing/Landsat-8__OLI_TIRS__LC81940242015091LGN00__32UNC_masks_L2C__compressed.bsq
testing/job_logs/
testing/meta_validation/
sandbox/ENVIcompressed.bsq
sandbox/ENVIcompressed.hdr
sandbox/ENVIcompressed.worked.bsq
sandbox/ENVIcompressed.worked.hdr
sandbox/ENVIselfcompressed.bsq
sandbox/Landsat-8__OLI_TIRS__LC81930242016103LGN00__33UVT_L2C_selfcompressed.bsq
sandbox/Landsat-8__OLI_TIRS__LC81930242016103LGN00__33UVT_L2C_selfcompressed.hdr
sandbox/Landsat-8__OLI_TIRS__LC81930242016103LGN00__33UVT_L2C_selfcompressed.hdr.BAK
sandbox/Landsat-8__OLI_TIRS__LC81930242016119LGN00__32UPB_masks_L2C.bsq
sandbox/Landsat-8__OLI_TIRS__LC81930242016119LGN00__32UPB_masks_L2C.hdr
sandbox/Landsat-8__OLI_TIRS__LC81940242015091LGN00__32UNC_masks_L2C__compressed.bsq
sandbox/job_logs/
sandbox/meta_validation/
......@@ -13,3 +13,4 @@ Contributors
* Robert Behling <robert.behling@gfz-potsdam.de>
* Daniel Eggert <daniel.eggert@gfz-potsdam.de>
* Jessica Palka <jessica.palka@gfz-potsdam.de>
......@@ -7,7 +7,7 @@ GeoMultiSens - Scalable Multi-Sensor Analysis of Remote Sensing Data
* Free software: GNU General Public License v3
* Documentation: https://geomultisens.readthedocs.io.
* Documentation: http://danschef.gitext.gfz-potsdam.de/GeoMultiSens/doc/
Status
......@@ -27,13 +27,36 @@ Status
:target: https://pyup.io/repos/github/danschef/geomultisens/
:alt: Updates
.. image:: https://gitext.gfz-potsdam.de/danschef/GeoMultiSens/badges/master/build.svg
.. image:: https://gitext.gfz-potsdam.de/danschef/GeoMultiSens/badges/master/coverage.svg
See also the latest coverage_ report.
Features
--------
* TODO
Installation
------------
Clone the following repository and install GeoMultiSens to your local python:
.. code-block:: console
git clone https://gitext.gfz-potsdam.de:danschef/GeoMultiSens.git
cd GeoMultiSens; make install
.. .. The pip installer does not work as long as the packages arosics, geoarray, py_tools_ds and sicor are not public.
.. .. * Use the pip-Installer:
.. .. pip install git+https://gitext.gfz-potsdam.de/danschef/GeoMultiSens.git # for HTTPS authentification
.. .. pip install git+git@gitext.gfz-potsdam.de:danschef/GeoMultiSens.git # for SSH-Key authentification
Credits
-------
......@@ -41,4 +64,4 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack
.. _Cookiecutter: https://github.com/audreyr/cookiecutter
.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
.. _coverage: http://danschef.gitext.gfz-potsdam.de/GeoMultiSens/coverage/
......@@ -19,11 +19,6 @@ export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/scheffler/python/GeoMultiSens # ne
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/scheffler/python/geoarray # needed to find geoarray
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/scheffler/python/py_tools_ds # needed to find py_tools_ds
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/scheffler/python/arosics # needed to find e.g. AROSICS
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/hollstein/python/AT4P # needed to find AT4P
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/hollstein/python/ECMWF # needed to find ECMWF
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/hollstein/python/S2MSI # needed to find S2MSI
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/hollstein/python/S2SCAPEM # needed to find S2SCAPEM
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/hollstein/python/cBc # needed to find cBc
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/hollstein/python/sicor # needed to find sicor
......
......@@ -19,11 +19,6 @@ export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/scheffler/python_deployed/GeoMultiS
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/scheffler/python_deployed/geoarray # needed to find geoarray
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/scheffler/python_deployed/py_tools_ds # needed to find py_tools_ds
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/scheffler/python_deployed/arosics # needed to find e.g. AROSICS
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/hollstein/python/AT4P # needed to find AT4P
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/hollstein/python/ECMWF # needed to find ECMWF
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/hollstein/python/S2MSI # needed to find S2MSI
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/hollstein/python/S2SCAPEM # needed to find S2SCAPEM
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/hollstein/python/cBc # needed to find cBc
export PYTHONPATH=${PYTHONPATH}:/home/gfz-fe/hollstein/python/sicor # needed to find sicor
......
......@@ -659,7 +659,7 @@ class AtmCorr(object):
def _get_srf(self):
"""Returns an instance of SRF in the same structure like S2MSI.Tools.S2SRF.S2SRF
"""Returns an instance of SRF in the same structure like sicor.sensors.SRF.SensorSRF
"""
# FIXME calculation of center wavelengths within SRF() used not the GMS algorithm
# SRF instance must be created for all bands and the previous proc level
......
......@@ -62,8 +62,9 @@ GMS_config = GMS_configuration()
class Job:
def __init__(self, call_type, ID, exec_mode='Python', db_host='localhost', exec_L1AP=None, exec_L1BP=None,
exec_L1CP=None, exec_L2AP=None, exec_L2BP=None, exec_L2CP=None, CPUs=None, sub_multiProc=True,
exc_handler=True, log_level='INFO', blocksize=(2048,2048), profiling=False, bench_all=False,
bench_cloudMask=False, path_procdata_scenes=None, path_procdata_MGRS=None, path_archive=None):
exc_handler=True, log_level='INFO', blocksize=(2048,2048), is_test=False, profiling=False,
bench_all=False, bench_cloudMask=False, path_procdata_scenes=None, path_procdata_MGRS=None,
path_archive=None):
"""Create a job configuration
......@@ -84,6 +85,7 @@ class Job:
:param log_level: the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';
default: 'INFO')
:param blocksize: X/Y block size to be used for any tiling process (default: (2048,2048)
:param is_test: whether the curent job represents a test job (run by a test runner) or not (default=False)
:param profiling: enable/disable code profiling (default: False)
:param bench_all: enable/disable benchmark of the whole processing pipeline
:param bench_cloudMask: enable/disable benchmark of the of the cloud mask generator module
......@@ -115,6 +117,7 @@ class Job:
self.disable_exception_handler = exc_handler is False
self.log_level = log_level
self.tiling_block_size_XY = blocksize
self.is_test = is_test
self.profiling = profiling
self.benchmark_global = bench_all
self.bench_CLD_class = bench_cloudMask
......
......@@ -22,13 +22,14 @@ def check_dependencies(logger=None):
javaLibs = []
# check if spatial index mediator is running
SpatIdxSrv = SpatialIndexMediatorServer(CFG.job.path_spatIdxSrv)
if not SpatIdxSrv.is_running:
SpatIdxSrv.start()
_log_or_print('Spatial Index Mediator Server started successfully.', logger.info)
if not SpatIdxSrv.is_running:
_log_or_print('Attempt to start Spatial Index Mediator Server failed.', logger.info)
if not CFG.job.is_test:
SpatIdxSrv = SpatialIndexMediatorServer(CFG.job.path_spatIdxSrv)
if not SpatIdxSrv.is_running:
SpatIdxSrv.start()
_log_or_print('Spatial Index Mediator Server started successfully.', logger.info)
if not SpatIdxSrv.is_running:
_log_or_print('Attempt to start Spatial Index Mediator Server failed.', logger.info)
def check_paths():
# check existance of database paths, etc.
......
......@@ -8,12 +8,13 @@ RUN /bin/bash -i -c "cd /root; wget https://repo.continuum.io/archive/$anaconda_
rm -f /root/$anaconda_dl"
# 'conda install --yes -c conda-forge 'icu=58.*' lxml' fixing a bug for conda-forge gdal build, as of 06/2017
# install fmask and rios explicitly without dependencies -> install dependencies (one by one in order to definitly exclude gdal)
RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
conda install --yes pyqt coverage ; \
conda install --yes pyqt coverage numba; \
conda install --yes -c conda-forge gdal ; \
conda install --yes -c conda-forge pyfftw=0.10.4 ; \
conda install --yes -c conda-forge 'icu=58.*' lxml ; \
conda install --yes -c conda-forge glymur pygrib rasterio pyproj cachetools pyhdf ; \
conda install --yes -c conda-forge glymur pygrib rasterio pyproj cachetools pyhdf ephem ; \
conda install --yes -c conda-forge python-fmask=0.4.4 rios=1.4.3 --no-deps ; \
conda install --yes -c conda-forge freexl=1.0.2 ; \
conda install --yes -c conda-forge giflib=5.1.4 ; \
......@@ -24,10 +25,9 @@ RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
conda install --yes -c conda-forge olefile=0.44 ; \
conda install --yes xerces-c ; \
conda install --yes -c conda-forge glymur pygrib rasterio pyproj cachetools pyhdf ; \
conda install --yes -c conda-forge glymur pygrib rasterio pyproj cachetools pyhdf ; \
conda install --yes -c conda-forge glymur pygrib rasterio pyproj cachetools pyhdf ; \
pip install https://software.ecmwf.int/wiki/download/attachments/56664858/ecmwf-api-client-python.tgz ; \
pip install shapely geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse spectral psycopg2" # must include all the requirements needed to build the docs!
pip install shapely geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse \
spectral psycopg2 pyorbital pyinstrument geoalchemy2" # must include all the requirements needed to build the docs!
# copy some needed stuff to /root
#COPY *.pkl /root/ # EXAMPLE
......
......@@ -81,7 +81,8 @@ class Test_Landsat8_PreCollectionDataNew(BaseTestCases.TestAll):
job_config_kwargs = dict(
path_archive = os.path.join(gmsRepo_rootpath, 'tests', 'data', 'archive_data'),
path_procdata_scenes = os.path.join(gmsRepo_rootpath, 'tests', 'data', 'output_scenes'),
path_procdata_MGRS = os.path.join(gmsRepo_rootpath, 'tests', 'data', 'output_mgrs_tiles'))
path_procdata_MGRS = os.path.join(gmsRepo_rootpath, 'tests', 'data', 'output_mgrs_tiles'),
is_test = True)
cls.PC = process_controller(26186196, parallelization_level='scenes', db_host='geoms', delete_old_output=True,
job_config_kwargs=job_config_kwargs)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment