Commit c54c93b5 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'enhancement/revise_docker_env' into 'master'

Enhancement/revise docker env

See merge request !11
parents 6ecc002b d7f8936d
Pipeline #14518 failed with stage
in 12 minutes and 21 seconds
......@@ -57,7 +57,7 @@ test_gms_preprocessing_install:
- source /root/miniconda3/bin/activate
- conda update conda
- conda create -y -q -c conda-forge --name gms_test python=3
- source activate gms_test
- conda activate gms_test
# update conda and python
- conda update -n base -c conda-forge conda
......@@ -74,7 +74,6 @@ test_gms_preprocessing_install:
# install not pip-installable deps of sicor
- conda install -y -q -c conda-forge glymur pygrib cachetools pyhdf h5py pytables llvmlite numba
- pip install ecmwf-api-client # FIXME remove as long as this is no more needed for the hyperspectral part of SICOR
# install gms_preprocessing
- pip install -e .
......
......@@ -2,6 +2,16 @@
History
=======
0.18.7 (2020-10-13)
-------------------
Added missing requirement 'openpyxl'.
Replaced deprecated 'source activate' by 'conda activate'.
Removed ecmwf-api-client installation as this is now included in SICOR.
Pinned scipy to 1.1.0 in environment_gms_preprocessing.yml to make CI work for now.
Revised docker container setup files.
0.18.6 (2020-10-12)
-------------------
......
......@@ -134,7 +134,7 @@ Using conda_, the recommended approach is:
# create virtual environment for gms_preprocessing, this is optional
conda create -c conda-forge --name gms_preprocessing python=3
source activate gms_preprocessing
conda activate gms_preprocessing
# install some dependencies that cause trouble when installed via pip
conda install -c conda-forge numpy gdal scikit-image pyproj geopandas ipython matplotlib cartopy scikit-learn=0.23.2 shapely pyhdf python-fmask holoviews
......@@ -144,7 +144,6 @@ Using conda_, the recommended approach is:
# install not pip-installable deps of sicor
conda install -c conda-forge glymur pygrib cachetools pyhdf h5py pytables llvmlite numba
pip install ecmwf-api-client
# install gms_preprocessing
git clone https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing.git
......
......@@ -24,5 +24,5 @@
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
__version__ = '0.18.6'
__versionalias__ = '20201012.01'
__version__ = '0.18.7'
__versionalias__ = '20201013.01'
......@@ -13,6 +13,7 @@ matplotlib
natsort
nested_dict
numpy
openpyxl # implicitly needed by pandas to write GMS job summaries as excel files
pandas
psycopg2
# pyhdf # not pip installable
......
scipy
ephem
pyorbital
cerberus
dill
pytz
spectral
ephem
geoalchemy2
geopandas
iso8601
pyinstrument
geoalchemy2
sqlalchemy
tqdm
jsmin
cerberus
natsort
nested_dict
timeout_decorator
redis
retools
redis-semaphore
openpyxl
psutil
pyinstrument
pyorbital
pyrsr>=0.3.1
natsort
pytz
redis
redis-semaphore
retools
scipy
spectral
sqlalchemy
timeout_decorator
tqdm
......@@ -54,6 +54,7 @@ requirements = [
'natsort',
'nested_dict',
'numpy',
'openpyxl', # implicitly needed by pandas to write GMS job summaries as excel files
'pandas',
'psutil',
'psycopg2',
......
......@@ -2,13 +2,9 @@
context_dir="./context"
dockerfile="gms_preprocessing_ci.docker"
tag="gms_preprocessing_ci:0.18.3"
tag="gms_preprocessing_ci:0.18.5"
gitlab_runner="gms_preprocessing_gitlab_CI_runner"
# get sicor project
rm -rf context/sicor
git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor
# download sicor cache (fastens SICOR CI tests a lot, but cache needs to be updated manually using a local sicor repo:
# 1. clone a fresh copy of sicor or delete sicor/sicor/aerosol_0_ch4_34d3778719cc87188787de09bb8f870d16050078.pkl.zip
# 2. run a sicor test including sicor_ac or enmap_ac (recreates cache file) -> upload newly created cache file
......
......@@ -6,59 +6,45 @@ channels:
dependencies:
- python>=3.7.*
- pip
- cartopy
- arosics>=1.0.0
- gdal
- geoarray>=0.9.0
- geopandas
- holoviews
- ipython
- pyhdf
- python-fmask
- matplotlib
- numpy
- pyproj
- pyrsr>=0.3.1
- py-tools-ds>=0.12.4
- scikit-image
- scikit-learn=0.23.2 # pinned due to spechomo classifiers created with this version
- shapely
# arosics
- pyfftw
- pykrige
# sicor
- cachetools
- glymur
- h5py
- llvmlite
- numba
- pygrib
- pytables
- scipy<=1.1.0 # pinned due to https://gitext.gfz-potsdam.de/EnMAP/sicor/-/issues/63
- sicor
- spechomo
- pip:
- arosics>=1.0.0
- cerberus
- dill
- ecmwf-api-client
- ephem
- geoalchemy2
- geoarray>=0.9.0
- jsmin
- iso8601
- natsort
- nested_dict
- openpyxl
- pandas
- psutil
- psycopg2
- pyinstrument
- pyorbital
- pyrsr>=0.3.1
- py_tools_ds>=0.12.4
- pytz
- redis
- retools
- redis-semaphore
- scipy
- sicor
- spechomo>=0.16
- spectral
- sqlalchemy
- timeout_decorator
......@@ -74,6 +60,3 @@ dependencies:
- nose-htmloutput
- rednose
- sphinx-argparse
- sqlalchemy
- psycopg2
......@@ -2,6 +2,7 @@ FROM ci_base_centos:0.1
# copy some needed stuff to /root
COPY *.yml /root/
COPY *.sh /root/
# update the ci_env environment (that already contains all packages installed via 'docker_pyenvs' repo)
RUN /bin/bash -i -c "\
......@@ -10,14 +11,8 @@ RUN /bin/bash -i -c "\
source activate ci_env; \
conda env update -n ci_env -f /root/environment_gms_preprocessing.yml"
# copy sicor code to /tmp
COPY sicor /tmp/sicor
RUN bash -i -c "\
source /root/miniconda3/bin/activate ci_env; \
cd /tmp/sicor/ ; \
make install"
# copy sicor cache files to sicor root directory (speeds up SICOR CI tests because table subsets dont have to be created each time)
# -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options
COPY *.zip /tmp/sicor/sicor/
# download SICOR atmospheric lookup tables and copy already created cache files to SICOR root dir to speed up SICOR in CI jobs
RUN mkdir /tmp/sicor_cache
COPY *.zip /tmp/sicor_cache/
RUN /bin/bash -i -c "\
bash /root/setup_sicor_tables_and_cachefiles.sh"
#!/usr/bin/env bash
conda /root/miniconda3/bin/activate ci_env
# copy sicor cache files to sicor root directory (speeds up SICOR CI tests because table subsets dont have to be created each time)
# -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options
SICOR_PATH=$(python -c "from sicor import __path__; print(__path__[0])")
echo Copying SICOR cache files to ${SICOR_PATH}/...
cp /tmp/sicor_cache/*.zip ${SICOR_PATH}/
# download atmospheric lookup tables to save time when running CI jobs later
echo Downloading SICOR atmospheric lookup tables...
python -c "from sicor.tables import get_tables; get_tables(sensor=\"s2\"); get_tables(sensor=\"l8\")"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment