Commit 4928143f authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Updated CI setup files, environment_gms_preprocessing.yml and .gitlab-ci.yml.


Signed-off-by: Daniel Scheffler's avatarDaniel Scheffler <danschef@gfz-potsdam.de>
parent 92562eb9
Pipeline #12276 failed with stages
in 5 minutes and 30 seconds
......@@ -17,14 +17,11 @@ stages:
test_gms_preprocessing:
stage: test
script:
- source /root/miniconda3/bin/activate
- source activate ci_env
- export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- source /root/miniconda3/bin/activate ci_env
- export GMS_db_host=${GMS_DB_HOST}
- export GMS_index_host=${GMS_INDEX_HOST}
- export GMS_index_port=${GMS_INDEX_PORT}
- pip install "pyrsr>=0.3.1" # TODO remove as soon as docker runner is recreated
# update sicor
# - conda install -y -q -c conda-forge basemap
# - rm -rf context/sicor
......@@ -34,8 +31,7 @@ test_gms_preprocessing:
# - python setup.py install
# - cd ../../
# make tests
- pip install git+https://gitext.gfz-potsdam.de/geomultisens/spechomo.git # TODO remove as soon as this is included in the CI container
- pip install -U sicor
# run tests
- make nosetests
......@@ -52,11 +48,7 @@ test_gms_preprocessing:
test_styles:
stage: test
script:
- source /root/miniconda3/bin/activate
- source activate ci_env
- export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
# - pip install "pycodestyle>=2.0.0,!=2.4.0" # TODO remove as soon as docker runner is recreated
- source /root/miniconda3/bin/activate ci_env
- make lint
artifacts:
paths:
......@@ -77,12 +69,16 @@ pages: # this job must be called 'pages' to advise GitLab to upload content to
- mkdir -p public/doc
- mkdir -p public/coverage
- mkdir -p public/nosetests_reports
# Copy over the docs
- cp -r docs/_build/html/* public/doc/
# Copy over the coverage reports
- cp -r htmlcov/* public/coverage/
# Copy over the nosetests reports
- cp nosetests.* public/nosetests_reports/
# Check if everything is working great
- ls -al public
- ls -al public/doc
......
......@@ -2,34 +2,61 @@
context_dir="./context"
dockerfile="gms_preprocessing_ci.docker"
tag="gms_preprocessing_ci:0.16.2"
tag="gms_preprocessing_ci:0.18.3"
gitlab_runner="gms_preprocessing_gitlab_CI_runner"
# get sicor project
rm -rf context/sicor
git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor
# download sicor cache (fastens SICOR CI tests a lot, but cache needs to be updated manually using a local sicor repo:
# 1. clone a fresh copy of sicor or delete sicor/sicor/aerosol_0_ch4_34d3778719cc87188787de09bb8f870d16050078.pkl.zip
# 2. run a sicor test including sicor_ac or enmap_ac (recreates cache file) -> upload newly created cache file
# wget http://ouo.io/uCQxof -P ./context/
echo "#### Build runner docker image"
sudo docker rmi ${tag}
sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir}
docker rmi ${tag}
docker build ${context_dir} \
--no-cache \
-f ${context_dir}/${dockerfile} \
-m 20G \
-t ${tag}
# create the gitlab-runner docker container for the current project
# NOTE: The 'gitlab-runner' and 'gitlab-ci-multi-runner' services will run within this container.
# The runner uses a 'config.toml' configuration file at /etc/gitlab-runner within the container which can be
# modified through additional parameters of the 'gitlab-runner register' command.
echo "#### Create gitlab-runner (daemon) container with tag; ${tag}"
sudo docker stop ${gitlab_runner}
sudo docker rm ${gitlab_runner}
sudo docker run -d --name ${gitlab_runner} --restart always \
-v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest
docker stop ${gitlab_runner}
docker rm ${gitlab_runner}
docker run \
-d \
--name ${gitlab_runner} \
--restart always \
-v /var/run/docker.sock:/var/run/docker.sock \
gitlab/gitlab-runner:latest
# register the runner at the corresponding GitLab repository via a registration-token
echo "#### Register container at gitlab, get token here https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/settings/ci_cd"
read -p "Please enter gitlab token: " token
echo ""
read -p "Please enter gitlab runner name: " runner_name
echo "New gitlab runner image will named ${gitlab_runner}"
sudo docker exec -it ${gitlab_runner} /bin/bash -c "export RUNNER_EXECUTOR=docker && gitlab-ci-multi-runner register -n \
# NOTE: In case of locally stored images (like here), the docker pull policy 'never' must be used
# (see https://docs.gitlab.com/runner/executors/docker.html#how-pull-policies-work).
docker exec -it ${gitlab_runner} /bin/bash -c "\
export RUNNER_EXECUTOR=docker && \
gitlab-ci-multi-runner register \
--non-interactive \
--executor 'docker' \
--docker-image '${tag}' \
--url 'https://gitext.gfz-potsdam.de/ci' \
--registration-token '${token}' \
--run-untagged=true \
--locked=true \
--tag-list gms_preprocessing_ci_client \
--description '${runner_name}' \
--docker-image '${tag}' "
--tag-list gms_preprocessing_ci_client \
--run-untagged='true' \
--locked='true' \
--access-level='not_protected' \
--docker-pull-policy='never'
"
ls
name: gms_preprocessing
channels:
- http://conda.anaconda.org/ioam # only for holoviews
- http://conda.anaconda.org/conda-forge
- conda-forge
dependencies:
- python=3.*.*
......@@ -10,9 +9,7 @@ dependencies:
- pyqt
- numpy
- gdal
- conda-forge::libgdal # force to use conda-forge for libgdal to avoid package version incompatiblies due to mixed channels
- scikit-image
- rasterio
- pyproj
- lxml
- geopandas
......@@ -38,7 +35,6 @@ dependencies:
- h5py
- pytables
- pip:
- dicttoxml
- jsmin
......@@ -52,7 +48,6 @@ dependencies:
- six
- spectral
- flake8
- pycodestyle<2.4.0 # fixes ImportError: module 'pycodestyle' has no attribute 'break_around_binary_operator'
- pylint
- pydocstyle
- nose
......@@ -84,7 +79,7 @@ dependencies:
- geoarray>=0.8.0
- arosics>=0.8.7
- pyrsr>=0.3.1
- git+https://gitext.gfz-potsdam.de/geomultisens/spechomo.git#egg=spechomo
- spechomo
# sicor
- https://software.ecmwf.int/wiki/download/attachments/56664858/ecmwf-api-client-python.tgz
FROM gms_base_centos:0.2
FROM ci_base_centos:0.1
# copy some needed stuff to /root
COPY *.yml /root/
......@@ -6,6 +6,7 @@ COPY *.yml /root/
# update the ci_env environment (that already contains all packages installed via 'docker_pyenvs' repo)
RUN /bin/bash -i -c "\
source /root/miniconda3/bin/activate; \
conda update -n base -c conda-forge conda;\
source activate ci_env; \
conda env update -n ci_env -f /root/environment_gms_preprocessing.yml"
......@@ -13,10 +14,13 @@ RUN /bin/bash -i -c "\
COPY sicor /tmp/sicor
RUN bash -i -c "\
source /root/miniconda3/bin/activate; \
source activate ci_env; \
source /root/miniconda3/bin/activate ci_env; \
cd /tmp/sicor/ ; \
make clean ; \
make requirements ; \
make download-tables ; \
pip install . --no-cache-dir"
# copy sicor cache files to sicor root directory (speeds up SICOR CI tests because table subsets dont have to be created each time)
# -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options
COPY *.zip /tmp/sicor/sicor
......@@ -90,7 +90,7 @@ class Base_CLITester:
self.assertEqual(self.current_CFG.virtual_sensor_id, vid)
# test if parameter fallbacks are working ('CPUs' has a fallback)
self.assertNotEquals(self.current_CFG.CPUs, None)
self.assertNotEqual(self.current_CFG.CPUs, None)
self.assertNotIsInstance(self.current_CFG.CPUs, str)
def test_json_opts(self):
......
......@@ -58,7 +58,7 @@ class Test_MultiSlotLock(unittest.TestCase):
def test_with_statement(self):
with MultiSlotLock('unittest', allowed_slots=15) as lock:
self.assertNotEquals(lock, None)
self.assertNotEqual(lock, None)
class Test_ProcessLock(unittest.TestCase):
......@@ -79,7 +79,7 @@ class Test_ProcessLock(unittest.TestCase):
def test_with_statement(self):
with ProcessLock(allowed_slots=15) as lock:
self.assertNotEquals(lock, None)
self.assertNotEqual(lock, None)
class Test_MemoryReserver(unittest.TestCase):
......@@ -100,4 +100,4 @@ class Test_MemoryReserver(unittest.TestCase):
def test_with_statement(self):
with MemoryReserver(mem2lock_gb=20) as lock:
self.assertNotEquals(lock, None)
self.assertNotEqual(lock, None)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment