Commit c138ec38 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

First setup of CI runner.

Former-commit-id: 81ad13a1
Former-commit-id: 0e831c0b
parent b33e84a6
before_script:
- git lfs pull
test_geomultisens:
script:
- source /root/anaconda3/bin/activate
- export GDAL_DATA=/root/anaconda3/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- make coverage
- make docs
artifacts:
paths:
- htmlcov/
- docs/_build/html/
pages:
stage: deploy
dependencies:
- test_geomultisens
script:
- mkdir -p public/coverage
- cp -r htmlcov/* public/coverage/
- mkdir -p public/doc
- cp -r docs/_build/html/* public/doc/
artifacts:
paths:
- public
expire_in: 30 days
only:
- master
......@@ -51,16 +51,16 @@ lint: ## check style with flake8
flake8 geomultisens tests
test: ## run tests quickly with the default Python
python setup.py test
test-all: ## run tests on every Python version with tox
tox
coverage: ## check code coverage quickly with the default Python
coverage run --source geomultisens setup.py test
coverage report -m
coverage html
$(BROWSER) htmlcov/index.html
......@@ -86,4 +86,8 @@ dist: clean ## builds source and wheel package
ls -l dist
install: clean ## install the package to the active Python's site-packages
pip install -r requirements.txt
python setup.py install
gitlab_CI_docker: ## Build a docker image for CI use within gitlab
cd ./tests/CI_docker/; bash ./build_testsuite_image.sh
......@@ -27,7 +27,7 @@ import numpy as np
from geopandas import GeoDataFrame
from shapely.geometry import box
from CoReg_Sat import COREG, DESHIFTER
from CoReg_Sat import COREG, DESHIFTER # FIXME replace by arosics
from geoarray import GeoArray
from py_tools_ds.geo.coord_grid import is_coord_grid_equal
from py_tools_ds.geo.coord_calc import corner_coord_to_minmax
......
......@@ -2,13 +2,13 @@ from copy import copy
from random import sample
from operator import itemgetter
import random
from inspect import getargspec
from inspect import getargspec # FIXME
import numpy as np
from scipy.ndimage.filters import gaussian_filter
from sklearn.ensemble import AdaBoostClassifier
from sklearn.feature_selection import chi2
from sklearn.cross_validation import train_test_split
from sklearn.cross_validation import train_test_split # FIXME
__author__ = "Andre Hollstein"
......
......@@ -1265,6 +1265,10 @@ class GMS_object(Dataset):
def delete_tempFiles(self):
"""Delete all temporary files that have been written during GMS object processing.
"""
# TODO ensure that all GDAL dataset are closed before. Otherwise the system creates a .fuse_hiddenXXX...
# TODO right in the moment when we delete the data that are opened in GDAL
self.logger.info('Deleting temporary data...')
if sys.platform.startswith('linux'):
......
git+https://gitext.gfz-potsdam.de/danschef/py_tools_ds.git
git+https://gitext.gfz-potsdam.de/danschef/geoarray.git
git+https://gitext.gfz-potsdam.de/danschef/arosics.git
git+https://gitext.gfz-potsdam.de/hollstein/sicor.git
git+https://gitext.gfz-potsdam.de/hollstein/S2SCAPEM.git
git+https://gitext.gfz-potsdam.de/hollstein/S2MSI.git
matplotlib
numpy
scikit-learn
scipy
gdal
pyproj
shapely
pyephem
pyorbital
dill
pandas
numba
spectral
pyhdf
geopandas
iso8601
pyinstrument
geoalchemy2
sqlalchemy
......@@ -11,13 +11,17 @@ with open('HISTORY.rst') as history_file:
requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'pyephem', 'pyorbital', 'dill',
'pandas', 'numba', 'spectral', 'pyhdf', 'geopandas', 'iso8601', 'pyinstrument',
'pandas', 'numba', 'spectral', 'pyhdf', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
#'py_tools_ds', # pip install git+https://gitext.gfz-potsdam.de/danschef/py_tools_ds.git
#'geoarray', # pip install git+https://gitext.gfz-potsdam.de/danschef/geoarray.git
#'arosics', # pip install git+https://gitext.gfz-potsdam.de/danschef/arosics.git # FIXME replace CoReg_Sat by arosics
#'sicor', # pip install git+https://gitext.gfz-potsdam.de/hollstein/sicor.git
#'S2SCAPEM', # pip install git+https://gitext.gfz-potsdam.de/hollstein/S2SCAPEM.git
#'S2MSI', # pip install git+https://gitext.gfz-potsdam.de/hollstein/S2MSI.git
]
setup_requirements = [] # TODO(danschef): put setup requirements (distutils extensions, etc.) here
test_requirements = requirements+ ['coverage']
test_requirements = [
# TODO: put package test requirements here
]
# TODO add data files
setup(
name='geomultisens',
version='0.3.4',
......@@ -29,24 +33,20 @@ setup(
packages=find_packages(), # searches for packages with an __init__.py and returns them as properly formatted list
package_dir={'geomultisens':
'geomultisens'},
package_data={"database": ["geomultisens/database/*"]},
include_package_data=True,
install_requires=requirements,
scripts=["bin/run_gms.py"],
license="GNU General Public License v3",
zip_safe=False,
keywords='geomultisens',
classifiers=[
'Development Status :: 2 - Pre-Alpha',
'Development Status :: 4 - Beta',
'Intended Audience :: Developers',
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Natural Language :: English',
"Programming Language :: Python :: 2",
'Programming Language :: Python :: 2.6',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.4+',
],
test_suite='tests',
tests_require=test_requirements
......
#!/usr/bin/env bash
context_dir="./context"
dockerfile="geomultisens_ci.docker"
tag="geomultisens_ci:latest"
gitlab_runner="geomultisens_gitlab_CI_runner"
# get py_tools_ds project # TODO move this to setup.py as soon as package is public
rm -rf context/py_tools_ds
git clone https://gitext.gfz-potsdam.de/danschef/py_tools_ds.git ./context/py_tools_ds
# get geoarray project # TODO move this to setup.py as soon as package is public
rm -rf context/geoarray
git clone https://gitext.gfz-potsdam.de/danschef/geoarray.git ./context/geoarray
# get arosics project # TODO move this to setup.py as soon as package is public
rm -rf context/arosics
git clone https://gitext.gfz-potsdam.de/danschef/arosics.git ./context/arosics
# get sicor project # TODO move this to setup.py as soon as package is public
rm -rf context/sicor
git clone https://gitext.gfz-potsdam.de/hollstein/sicor.git ./context/sicor
# get S2SCAPEM project # TODO move this to setup.py as soon as package is public
rm -rf context/S2SCAPEM
git clone https://gitext.gfz-potsdam.de/hollstein/S2SCAPEM.git ./context/S2SCAPEM
# get S2MSI project # TODO move this to setup.py as soon as package is public
rm -rf context/S2MSI
git clone https://gitext.gfz-potsdam.de/hollstein/S2MSI.git ./context/S2MSI
echo "#### Build runner docker image"
sudo docker rmi ${tag}
sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir}
echo "#### Create gitlab-runner (daemon) container with tag; ${tag}"
sudo docker stop ${gitlab_runner}
sudo docker rm ${gitlab_runner}
sudo docker run -d --name ${gitlab_runner} --restart always \
-v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest
echo "#### Register container at gitlab, get token here https://gitext.gfz-potsdam.de/danschef/geomultisens/settings/ci_cd"
read -p "Please enter gitlab token: " token
echo ""
read -p "Please enter gitlab runner name: " runner_name
echo "New gitlab runner image will named ${gitlab_runner}"
sudo docker exec -it ${gitlab_runner} /bin/bash -c "export RUNNER_EXECUTOR=docker && gitlab-ci-multi-runner register -n \
--url 'https://gitext.gfz-potsdam.de/ci' \
--registration-token '${token}' \
--run-untagged=true \
--locked=true \
--tag-list geomultisens_ci_client \
--description '${runner_name}' \
--docker-image '${tag}' "
ls
FROM centos:7
RUN yum update -y && \
yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git # TODO
ENV anaconda_dl 'Anaconda3-4.3.1-Linux-x86_64.sh'
RUN /bin/bash -i -c "cd /root; wget https://repo.continuum.io/archive/$anaconda_dl ; \
bash -i /root/$anaconda_dl -b ; \
rm -f /root/$anaconda_dl"
# 'conda install --yes -c conda-forge 'icu=58.*' lxml' fixing a bug for conda-forge gdal build, as of 06/2017
RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
conda install --yes pyqt coverage ; \
conda install --yes -c conda-forge gdal ; \
conda install --yes -c conda-forge pyfftw=0.10.4 ; \
conda install --yes -c conda-forge 'icu=58.*' lxml ; \
conda install --yes -c conda-forge glymur pygrib rasterio pyproj cachetools ; \
pip install shapely geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse spectral" # must include all the requirements needed to build the docs!
# copy some needed stuff to /root
#COPY *.pkl /root/ # EXAMPLE
#COPY *.h5 /root/ # EXAMPLE
# copy py_tools_ds and geoarray code to /tmp
COPY py_tools_ds /tmp/py_tools_ds
COPY geoarray /tmp/geoarray
COPY arosics /tmp/arosics
COPY sicor /tmp/sicor
COPY sicor /tmp/S2SCAPEM
COPY sicor /tmp/S2MSI
# install git lfs
RUN /bin/bash -i -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | bash"
# installing git large file support, see here: https://git-lfs.github.com/ , change version number below for future upgrades
ENV git_lfs_v='2.1.1'
RUN /bin/bash -i -c "wget https://github.com/git-lfs/git-lfs/releases/download/v${git_lfs_v}/git-lfs-linux-amd64-${git_lfs_v}.tar.gz; \
tar -zxvf git-lfs-linux-amd64-${git_lfs_v}.tar.gz; \
cd git-lfs-${git_lfs_v}; \
bash ./install.sh"
# set matplotlib backend to Agg as default, Qt won't work in headless docker containers
RUN bash -i -c 'source /root/anaconda3/bin/activate; \
sed -i "/backend : tkagg/c\backend : Agg" $(python -c "import matplotlib; print(matplotlib.matplotlib_fname())"); \
sed -i "/backend : Qt4Agg/c\backend : Agg" $(python -c "import matplotlib; print(matplotlib.matplotlib_fname())"); \
sed -i "/backend : qt5agg/c\backend : Agg" $(python -c "import matplotlib; print(matplotlib.matplotlib_fname())")'
RUN bash -i -c "source /root/anaconda3/bin/activate; \
cd /tmp/py_tools_ds/; python setup.py install; \
cd /tmp/geoarray/; python setup.py install; \
cd /tmp/arosics/; python setup.py install; \
cd /tmp/sicor/; python setup.py install; \
cd /tmp/S2SCAPEM/; python setup.py install; \
cd /tmp/S2MSI/; python setup.py install; \
" # TODO move this to setup.py as soon as package is public
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment