.gitlab-ci.yml 4.35 KB
Newer Older
Daniel Scheffler's avatar
Daniel Scheffler committed
1
before_script:
Daniel Scheffler's avatar
Daniel Scheffler committed
2
  - git lfs pull
Daniel Scheffler's avatar
Daniel Scheffler committed
3

4

5 6 7 8 9 10 11
# Advise GitLab that these environment vars should be loaded from the Variables config.
variables:
    GMS_DB_HOST: SECURE
    GMS_INDEX_HOST: SECURE
    GMS_INDEX_PORT: SECURE


12 13 14 15 16
stages:
    - test
    - deploy


17
test_gms_preprocessing:
18
  stage: test
Daniel Scheffler's avatar
Daniel Scheffler committed
19
  script:
Daniel Scheffler's avatar
Daniel Scheffler committed
20
    - source /root/miniconda3/bin/activate ci_env
21 22 23
    - export GMS_db_host=${GMS_DB_HOST}
    - export GMS_index_host=${GMS_INDEX_HOST}
    - export GMS_index_port=${GMS_INDEX_PORT}
24

25 26 27 28
    - conda install -c conda-forge cartopy  # TODO remove as soon as docker container is rebuilt
    - pip install "geoarray>=0.9.0"  # TODO remove as soon as docker container is rebuilt
    - pip install "arosics>=1.0.0"  # TODO remove as soon as docker container is rebuilt

29 30
    # update sicor
    # - rm -rf context/sicor
31
    # - git clone https://git.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor
32 33 34 35
    # - cd ./context/sicor
    # - make download-tables
    # - python setup.py install
    # - cd ../../
Daniel Scheffler's avatar
Daniel Scheffler committed
36
    # make tests
37
    - pip install -U sicor
38
    # - python -c 'from sicor.tables import get_tables; get_tables(sensor="s2"); get_tables(sensor="l8"); get_tables(sensor="enmap", optional_downloads=("ch4",))'
39

40
    - pip check
41

42
    # run nosetests
43
    - make nosetests
44 45 46

    # create the docs
    - pip install -U sphinx_rtd_theme  # Read-the-docs theme for SPHINX documentation
47
    - pip install -U sphinx-autodoc-typehints
Daniel Scheffler's avatar
Daniel Scheffler committed
48 49 50 51 52
    - make docs
  artifacts:
    paths:
    - htmlcov/
    - docs/_build/html/
53 54
    - nosetests.html
    - nosetests.xml
55
    when: always
Daniel Scheffler's avatar
Daniel Scheffler committed
56

57

58 59 60 61 62 63 64 65 66
test_urls:
  stage: test
  script:
    - source /root/miniconda3/bin/activate ci_env
    - pip install -U urlchecker
    - make urlcheck
  when: always


67 68 69
test_gms_preprocessing_install:
  stage: test
  script:
Daniel Scheffler's avatar
Daniel Scheffler committed
70
    - source /root/miniconda3/bin/activate
71 72

    # update conda and python
73 74 75 76 77 78
    - conda update -c conda-forge conda
    # - conda update -c conda-forge python

    # create gms_test environment
    - conda create -y -q -c conda-forge --name gms_test python=3
    - conda activate gms_test
79 80 81 82 83

    # avoid package incompatibilities
    # - conda config --set channel_priority strict  # otherwise gdal or libgdal may be installed from defaults channel

    # install some dependencies that cause trouble when installed via pip
84
    - conda install -y -c conda-forge numpy gdal scikit-image pyproj geopandas ipython matplotlib cartopy scikit-learn=0.23.2 shapely pyhdf python-fmask holoviews
85 86 87 88 89

    # install not pip-installable deps of arosics
    - conda install -y -c conda-forge pyfftw pykrige

    # install not pip-installable deps of sicor
90
    - conda install -y -c conda-forge glymur pygrib cachetools pyhdf h5py pytables llvmlite numba
91 92 93

    # install gms_preprocessing
    - pip install -e .
94
    - pip check
95 96 97 98 99 100 101 102 103 104 105 106
    - cd ..
    - pwd
    - ls

    # test importability
    - python -c "import gms_preprocessing; print(gms_preprocessing)"
    - python -c "from gms_preprocessing import ProcessController; print(ProcessController)"
  only:
    - master
    - dev


107 108 109
test_styles:
  stage: test
  script:
Daniel Scheffler's avatar
Daniel Scheffler committed
110
    - source /root/miniconda3/bin/activate ci_env
111 112 113 114 115 116 117 118 119
    - make lint
  artifacts:
    paths:
    - tests/linting/flake8.log
    - tests/linting/pycodestyle.log
    - tests/linting/pydocstyle.log
    when: always


120
pages:  # this job must be called 'pages' to advise GitLab to upload content to GitLab Pages
Daniel Scheffler's avatar
Daniel Scheffler committed
121 122
  stage: deploy
  dependencies:
123
    - test_gms_preprocessing
Daniel Scheffler's avatar
Daniel Scheffler committed
124
  script:
125 126 127 128
    # Create the public directory
    - rm -rf public
    - mkdir public
    - mkdir -p public/doc
Daniel Scheffler's avatar
Daniel Scheffler committed
129
    - mkdir -p public/coverage
Daniel Scheffler's avatar
Daniel Scheffler committed
130
    - mkdir -p public/nosetests_reports
131

132
    # Copy over the docs
Daniel Scheffler's avatar
Daniel Scheffler committed
133
    - cp -r docs/_build/html/* public/doc/
134

135 136
    # Copy over the coverage reports
    - cp -r htmlcov/* public/coverage/
137

138 139
    # Copy over the nosetests reports
    - cp nosetests.* public/nosetests_reports/
140

141 142 143 144 145
    # Check if everything is working great
    - ls -al public
    - ls -al public/doc
    - ls -al public/coverage
    - ls -al public/nosetests_reports
Daniel Scheffler's avatar
Daniel Scheffler committed
146 147 148
  artifacts:
    paths:
      - public
149
    expire_in: 10 days
Daniel Scheffler's avatar
Daniel Scheffler committed
150 151
  only:
    - master
152
    - enhancement/improve_docs
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167


deploy_pypi:
  stage: deploy
  dependencies:
    - test_gms_preprocessing
  script:
    - source /root/miniconda3/bin/activate ci_env
    - pip install -U twine
    - python setup.py sdist
    - twine upload dist/*  # requires creds as environment variables
  only:
    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
  except:
    - dev