diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 48f5591c9e9f12cca96e5f757315c9a378a6acad..e6659a4fb260bafbed4ae8ee90fe1c10bd2f6761 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,15 @@ test_gms_preprocessing: when: always +test_urls: + stage: test + script: + - source /root/miniconda3/bin/activate ci_env + - pip install -U urlchecker + - make urlcheck + when: always + + test_gms_preprocessing_install: stage: test script: diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0e9b423d8ff46d328e93578a05acf27edf04fce3..0000000000000000000000000000000000000000 --- a/.travis.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Config file for automatic testing at travis-ci.org -# This file will be regenerated if you run travis_pypi_setup.py - -language: python -python: 3.5 - -env: - - TOXENV=py35 - - TOXENV=py34 - - TOXENV=py33 - - TOXENV=py27 - - TOXENV=py26 - - TOXENV=pypy - -# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: pip install -U tox - -# command to run tests, e.g. python setup.py test -script: tox -e ${TOXENV} - -# After you create the Github repo and add it to Travis, run the -# travis_pypi_setup.py script to finish PyPI deployment setup -deploy: - provider: pypi - distributions: sdist bdist_wheel - user: geomultisens - password: - secure: PLEASE_REPLACE_ME - on: - tags: true - repo: geomultisens/gms_preprocessing - condition: $TOXENV == py27 diff --git a/HISTORY.rst b/HISTORY.rst index 2ea3df395b23b4f3e67d61508c83d8314266fa99..da4a3f2fdb89790b6c704eba4b48e6dc5394a6bb 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,10 +2,13 @@ History ======= -0.19.4 (coming soon) --------------------- +0.19.4 (2020-12-10) +------------------- * Added some docs regarding the PostgreSQL database and the pre-defined virtual sensors. +* Added URL checker and corresponding CI job. +* Removed travis related files. +* Fixed dead URLs. 0.19.3 (2020-12-03) diff --git a/Makefile b/Makefile index 24e2a3ed9c3c3ee97a69d7f947dd483f4bccdfed..c66325ee88a8d43ffbcc00e88f3ea1c53ff655a4 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,11 @@ lint: ## check style with flake8 pycodestyle gms_preprocessing --exclude="*.ipynb,*.ipynb*" --max-line-length=120 > ./tests/linting/pycodestyle.log -pydocstyle gms_preprocessing > ./tests/linting/pydocstyle.log +urlcheck: ## check for dead URLs + urlchecker check . \ + --file-types .py,.rst,.md,.json \ + --white-listed-patterns S2_PDI_Level-1C_Tile_Metadata.xsd,Sensor+Wavelength+Definitions + test: ## run tests quickly with the default Python python setup.py test diff --git a/README.rst b/README.rst index 6400f15033f199bf1ed6c8664f8d6c3c92739f83..517c61458ff7128b86b9607dd36123f71992519c 100644 --- a/README.rst +++ b/README.rst @@ -16,17 +16,6 @@ Landsat-5, Landsat-7, Landsat-8, Sentinel-2A and Sentinel-2B. Status ------ -.. .. image:: https://img.shields.io/travis/geomultisens/gms_preprocessing.svg - :target: https://travis-ci.org/geomultisens/gms_preprocessing - -.. .. image:: https://readthedocs.org/projects/gms_preprocessing/badge/?version=latest - :target: https://gms_preprocessing.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status - -.. .. image:: https://pyup.io/repos/github/geomultisens/gms_preprocessing/shield.svg - :target: https://pyup.io/repos/github/geomultisens/gms_preprocessing/ - :alt: Updates - .. image:: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/badges/master/pipeline.svg :target: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/commits/master .. image:: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/badges/master/coverage.svg diff --git a/gms_preprocessing/algorithms/L1A_P.py b/gms_preprocessing/algorithms/L1A_P.py index ad22f662b14fad7fe0ed53b3434c5ae9f1dcc942..23f1f40b997bd125bbdc10c3512c4e3fd5640823 100644 --- a/gms_preprocessing/algorithms/L1A_P.py +++ b/gms_preprocessing/algorithms/L1A_P.py @@ -410,7 +410,9 @@ class L1A_object(GMS_object): elif arr_desc == 'TOA_Ref': if conv == 'Rad': - """http://s2tbx.telespazio-vega.de/sen2three/html/r2rusage.html?highlight=quantification182 + """ + https://sentinel.esa.int/web/sentinel/technical-guides/sentinel-2-msi/level-1c/algorithm + https://forum.step.esa.int/t/radiometric-conversion-of-sentinel-2-images/1123/ rToa = (float)(DN_L1C_band / QUANTIFICATION_VALUE); L = (rToa * e0__SOLAR_IRRADIANCE_For_band * cos(Z__Sun_Angles_Grid_Zenith_Values)) / (PI * U__earth_sun_distance_correction_factor); diff --git a/gms_preprocessing/model/metadata.py b/gms_preprocessing/model/metadata.py index 278cc8307e1906337bd231ecae51e54d74b7f55a..42e81621f4c3788f3699f9ad1da891396e966bdd 100644 --- a/gms_preprocessing/model/metadata.py +++ b/gms_preprocessing/model/metadata.py @@ -662,7 +662,7 @@ class METADATA(object): '7': 2215.} if re.search(r"[\D]*7", self.Satellite): # Landsat 7; resource for center wavelength: - # http://opticks.org/confluence/display/opticksDev/Sensor+Wavelength+Definitions + # https://opticks.org/display/opticksDev/Sensor+Wavelength+Definitions # 6L(thermal),6H(thermal),B8(pan) self.SolIrradiance = {'1': 1969., '2': 1840., @@ -699,11 +699,11 @@ class METADATA(object): # Thermal Constants (K1 = [W*m-2*um-1]; K1 = [K]) if re.search(r"[\D]*5", self.Satellite): - # resource: http://www.yale.edu/ceo/Documentation/Landsat_DN_to_Kelvin.pdf + # resource: http://geography.middlebury.edu/data/gg1002/Handouts/Landsat_DN_Temp.pdf ThermalConstK1 = [0., 0., 0., 0., 0., 607.76, 0.] ThermalConstK2 = [0., 0., 0., 0., 0., 1260.56, 0.] if re.search(r"[\D]*7", self.Satellite): - # resource: http://www.yale.edu/ceo/Documentation/Landsat_DN_to_Kelvin.pdf + # resource: http://geography.middlebury.edu/data/gg1002/Handouts/Landsat_DN_Temp.pdf ThermalConstK1 = [0., 0., 0., 0., 0., 666.09, 666.09, 0., 0.] ThermalConstK2 = [0., 0., 0., 0., 0., 1282.71, 1282.71, 0., 0.] if re.search(r"[\D]*8", self.Satellite): # Landsat 8 diff --git a/setup.py b/setup.py index 19700b4609a2c0526cc99320c33fe53a98390d45..f7adc88c5100cfdee7e7a57c35f02a8eec6b585f 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ req = [ req_setup = ['setuptools-git'] # needed for package_data version controlled by GIT -req_test = ['coverage', 'nose', 'nose2', 'nose-htmloutput', 'rednose'] +req_test = ['coverage', 'nose', 'nose2', 'nose-htmloutput', 'rednose', 'urlchecker'] req_doc = ['sphinx-autodoc-typehint', 'sphinx-argparse', 'sphinx_rtd_theme'] diff --git a/tests/CI_docker/context/environment_gms_preprocessing.yml b/tests/CI_docker/context/environment_gms_preprocessing.yml index 4e4e8d4ef49300bbcccdd51718563b4568950422..6fd971d142f7fc4ecadabeff36dcb2a5aa6f9934 100644 --- a/tests/CI_docker/context/environment_gms_preprocessing.yml +++ b/tests/CI_docker/context/environment_gms_preprocessing.yml @@ -62,3 +62,4 @@ dependencies: - sphinx-autodoc-typehints - sphinx-argparse - sphinx_rtd_theme + - urlchecker diff --git a/travis_pypi_setup.py b/travis_pypi_setup.py deleted file mode 100644 index 7435709047dbd0b2557550153acebd2f0e9c5dc8..0000000000000000000000000000000000000000 --- a/travis_pypi_setup.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# gms_preprocessing, spatial and spectral homogenization of satellite remote sensing data -# -# Copyright (C) 2020 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de) -# -# This software was developed within the context of the GeoMultiSens project funded -# by the German Federal Ministry of Education and Research -# (project grant code: 01 IS 14 010 A-C). -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation, either version 3 of the License, or (at your option) any later version. -# Please note the following exception: `gms_preprocessing` depends on tqdm, which -# is distributed under the Mozilla Public Licence (MPL) v2.0 except for the files -# "tqdm/_tqdm.py", "setup.py", "README.rst", "MANIFEST.in" and ".gitignore". -# Details can be found here: https://github.com/tqdm/tqdm/blob/master/LICENCE. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License along -# with this program. If not, see . - -"""Update encrypted deploy password in Travis config file -""" - - -from __future__ import print_function -import base64 -import json -import os -from getpass import getpass -import yaml -from cryptography.hazmat.primitives.serialization import load_pem_public_key -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 - - -try: - from urllib import urlopen -except: - from urllib.request import urlopen - - -GITHUB_REPO = 'geomultisens/gms_preprocessing' -TRAVIS_CONFIG_FILE = os.path.join( - os.path.dirname(os.path.abspath(__file__)), '.travis.yml') - - -def load_key(pubkey): - """Load public RSA key, with work-around for keys using - incorrect header/footer format. - - Read more about RSA encryption with cryptography: - https://cryptography.io/latest/hazmat/primitives/asymmetric/rsa/ - """ - try: - return load_pem_public_key(pubkey.encode(), default_backend()) - except ValueError: - # workaround for https://github.com/travis-ci/travis-api/issues/196 - pubkey = pubkey.replace('BEGIN RSA', 'BEGIN').replace('END RSA', 'END') - return load_pem_public_key(pubkey.encode(), default_backend()) - - -def encrypt(pubkey, password): - """Encrypt password using given RSA public key and encode it with base64. - - The encrypted password can only be decrypted by someone with the - private key (in this case, only Travis). - """ - key = load_key(pubkey) - encrypted_password = key.encrypt(password, PKCS1v15()) - return base64.b64encode(encrypted_password) - - -def fetch_public_key(repo): - """Download RSA public key Travis will use for this repo. - - Travis API docs: http://docs.travis-ci.com/api/#repository-keys - """ - keyurl = 'https://api.travis-ci.org/repos/{0}/key'.format(repo) - data = json.loads(urlopen(keyurl).read().decode()) - if 'key' not in data: - errmsg = "Could not find public key for repo: {}.\n".format(repo) - errmsg += "Have you already added your GitHub repo to Travis?" - raise ValueError(errmsg) - return data['key'] - - -def prepend_line(filepath, line): - """Rewrite a file adding a line to its beginning. - """ - with open(filepath) as f: - lines = f.readlines() - - lines.insert(0, line) - - with open(filepath, 'w') as f: - f.writelines(lines) - - -def load_yaml_config(filepath): - with open(filepath) as f: - return yaml.load(f) - - -def save_yaml_config(filepath, config): - with open(filepath, 'w') as f: - yaml.dump(config, f, default_flow_style=False) - - -def update_travis_deploy_password(encrypted_password): - """Update the deploy section of the .travis.yml file - to use the given encrypted password. - """ - config = load_yaml_config(TRAVIS_CONFIG_FILE) - - config['deploy']['password'] = dict(secure=encrypted_password) - - save_yaml_config(TRAVIS_CONFIG_FILE, config) - - line = ('# This file was autogenerated and will overwrite' - ' each time you run travis_pypi_setup.py\n') - prepend_line(TRAVIS_CONFIG_FILE, line) - - -def main(args): - public_key = fetch_public_key(args.repo) - password = args.password or getpass('PyPI password: ') - update_travis_deploy_password(encrypt(public_key, password.encode())) - print("Wrote encrypted password to .travis.yml -- you're ready to deploy") - - -if '__main__' == __name__: - import argparse - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--repo', default=GITHUB_REPO, - help='GitHub repo (default: %s)' % GITHUB_REPO) - parser.add_argument('--password', - help='PyPI password (will prompt if not provided)') - - args = parser.parse_args() - main(args)