Added URL checker and corresponding CI job. Removed travis related files. Fixed dead URLs.

Signed-off-by: Daniel Scheffler <danschef@gfz-potsdam.de>

.gitlab-ci.yml

@@ -55,6 +55,15 @@ test_gms_preprocessing:
     when: always
 
 
+test_urls:
+  stage: test
+  script:
+    - source /root/miniconda3/bin/activate ci_env
+    - pip install -U urlchecker
+    - make urlcheck
+  when: always
+
+
 test_gms_preprocessing_install:
   stage: test
   script:

.travis.yml

-# Config file for automatic testing at travis-ci.org
-# This file will be regenerated if you run travis_pypi_setup.py
-
-language: python
-python: 3.5
-
-env:
-  - TOXENV=py35
-  - TOXENV=py34
-  - TOXENV=py33
-  - TOXENV=py27
-  - TOXENV=py26
-  - TOXENV=pypy
-
-# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
-install: pip install -U tox
-
-# command to run tests, e.g. python setup.py test
-script: tox -e ${TOXENV}
-
-# After you create the Github repo and add it to Travis, run the
-# travis_pypi_setup.py script to finish PyPI deployment setup
-deploy:
-  provider: pypi
-  distributions: sdist bdist_wheel
-  user: geomultisens
-  password:
-    secure: PLEASE_REPLACE_ME
-  on:
-    tags: true
-    repo: geomultisens/gms_preprocessing
-    condition: $TOXENV == py27

HISTORY.rst

@@ -2,10 +2,13 @@
 History
 =======
 
-0.19.4 (coming soon)
---------------------
+0.19.4 (2020-12-10)
+-------------------
 
 * Added some docs regarding the PostgreSQL database and the pre-defined virtual sensors.
+* Added URL checker and corresponding CI job.
+* Removed travis related files.
+* Fixed dead URLs.
 
 
 0.19.3 (2020-12-03)

Makefile

@@ -55,6 +55,11 @@ lint: ## check style with flake8
 	pycodestyle gms_preprocessing --exclude="*.ipynb,*.ipynb*" --max-line-length=120 > ./tests/linting/pycodestyle.log
 	-pydocstyle gms_preprocessing > ./tests/linting/pydocstyle.log
 
+urlcheck: ## check for dead URLs
+	urlchecker check . \
+		--file-types .py,.rst,.md,.json \
+		--white-listed-patterns S2_PDI_Level-1C_Tile_Metadata.xsd,Sensor+Wavelength+Definitions
+
 test: ## run tests quickly with the default Python
 	python setup.py test
 

README.rst

@@ -16,17 +16,6 @@ Landsat-5, Landsat-7, Landsat-8, Sentinel-2A and Sentinel-2B.
 Status
 ------
 
-.. .. image:: https://img.shields.io/travis/geomultisens/gms_preprocessing.svg
-        :target: https://travis-ci.org/geomultisens/gms_preprocessing
-
-.. .. image:: https://readthedocs.org/projects/gms_preprocessing/badge/?version=latest
-        :target: https://gms_preprocessing.readthedocs.io/en/latest/?badge=latest
-        :alt: Documentation Status
-
-.. .. image:: https://pyup.io/repos/github/geomultisens/gms_preprocessing/shield.svg
-     :target: https://pyup.io/repos/github/geomultisens/gms_preprocessing/
-     :alt: Updates
-
 .. image:: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/badges/master/pipeline.svg
         :target: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/commits/master
 .. image:: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/badges/master/coverage.svg

gms_preprocessing/algorithms/L1A_P.py

@@ -410,7 +410,9 @@ class L1A_object(GMS_object):
 
             elif arr_desc == 'TOA_Ref':
                 if conv == 'Rad':
-                    """http://s2tbx.telespazio-vega.de/sen2three/html/r2rusage.html?highlight=quantification182
+                    """
+                    https://sentinel.esa.int/web/sentinel/technical-guides/sentinel-2-msi/level-1c/algorithm
+                    https://forum.step.esa.int/t/radiometric-conversion-of-sentinel-2-images/1123/
                     rToa = (float)(DN_L1C_band / QUANTIFICATION_VALUE);
                     L = (rToa * e0__SOLAR_IRRADIANCE_For_band * cos(Z__Sun_Angles_Grid_Zenith_Values)) /
                         (PI * U__earth_sun_distance_correction_factor);

gms_preprocessing/model/metadata.py

@@ -662,7 +662,7 @@ class METADATA(object):
                             '7': 2215.}
             if re.search(r"[\D]*7", self.Satellite):
                 # Landsat 7; resource for center wavelength:
-                # http://opticks.org/confluence/display/opticksDev/Sensor+Wavelength+Definitions
+                # https://opticks.org/display/opticksDev/Sensor+Wavelength+Definitions
                 # 6L(thermal),6H(thermal),B8(pan)
                 self.SolIrradiance = {'1': 1969.,
                                       '2': 1840.,
@@ -699,11 +699,11 @@ class METADATA(object):
 
         # Thermal Constants (K1 = [W*m-2*um-1]; K1 = [K])
         if re.search(r"[\D]*5", self.Satellite):
-            # resource: http://www.yale.edu/ceo/Documentation/Landsat_DN_to_Kelvin.pdf
+            # resource: http://geography.middlebury.edu/data/gg1002/Handouts/Landsat_DN_Temp.pdf
             ThermalConstK1 = [0., 0., 0., 0., 0., 607.76, 0.]
             ThermalConstK2 = [0., 0., 0., 0., 0., 1260.56, 0.]
         if re.search(r"[\D]*7", self.Satellite):
-            # resource: http://www.yale.edu/ceo/Documentation/Landsat_DN_to_Kelvin.pdf
+            # resource: http://geography.middlebury.edu/data/gg1002/Handouts/Landsat_DN_Temp.pdf
             ThermalConstK1 = [0., 0., 0., 0., 0., 666.09, 666.09, 0., 0.]
             ThermalConstK2 = [0., 0., 0., 0., 0., 1282.71, 1282.71, 0., 0.]
         if re.search(r"[\D]*8", self.Satellite):  # Landsat 8

setup.py

@@ -82,7 +82,7 @@ req = [
 
 req_setup = ['setuptools-git']  # needed for package_data version controlled by GIT
 
-req_test = ['coverage', 'nose', 'nose2', 'nose-htmloutput', 'rednose']
+req_test = ['coverage', 'nose', 'nose2', 'nose-htmloutput', 'rednose', 'urlchecker']
 
 req_doc = ['sphinx-autodoc-typehint', 'sphinx-argparse', 'sphinx_rtd_theme']
 

tests/CI_docker/context/environment_gms_preprocessing.yml

@@ -62,3 +62,4 @@ dependencies:
     - sphinx-autodoc-typehints
     - sphinx-argparse
     - sphinx_rtd_theme
+    - urlchecker

travis_pypi_setup.py

-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-# gms_preprocessing, spatial and spectral homogenization of satellite remote sensing data
-#
-# Copyright (C) 2020  Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
-#
-# This software was developed within the context of the GeoMultiSens project funded
-# by the German Federal Ministry of Education and Research
-# (project grant code: 01 IS 14 010 A-C).
-#
-# This program is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free Software
-# Foundation, either version 3 of the License, or (at your option) any later version.
-# Please note the following exception: `gms_preprocessing` depends on tqdm, which
-# is distributed under the Mozilla Public Licence (MPL) v2.0 except for the files
-# "tqdm/_tqdm.py", "setup.py", "README.rst", "MANIFEST.in" and ".gitignore".
-# Details can be found here: https://github.com/tqdm/tqdm/blob/master/LICENCE.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
-# details.
-#
-# You should have received a copy of the GNU Lesser General Public License along
-# with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Update encrypted deploy password in Travis config file
-"""
-
-
-from __future__ import print_function
-import base64
-import json
-import os
-from getpass import getpass
-import yaml
-from cryptography.hazmat.primitives.serialization import load_pem_public_key
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15
-
-
-try:
-    from urllib import urlopen
-except:
-    from urllib.request import urlopen
-
-
-GITHUB_REPO = 'geomultisens/gms_preprocessing'
-TRAVIS_CONFIG_FILE = os.path.join(
-    os.path.dirname(os.path.abspath(__file__)), '.travis.yml')
-
-
-def load_key(pubkey):
-    """Load public RSA key, with work-around for keys using
-    incorrect header/footer format.
-
-    Read more about RSA encryption with cryptography:
-    https://cryptography.io/latest/hazmat/primitives/asymmetric/rsa/
-    """
-    try:
-        return load_pem_public_key(pubkey.encode(), default_backend())
-    except ValueError:
-        # workaround for https://github.com/travis-ci/travis-api/issues/196
-        pubkey = pubkey.replace('BEGIN RSA', 'BEGIN').replace('END RSA', 'END')
-        return load_pem_public_key(pubkey.encode(), default_backend())
-
-
-def encrypt(pubkey, password):
-    """Encrypt password using given RSA public key and encode it with base64.
-
-    The encrypted password can only be decrypted by someone with the
-    private key (in this case, only Travis).
-    """
-    key = load_key(pubkey)
-    encrypted_password = key.encrypt(password, PKCS1v15())
-    return base64.b64encode(encrypted_password)
-
-
-def fetch_public_key(repo):
-    """Download RSA public key Travis will use for this repo.
-
-    Travis API docs: http://docs.travis-ci.com/api/#repository-keys
-    """
-    keyurl = 'https://api.travis-ci.org/repos/{0}/key'.format(repo)
-    data = json.loads(urlopen(keyurl).read().decode())
-    if 'key' not in data:
-        errmsg = "Could not find public key for repo: {}.\n".format(repo)
-        errmsg += "Have you already added your GitHub repo to Travis?"
-        raise ValueError(errmsg)
-    return data['key']
-
-
-def prepend_line(filepath, line):
-    """Rewrite a file adding a line to its beginning.
-    """
-    with open(filepath) as f:
-        lines = f.readlines()
-
-    lines.insert(0, line)
-
-    with open(filepath, 'w') as f:
-        f.writelines(lines)
-
-
-def load_yaml_config(filepath):
-    with open(filepath) as f:
-        return yaml.load(f)
-
-
-def save_yaml_config(filepath, config):
-    with open(filepath, 'w') as f:
-        yaml.dump(config, f, default_flow_style=False)
-
-
-def update_travis_deploy_password(encrypted_password):
-    """Update the deploy section of the .travis.yml file
-    to use the given encrypted password.
-    """
-    config = load_yaml_config(TRAVIS_CONFIG_FILE)
-
-    config['deploy']['password'] = dict(secure=encrypted_password)
-
-    save_yaml_config(TRAVIS_CONFIG_FILE, config)
-
-    line = ('# This file was autogenerated and will overwrite'
-            ' each time you run travis_pypi_setup.py\n')
-    prepend_line(TRAVIS_CONFIG_FILE, line)
-
-
-def main(args):
-    public_key = fetch_public_key(args.repo)
-    password = args.password or getpass('PyPI password: ')
-    update_travis_deploy_password(encrypt(public_key, password.encode()))
-    print("Wrote encrypted password to .travis.yml -- you're ready to deploy")
-
-
-if '__main__' == __name__:
-    import argparse
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('--repo', default=GITHUB_REPO,
-                        help='GitHub repo (default: %s)' % GITHUB_REPO)
-    parser.add_argument('--password',
-                        help='PyPI password (will prompt if not provided)')
-
-    args = parser.parse_args()
-    main(args)