Commit 2b5ea868 authored by Daniel Scheffler's avatar Daniel Scheffler

Added URL checker and corresponding CI job. Removed travis related files. Fixed dead URLs.

Signed-off-by: Daniel Scheffler's avatarDaniel Scheffler <danschef@gfz-potsdam.de>
parent 6586fe0e
Pipeline #16720 failed with stages
in 20 minutes and 41 seconds
......@@ -55,6 +55,15 @@ test_gms_preprocessing:
when: always
test_urls:
stage: test
script:
- source /root/miniconda3/bin/activate ci_env
- pip install -U urlchecker
- make urlcheck
when: always
test_gms_preprocessing_install:
stage: test
script:
......
# Config file for automatic testing at travis-ci.org
# This file will be regenerated if you run travis_pypi_setup.py
language: python
python: 3.5
env:
- TOXENV=py35
- TOXENV=py34
- TOXENV=py33
- TOXENV=py27
- TOXENV=py26
- TOXENV=pypy
# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
install: pip install -U tox
# command to run tests, e.g. python setup.py test
script: tox -e ${TOXENV}
# After you create the Github repo and add it to Travis, run the
# travis_pypi_setup.py script to finish PyPI deployment setup
deploy:
provider: pypi
distributions: sdist bdist_wheel
user: geomultisens
password:
secure: PLEASE_REPLACE_ME
on:
tags: true
repo: geomultisens/gms_preprocessing
condition: $TOXENV == py27
......@@ -2,10 +2,13 @@
History
=======
0.19.4 (coming soon)
--------------------
0.19.4 (2020-12-10)
-------------------
* Added some docs regarding the PostgreSQL database and the pre-defined virtual sensors.
* Added URL checker and corresponding CI job.
* Removed travis related files.
* Fixed dead URLs.
0.19.3 (2020-12-03)
......
......@@ -55,6 +55,11 @@ lint: ## check style with flake8
pycodestyle gms_preprocessing --exclude="*.ipynb,*.ipynb*" --max-line-length=120 > ./tests/linting/pycodestyle.log
-pydocstyle gms_preprocessing > ./tests/linting/pydocstyle.log
urlcheck: ## check for dead URLs
urlchecker check . \
--file-types .py,.rst,.md,.json \
--white-listed-patterns S2_PDI_Level-1C_Tile_Metadata.xsd,Sensor+Wavelength+Definitions
test: ## run tests quickly with the default Python
python setup.py test
......
......@@ -16,17 +16,6 @@ Landsat-5, Landsat-7, Landsat-8, Sentinel-2A and Sentinel-2B.
Status
------
.. .. image:: https://img.shields.io/travis/geomultisens/gms_preprocessing.svg
:target: https://travis-ci.org/geomultisens/gms_preprocessing
.. .. image:: https://readthedocs.org/projects/gms_preprocessing/badge/?version=latest
:target: https://gms_preprocessing.readthedocs.io/en/latest/?badge=latest
:alt: Documentation Status
.. .. image:: https://pyup.io/repos/github/geomultisens/gms_preprocessing/shield.svg
:target: https://pyup.io/repos/github/geomultisens/gms_preprocessing/
:alt: Updates
.. image:: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/badges/master/pipeline.svg
:target: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/commits/master
.. image:: https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/badges/master/coverage.svg
......
......@@ -410,7 +410,9 @@ class L1A_object(GMS_object):
elif arr_desc == 'TOA_Ref':
if conv == 'Rad':
"""http://s2tbx.telespazio-vega.de/sen2three/html/r2rusage.html?highlight=quantification182
"""
https://sentinel.esa.int/web/sentinel/technical-guides/sentinel-2-msi/level-1c/algorithm
https://forum.step.esa.int/t/radiometric-conversion-of-sentinel-2-images/1123/
rToa = (float)(DN_L1C_band / QUANTIFICATION_VALUE);
L = (rToa * e0__SOLAR_IRRADIANCE_For_band * cos(Z__Sun_Angles_Grid_Zenith_Values)) /
(PI * U__earth_sun_distance_correction_factor);
......
......@@ -662,7 +662,7 @@ class METADATA(object):
'7': 2215.}
if re.search(r"[\D]*7", self.Satellite):
# Landsat 7; resource for center wavelength:
# http://opticks.org/confluence/display/opticksDev/Sensor+Wavelength+Definitions
# https://opticks.org/display/opticksDev/Sensor+Wavelength+Definitions
# 6L(thermal),6H(thermal),B8(pan)
self.SolIrradiance = {'1': 1969.,
'2': 1840.,
......@@ -699,11 +699,11 @@ class METADATA(object):
# Thermal Constants (K1 = [W*m-2*um-1]; K1 = [K])
if re.search(r"[\D]*5", self.Satellite):
# resource: http://www.yale.edu/ceo/Documentation/Landsat_DN_to_Kelvin.pdf
# resource: http://geography.middlebury.edu/data/gg1002/Handouts/Landsat_DN_Temp.pdf
ThermalConstK1 = [0., 0., 0., 0., 0., 607.76, 0.]
ThermalConstK2 = [0., 0., 0., 0., 0., 1260.56, 0.]
if re.search(r"[\D]*7", self.Satellite):
# resource: http://www.yale.edu/ceo/Documentation/Landsat_DN_to_Kelvin.pdf
# resource: http://geography.middlebury.edu/data/gg1002/Handouts/Landsat_DN_Temp.pdf
ThermalConstK1 = [0., 0., 0., 0., 0., 666.09, 666.09, 0., 0.]
ThermalConstK2 = [0., 0., 0., 0., 0., 1282.71, 1282.71, 0., 0.]
if re.search(r"[\D]*8", self.Satellite): # Landsat 8
......
......@@ -82,7 +82,7 @@ req = [
req_setup = ['setuptools-git'] # needed for package_data version controlled by GIT
req_test = ['coverage', 'nose', 'nose2', 'nose-htmloutput', 'rednose']
req_test = ['coverage', 'nose', 'nose2', 'nose-htmloutput', 'rednose', 'urlchecker']
req_doc = ['sphinx-autodoc-typehint', 'sphinx-argparse', 'sphinx_rtd_theme']
......
......@@ -62,3 +62,4 @@ dependencies:
- sphinx-autodoc-typehints
- sphinx-argparse
- sphinx_rtd_theme
- urlchecker
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# gms_preprocessing, spatial and spectral homogenization of satellite remote sensing data
#
# Copyright (C) 2020 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
#
# This software was developed within the context of the GeoMultiSens project funded
# by the German Federal Ministry of Education and Research
# (project grant code: 01 IS 14 010 A-C).
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later version.
# Please note the following exception: `gms_preprocessing` depends on tqdm, which
# is distributed under the Mozilla Public Licence (MPL) v2.0 except for the files
# "tqdm/_tqdm.py", "setup.py", "README.rst", "MANIFEST.in" and ".gitignore".
# Details can be found here: https://github.com/tqdm/tqdm/blob/master/LICENCE.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""Update encrypted deploy password in Travis config file
"""
from __future__ import print_function
import base64
import json
import os
from getpass import getpass
import yaml
from cryptography.hazmat.primitives.serialization import load_pem_public_key
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15
try:
from urllib import urlopen
except:
from urllib.request import urlopen
GITHUB_REPO = 'geomultisens/gms_preprocessing'
TRAVIS_CONFIG_FILE = os.path.join(
os.path.dirname(os.path.abspath(__file__)), '.travis.yml')
def load_key(pubkey):
"""Load public RSA key, with work-around for keys using
incorrect header/footer format.
Read more about RSA encryption with cryptography:
https://cryptography.io/latest/hazmat/primitives/asymmetric/rsa/
"""
try:
return load_pem_public_key(pubkey.encode(), default_backend())
except ValueError:
# workaround for https://github.com/travis-ci/travis-api/issues/196
pubkey = pubkey.replace('BEGIN RSA', 'BEGIN').replace('END RSA', 'END')
return load_pem_public_key(pubkey.encode(), default_backend())
def encrypt(pubkey, password):
"""Encrypt password using given RSA public key and encode it with base64.
The encrypted password can only be decrypted by someone with the
private key (in this case, only Travis).
"""
key = load_key(pubkey)
encrypted_password = key.encrypt(password, PKCS1v15())
return base64.b64encode(encrypted_password)
def fetch_public_key(repo):
"""Download RSA public key Travis will use for this repo.
Travis API docs: http://docs.travis-ci.com/api/#repository-keys
"""
keyurl = 'https://api.travis-ci.org/repos/{0}/key'.format(repo)
data = json.loads(urlopen(keyurl).read().decode())
if 'key' not in data:
errmsg = "Could not find public key for repo: {}.\n".format(repo)
errmsg += "Have you already added your GitHub repo to Travis?"
raise ValueError(errmsg)
return data['key']
def prepend_line(filepath, line):
"""Rewrite a file adding a line to its beginning.
"""
with open(filepath) as f:
lines = f.readlines()
lines.insert(0, line)
with open(filepath, 'w') as f:
f.writelines(lines)
def load_yaml_config(filepath):
with open(filepath) as f:
return yaml.load(f)
def save_yaml_config(filepath, config):
with open(filepath, 'w') as f:
yaml.dump(config, f, default_flow_style=False)
def update_travis_deploy_password(encrypted_password):
"""Update the deploy section of the .travis.yml file
to use the given encrypted password.
"""
config = load_yaml_config(TRAVIS_CONFIG_FILE)
config['deploy']['password'] = dict(secure=encrypted_password)
save_yaml_config(TRAVIS_CONFIG_FILE, config)
line = ('# This file was autogenerated and will overwrite'
' each time you run travis_pypi_setup.py\n')
prepend_line(TRAVIS_CONFIG_FILE, line)
def main(args):
public_key = fetch_public_key(args.repo)
password = args.password or getpass('PyPI password: ')
update_travis_deploy_password(encrypt(public_key, password.encode()))
print("Wrote encrypted password to .travis.yml -- you're ready to deploy")
if '__main__' == __name__:
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--repo', default=GITHUB_REPO,
help='GitHub repo (default: %s)' % GITHUB_REPO)
parser.add_argument('--password',
help='PyPI password (will prompt if not provided)')
args = parser.parse_args()
main(args)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment