Commit 4ef85479 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Revised SICOR wrapper to get Landsat AC to work.


Former-commit-id: a990e586
parent 1325dc8c
before_script:
- git lfs pull
# update sicor
- source /root/anaconda3/bin/activate
- export GDAL_DATA=/root/anaconda3/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- rm -rf context/sicor
- git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor
- cd ./context/sicor
- python setup.py install
stages:
......@@ -13,7 +21,7 @@ test_gms_preprocessing:
- source /root/anaconda3/bin/activate
- export GDAL_DATA=/root/anaconda3/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- pip install rednose
- pip install rednose # TODO remove as soon as docker container is rebuilt
- make nosetests
- make docs
artifacts:
......
......@@ -7,6 +7,8 @@ import logging
import dill
import traceback
from typing import List # noqa F401 # flake8 issue
from time import time
import os
import numpy as np
......@@ -24,6 +26,7 @@ from ..io.input_reader import SRF
from sicor.sicor_ac import ac_gms
from sicor.sensors import RSImage
from sicor.Mask import S2Mask
from sicor.ECMWF import download_variables
__author__ = 'Daniel Scheffler'
......@@ -432,7 +435,6 @@ class AtmCorr(object):
____________ key:YDIM, value_type:<class 'int'>, repr: -20
______ key:SENSING_TIME, value_type:<class 'datetime.datetime'>, repr: 2016-03-26 10:34:06.538000+00:00
"""
# TODO add SRF object
if not self._metadata:
del self.logger # otherwise each input object would have multiple fileHandlers
......@@ -718,6 +720,50 @@ class AtmCorr(object):
mask_legend=cm_legend,
geo_coding=cm_geocoding)
def _check_or_download_ECMWF_data(self):
"""Check if ECMWF files are already downloaded. If not, start the downloader."""
self.logger.info('Checking if ECMWF data are available... (if not, run download!)')
default_products = [
"fc_T2M",
"fc_O3",
"fc_SLP",
"fc_TCWV",
"fc_GMES_ozone",
"fc_total_AOT_550nm",
"fc_sulphate_AOT_550nm",
"fc_black_carbon_AOT_550nm",
"fc_dust_AOT_550nm",
"fc_organic_matter_AOT_550nm",
"fc_sea_salt_AOT_550nm"]
try:
t0 = time()
results = download_variables(date_from=self.inObjs[0].acq_datetime,
date_to=self.inObjs[0].acq_datetime,
db_path=CFG.job.path_ECMWF_db,
max_step=120, # default
ecmwf_variables=default_products,
processes=0, # singleprocessing
force=False) # dont force download if files already exist
t1 = time()
self.logger.info("Runtime: %.2f" % (t1 - t0))
for result in results:
self.logger.info(result)
except Exception as err:
self.logger.error("ECMWF data download failed for scene %s (entity ID: %s). Traceback: "
% (self.inObjs[0].scene_ID, self.inObjs[0].entity_ID))
self.logger.error(traceback.format_exc())
def _validate_snr_source(self):
"""Check if the given file path for the SNR model exists - if not, use a constant SNR of 500."""
if not os.path.isfile(self.options["uncertainties"]["snr_model"]):
warnings.warn('No valid SNR model found for %s %s. Using constant SNR to compute uncertainties of '
'atmospheric correction.' % (self.inObjs[0].satellite, self.inObjs[0].sensor))
# self.options["uncertainties"]["snr_model"] = np.nan # causes the computed uncertainties to be np.nan
self.options["uncertainties"]["snr_model"] = 500 # use a constant SNR of 500 to compute uncertainties
def run_atmospheric_correction(self, dump_ac_input=False):
# type: (bool) -> list
"""Collects all input data for atmospheric correction, runs the AC and returns the corrected L1C objects
......@@ -744,6 +790,12 @@ class AtmCorr(object):
script = False
# check if ECMWF data are available - if not, start the download
self._check_or_download_ECMWF_data()
# validate SNR
self._validate_snr_source()
# create an instance of RSImage
rs_image = RSImage(**rs_data)
......
......@@ -169,7 +169,6 @@ class Job(object):
self.path_ac_tables = self.DB_config['path_ac_tables']
self.path_SNR_models = self.DB_config['path_SNR_models']
self.path_dem_proc_srtm_90m = self.DB_config['path_dem_proc_srtm_90m']
self.path_ECMWF_db = self.DB_config['path_ECMWF_db']
if not self.is_test:
self.path_fileserver = self.DB_config['path_data_root']
......@@ -180,6 +179,7 @@ class Job(object):
self.path_SRFs = self.DB_config['path_SRFs']
self.path_cloud_classif = self.DB_config['path_cloud_classif']
self.path_solar_irr = self.DB_config['path_solar_irr']
self.path_ECMWF_db = self.DB_config['path_ECMWF_db']
self.path_testing = self.DB_config['path_testing']
self.path_benchmarks = self.DB_config['path_benchmarks']
self.path_job_logs = self.DB_config['path_job_logs']
......@@ -194,6 +194,7 @@ class Job(object):
self.path_cloud_classif = self.absP('./database/cloud_classifier/')
self.path_solar_irr = self.absP(
'./database/solar_irradiance/SUNp1fontenla__350-2500nm_@0.1nm_converted.txt')
self.path_ECMWF_db = self.absP('../tests/data/processed_ECMWF/')
self.path_testing = self.absP('./sandbox/') # FIXME
self.path_benchmarks = self.absP('./benchmarks/')
self.path_job_logs = self.absP('./logs/job_logs/')
......
......@@ -330,11 +330,17 @@ def get_path_ac_options(GMS_identifier):
fName_optFile = None
if fName_optFile:
# from sicor import options
# return os.path.join(os.path.dirname(options.__file__), fName_optFile)
# FIXME this is a temporary workaround for issue #6 of sicor
from gms_preprocessing import __file__
return os.path.join(os.path.dirname(__file__), 'database', 'sicor_options', fName_optFile)
try:
from sicor import options
path_ac = os.path.join(os.path.dirname(options.__file__), fName_optFile)
assert os.path.exists(path_ac)
except AssertionError:
# FIXME this is a temporary workaround for issue #6 of sicor
warnings.warn('Could not locate options file for atmopsheric correction within SICOR installation folder. '
'Using the one provided with gms_preprocessing (maybe outdated).')
from gms_preprocessing import __file__
path_ac = os.path.join(os.path.dirname(__file__), 'database', 'sicor_options', fName_optFile)
return path_ac
else:
return None
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment