Commit 7766738d authored by Pablo Iturrieta Rebolledo's avatar Pablo Iturrieta Rebolledo
Browse files

Modified filepaths to quickly accesss stored results and Figs, according to...

Modified filepaths to quickly accesss stored results and Figs, according to TestName/Years/SpecificType
Reordered results folder structure
Modified run() functions for all tests.to include the modifications
Modified catalog accesss
Pending:
#  Complete get_multiscore_fig.py to be compatible with the changes above
#  Decide if Metadata is needed to be removed from Models=[CSEPGriddedForecast, Metadata] when importing.
# Create final figures, and go to spatialentropy
parent a9f3534b
......@@ -4,6 +4,7 @@ from matplotlib.lines import Line2D
from csep.core import poisson_evaluations as poisson
from csep.utils import plots
import numpy as np
import itertools
import copy
import scipy.stats
import pickle
......@@ -67,7 +68,7 @@ def plot_results(Results, years, savepath=False):
ax.legend(handles=legend_elements, loc='lower left')
if savepath:
plt.savefig(os.path.join(savepath, 'CL_%i_.png' % years), dpi=300)
plt.savefig(os.path.join(savepath), dpi=300)
plt.show()
......@@ -93,7 +94,7 @@ def plot_merged_batchs(batch, savepath):
ax.legend(handles=legend_elements, loc='lower left')
if savepath:
plt.savefig(os.path.join(savepath, 'CL_5&10_.png' ), dpi=300)
plt.savefig(os.path.join(savepath), dpi=300)
plt.show()
......@@ -108,25 +109,51 @@ def get_score(batch):
return Scores
def get_ref_IG(batch):
Scores = []
ref = 'MPS04after'
for i in batch:
if i.sim_name == ref:
ref_stat = i.observed_statistic
def run(use_saved=False):
Models5 = get_models.get_models_5yr()
cat5 = get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008())
Models10 = get_models.get_models_10yr()
cat10 = get_catalogs.filter_10yr_01_2010(get_catalogs.buolletino())
for R in batch:
score = R.observed_statistic - ref_stat
Scores.append([R.sim_name, score])
print(R.sim_name, score)
if not use_saved:
Results5 = process_forecasts_batch([i[0] for i in Models5], cat5, save_obj=filepaths.test_path['CL_5'])
Results10 = process_forecasts_batch([i[0] for i in Models10], cat10, save_obj=filepaths.test_path['CL_10'])
else:
Results5 = process_forecasts_batch(None, None, load_obj=filepaths.test_path['CL_5'])
Results10 = process_forecasts_batch(None, None, load_obj=filepaths.test_path['CL_10'])
return Scores
plot_results(Results5, years=5, savepath=filepaths.csep_figs)
plot_results(Results10, years=10, savepath=filepaths.csep_figs)
plot_merged_batchs(merge_results_batchs(Results10, Results5), savepath=filepaths.csep_figs)
get_score(Results5)
def run(use_saved=False):
years = [5, 10]
test = 'CL'
types = ['']
models = {5: get_models.get_models_5yr(),
10: get_models.get_models_10yr()}
catalogs = {5: get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008()),
10: get_catalogs.filter_10yr_01_2010(get_catalogs.bollettino())}
Results = {}
for yr, type in itertools.product(years, types):
result_path = filepaths.get_csep_result_path(test, yr, type)
if use_saved:
Result = process_forecasts_batch([i[0] for i in models[yr]], catalogs[yr], load_obj=result_path)
else:
Result = process_forecasts_batch([i[0] for i in models[yr]], catalogs[yr], save_obj=result_path)
Results[yr] = Result
plot_results(Result, years=yr, savepath=filepaths.get_csep_fig_path(test, yr, type))
plot_merged_batchs(merge_results_batchs(Results[years[0]], Results[years[1]]),
savepath=filepaths.get_csep_fig_path(test, "%i-%i" % tuple(years), types[0]))
print('Scores 5yr')
get_score(Results[5])
print('Scores 10yr')
get_score(Results[10])
print('Information Gain 5yr')
get_ref_IG(Results[5])
print('Information Gain 10yr')
get_ref_IG(Results[10])
if __name__ == '__main__':
......
......@@ -6,6 +6,7 @@ from csep.utils import plots
import numpy as np
import copy
import scipy.stats
import itertools
import pickle
import os
from codes import get_catalogs, get_models, filepaths
......@@ -69,7 +70,7 @@ def plot_results(Results, years, savepath=False):
ax.legend(handles=legend_elements, loc='lower left')
if savepath:
plt.savefig(os.path.join(savepath, 'L_%i.png' % years), dpi=300)
plt.savefig(os.path.join(savepath), dpi=300)
plt.show()
......@@ -95,7 +96,7 @@ def plot_merged_batchs(batch, savepath):
ax.legend(handles=legend_elements, loc='lower left')
if savepath:
plt.savefig(os.path.join(savepath, 'L_5&10.png' ), dpi=300)
plt.savefig(os.path.join(savepath), dpi=300)
plt.show()
......@@ -110,7 +111,7 @@ def get_score(batch, catalog):
return Scores
def get_IG(batch):
def get_ref_IG(batch):
Scores = []
ref = 'MPS04after'
for i in batch:
......@@ -125,24 +126,36 @@ def get_IG(batch):
return Scores
def run(use_saved=False):
Models5 = get_models.get_models_5yr()
cat5 = get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008())
Models10 = get_models.get_models_10yr()
cat10 = get_catalogs.filter_10yr_01_2010(get_catalogs.buolletino())
if not use_saved:
Results5 = process_forecasts_batch([i[0] for i in Models5], cat5, save_obj=filepaths.test_path['L_5'])
Results10 = process_forecasts_batch([i[0] for i in Models10], cat10, save_obj=filepaths.test_path['L_10'])
else:
Results5 = process_forecasts_batch(None, None, load_obj=filepaths.test_path['L_5'])
Results10 = process_forecasts_batch(None, None, load_obj=filepaths.test_path['L_10'])
plot_results(Results5, years=5, savepath=filepaths.csep_figs)
plot_results(Results10, years=10, savepath=filepaths.csep_figs)
plot_merged_batchs(merge_results_batchs(Results10, Results5), savepath=filepaths.csep_figs)
get_score(Results5, cat5)
get_IG(Results5)
years = [5, 10]
test = 'L'
types = ['']
models = {5: get_models.get_models_5yr(),
10: get_models.get_models_10yr()}
catalogs = {5: get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008()),
10: get_catalogs.filter_10yr_01_2010(get_catalogs.bollettino())}
Results = {}
for yr, type in itertools.product(years, types):
result_path = filepaths.get_csep_result_path(test, yr, type)
if use_saved:
Result = process_forecasts_batch([i[0] for i in models[yr]], catalogs[yr], load_obj=result_path)
else:
Result = process_forecasts_batch([i[0] for i in models[yr]], catalogs[yr], save_obj=result_path)
Results[yr] = Result
plot_results(Result, years=yr, savepath=filepaths.get_csep_fig_path(test, yr, type))
plot_merged_batchs(merge_results_batchs(Results[years[0]], Results[years[1]]),
savepath=filepaths.get_csep_fig_path(test, "%i-%i" % tuple(years), types[0]))
print('Scores 5yr')
get_score(Results[5], catalogs[5])
print('Scores 10yr')
get_score(Results[10], catalogs[10])
print('Information Gain 5yr')
get_ref_IG(Results[5])
print('Information Gain 10yr')
get_ref_IG(Results[10])
if __name__ == '__main__':
use_saved = False
......
......@@ -5,6 +5,7 @@ from csep.core import poisson_evaluations as poisson
from csep.utils import plots
import numpy as np
import copy
import itertools
import scipy.stats
import pickle
import os
......@@ -52,7 +53,7 @@ def plot_results(Results, years, savepath=False):
ax.legend(handles=legend_elements, loc='lower left')
if savepath:
plt.savefig(os.path.join(savepath, 'M_%i.png' % years), dpi=300)
plt.savefig(os.path.join(savepath), dpi=300)
plt.show()
......@@ -95,7 +96,7 @@ def plot_merged_batchs(batch, savepath):
ax.legend(handles=legend_elements, loc='lower left')
if savepath:
plt.savefig(os.path.join(savepath, 'M_5&10.png' ), dpi=300)
plt.savefig(os.path.join(savepath), dpi=300)
plt.show()
......@@ -111,38 +112,35 @@ def get_score(batch):
return Scores
def run(used_saved=False):
Models5 = get_models.get_models_5yr()
cat5 = get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008())
def run(use_saved=False):
years = [5, 10]
test = 'M'
types = ['']
Models10 = get_models.get_models_10yr()
cat10 = get_catalogs.filter_10yr_01_2010(get_catalogs.buolletino())
models = {5 : get_models.get_models_5yr(),
10 : get_models.get_models_10yr()}
catalogs = {5 : get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008()),
10 : get_catalogs.filter_10yr_01_2010(get_catalogs.bollettino())}
Results = {}
if not used_saved:
Results5 = process_forecasts_batch([i[0] for i in Models5], cat5, save_obj=filepaths.test_path['M_5'])
Results10 = process_forecasts_batch([i[0] for i in Models10], cat10, save_obj=filepaths.test_path['M_10'])
else:
Results5 = process_forecasts_batch(None, None, load_obj=filepaths.test_path['M_5'])
Results10 = process_forecasts_batch(None, None, load_obj=filepaths.test_path['M_10'])
for yr, type in itertools.product(years, types):
result_path = filepaths.get_csep_result_path(test, yr, type)
if use_saved:
Result = process_forecasts_batch([i[0] for i in models[yr]],catalogs[yr], load_obj=result_path)
else:
Result = process_forecasts_batch([i[0] for i in models[yr]],catalogs[yr], save_obj=result_path)
Results[yr] = Result
plot_results(Result, years=yr, savepath=filepaths.get_csep_fig_path(test, yr, type))
plot_results(Results5, years=5, savepath=filepaths.csep_figs)
plot_results(Results10, years=10, savepath=filepaths.csep_figs)
plot_merged_batchs(merge_results_batchs(Results10, Results5), savepath=filepaths.csep_figs)
get_score(Results5)
plot_merged_batchs(merge_results_batchs(Results[years[0]], Results[years[1]]),
savepath=filepaths.get_csep_fig_path(test, "%i-%i" % tuple(years), types[0]))
print('Scores 5yr')
get_score(Results[5])
print('Scores 10yr')
get_score(Results[10])
if __name__ == '__main__':
used_saved = False
run(used_saved)
use_saved = False
run(use_saved)
......@@ -4,6 +4,7 @@ from matplotlib.lines import Line2D
from csep.core import poisson_evaluations as poisson
from csep.utils import plots
import numpy as np
import itertools
import pickle
import copy
import os
......@@ -70,7 +71,7 @@ def plot_results(Results, years, savepath=False):
ax.legend(handles=legend_elements, loc='lower right')
if savepath:
plt.savefig(os.path.join(savepath, 'N_%i.png' % years), dpi=300)
plt.savefig(os.path.join(savepath), dpi=300)
plt.show()
def plot_merged_batchs(batch, savepath=False):
......@@ -99,7 +100,7 @@ def plot_merged_batchs(batch, savepath=False):
ax.legend(handles=legend_elements, loc='lower right')
if savepath:
plt.savefig(os.path.join(savepath, 'N_5&10.png'), dpi = 300)
plt.savefig(os.path.join(savepath), dpi = 300)
plt.show()
......@@ -113,29 +114,47 @@ def get_score(batch):
return Scores
def run(used_saved=False):
Models5 = get_models.get_models_5yr()
cat5 = get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008())
Models10 = get_models.get_models_10yr()
cat10 = get_catalogs.filter_10yr_01_2010(get_catalogs.buolletino())
if not used_saved:
Results5 = process_forecasts_batch([i[0] for i in Models5], cat5,save_obj=filepaths.test_path['N_5']) # todo, models should be dict with items Model/metadata
Results10 = process_forecasts_batch([i[0] for i in Models10], cat10, save_obj=filepaths.test_path['N_10'])
else:
Results5 = process_forecasts_batch(None, None, load_obj=filepaths.test_path['N_5'])
Results10 = process_forecasts_batch(None, None, load_obj=filepaths.test_path['N_10'])
plot_results(Results5, years=5, savepath=filepaths.csep_figs)
plot_results(Results10, years=10, savepath=filepaths.csep_figs)
plot_merged_batchs(merge_results_batchs(Results10, Results5), savepath=filepaths.csep_figs)
get_score(Results5)
def run(use_saved=False):
"""
R
:param use_saved:
:return:
"""
years = [5, 10]
test = 'N'
types = [''] ## N-test does not have different types
## Get Models and Catalogs as dictionaries
models = {5 : get_models.get_models_5yr(),
10 : get_models.get_models_10yr()}
catalogs = {5 : get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008()),
10 : get_catalogs.filter_10yr_01_2010(get_catalogs.bollettino())}
## Iterate through years and types to get the results
Results = {}
for yr, type in itertools.product(years, types):
## Get the name of the python object where to save results, or to load them
result_path = filepaths.get_csep_result_path(test, yr, type)
if use_saved:
Result = process_forecasts_batch([i[0] for i in models[yr]],catalogs[yr], load_obj=result_path)
else:
Result = process_forecasts_batch([i[0] for i in models[yr]],catalogs[yr], save_obj=result_path)
Results[yr] = Result
plot_results(Result, years=yr, savepath=filepaths.get_csep_fig_path(test, yr, type))
plot_merged_batchs(merge_results_batchs(Results[years[0]], Results[years[1]]),
savepath=filepaths.get_csep_fig_path(test, "%i-%i" % tuple(years), types[0]))
print('Scores 5yr')
get_score(Results[5])
print('Scores 10yr')
get_score(Results[10])
if __name__ == '__main__':
used_saved = False
a =run(used_saved=True)
use_saved = False
a =run(use_saved)
......@@ -6,7 +6,7 @@ import os
import numpy as np
import scipy.stats
from codes import get_catalogs, get_models, filepaths
import itertools
def pgs(Forecasts, Catalog, ref='avg', bin='spatial_magnitude', save_obj=False, load_obj=False):
......@@ -88,7 +88,7 @@ def plot_pgs(Results, years, ref, savepath=False):
ax.legend(handles=legend_elements, loc='lower right')
fig.tight_layout()
if savepath:
fig.savefig(os.path.join(savepath, 'pgs_%iyr_%s.png' % (years, ref)) , dpi=300 )
fig.savefig(os.path.join(savepath) , dpi=300 )
fig.show()
def plot_merged_batchs(Results1, Results2, ref, savepath=False):
......@@ -125,27 +125,53 @@ def plot_merged_batchs(Results1, Results2, ref, savepath=False):
ax.legend(handles=legend_elements, loc='lower right')
fig.tight_layout()
if savepath:
fig.savefig(os.path.join(savepath, 'pgs_both_%s.png' % ref) , dpi=300 )
fig.savefig(os.path.join(savepath) , dpi=300 )
fig.show()
def run():
models5 = get_models.get_models_5yr()
models10 = get_models.get_models_10yr()
catalog5 = get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008())
catalog10 = get_catalogs.filter_10yr_01_2010(get_catalogs.buolletino())
results5 = pgs(models5, catalog5, bin='spatial_magnitude', ref=11)
plot_pgs(results5, 5, 'MPS04after', savepath=filepaths.csep_figs)
results10 = pgs(models10, catalog10, bin='spatial_magnitude', ref=11)
plot_pgs(results10, 10, 'MPS04after', savepath=filepaths.csep_figs)
plot_merged_batchs(results10, results5, 'MPS04after', savepath=filepaths.csep_figs)
results5 = pgs(models5, catalog5, bin='spatial_magnitude', ref='avg', save_obj=filepaths.test_path['PGS_5'])
plot_pgs(results5, 5, 'Average', savepath=filepaths.csep_figs)
results10 = pgs(models10, catalog10, bin='spatial_magnitude', ref='avg', save_obj=filepaths.test_path['PGS_10'])
plot_pgs(results10, 10, 'Average', savepath=filepaths.csep_figs)
plot_merged_batchs(results10, results5, 'Average', savepath=filepaths.csep_figs)
def run(use_saved=False):
# models5 = get_models.get_models_5yr()
# models10 = get_models.get_models_10yr()
# catalog5 = get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008())
# catalog10 = get_catalogs.filter_10yr_01_2010(get_catalogs.bollettino())
#
# results5 = pgs(models5, catalog5, bin='spatial_magnitude', ref=11)
# plot_pgs(results5, 5, 'MPS04after', savepath=filepaths.csep_figs)
# results10 = pgs(models10, catalog10, bin='spatial_magnitude', ref=11)
# plot_pgs(results10, 10, 'MPS04after', savepath=filepaths.csep_figs)
# plot_merged_batchs(results10, results5, 'MPS04after', savepath=filepaths.csep_figs)
#
# results5 = pgs(models5, catalog5, bin='spatial_magnitude', ref='avg', save_obj=filepaths.res_path['PGS_5'])
# plot_pgs(results5, 5, 'Average', savepath=filepaths.csep_figs)
# results10 = pgs(models10, catalog10, bin='spatial_magnitude', ref='avg', save_obj=filepaths.res_path['PGS_10'])
# plot_pgs(results10, 10, 'Average', savepath=filepaths.csep_figs)
# plot_merged_batchs(results10, results5, 'Average', savepath=filepaths.csep_figs)
years = [5, 10]
test = 'PGS'
types = ['avg', 11]
models = {5 : get_models.get_models_5yr(),
10 : get_models.get_models_10yr()}
catalogs = {5 : get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008()),
10 : get_catalogs.filter_10yr_01_2010(get_catalogs.bollettino())}
Results = {}
for yr, typ in itertools.product(years, types):
result_path = filepaths.get_csep_result_path(test, yr, typ)
if use_saved:
Result = pgs(models[yr],catalogs[yr], ref=typ, load_obj=result_path)
else:
Result = pgs(models[yr], catalogs[yr], ref=typ, save_obj=result_path)
Results[(yr, typ)] = Result
if typ==int:
typ = models[yr][typ][0].name
plot_pgs(Result, yr, typ, savepath=filepaths.get_csep_fig_path(test, yr, 'ref%s'% typ))
for typ in types:
plot_merged_batchs(Results[(10, typ)], Results[(5, typ)], typ,
savepath=filepaths.get_csep_fig_path(test, '%i-%i' % (5,10), typ))
if __name__ == '__main__':
run()
import n_test
import m_test
import s_test
import l_test
import cl_test
import t_test
import pgs
use_saved = False
# n_test.run(use_saved)
# m_test.run(use_saved)
# s_test.run(use_saved)
# l_test.run(use_saved)
# cl_test.run(use_saved)
# t_test.run(use_saved)
pgs.run(use_saved)
......@@ -9,7 +9,7 @@ import scipy.stats
import pickle
import os
from codes import get_catalogs, get_models, filepaths
import itertools
def process_forecasts_batch(Model_batch, catalog, save_obj=False, load_obj=False):
......@@ -67,7 +67,7 @@ def plot_results(Results, years, savepath=False):
ax.legend(handles=legend_elements, loc='lower left')
if savepath:
plt.savefig(os.path.join(savepath, 'S_%i.png' % years), dpi=300)
plt.savefig(os.path.join(savepath), dpi=300)
plt.show()
......@@ -93,7 +93,7 @@ def plot_merged_batchs(batch, savepath):
ax.legend(handles=legend_elements, loc='lower left')
if savepath:
plt.savefig(os.path.join(savepath, 'S_5&10.png' ), dpi=300)
plt.savefig(os.path.join(savepath), dpi=300)
plt.show()
......@@ -109,23 +109,31 @@ def get_score(batch):
return Scores
def run(use_saved=False):
Models5 = get_models.get_models_5yr()
cat5 = get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008())
Models10 = get_models.get_models_10yr()
cat10 = get_catalogs.filter_10yr_01_2010(get_catalogs.buolletino())
if not use_saved:
Results5 = process_forecasts_batch([i[0] for i in Models5], cat5, save_obj=filepaths.test_path['S_5'])
Results10 = process_forecasts_batch([i[0] for i in Models10], cat10, save_obj=filepaths.test_path['S_10'])
else:
Results5 = process_forecasts_batch(None, None, load_obj=filepaths.test_path['S_5'])
Results10 = process_forecasts_batch(None, None, load_obj=filepaths.test_path['S_10'])
plot_results(Results5, years=5, savepath=filepaths.csep_figs)
plot_results(Results10, years=10, savepath=filepaths.csep_figs)
plot_merged_batchs(merge_results_batchs(Results10, Results5), savepath=filepaths.csep_figs)
get_score(Results5)
years = [5, 10]
test = 'S'
types = ['']
models = {5: get_models.get_models_5yr(),
10: get_models.get_models_10yr()}
catalogs = {5: get_catalogs.filter_5yr_08_2009(get_catalogs.taroni2008()),
10: get_catalogs.filter_10yr_01_2010(get_catalogs.bollettino())}
Results = {}
for yr, type in itertools.product(years, types):
result_path = filepaths.get_csep_result_path(test, yr, type)
if use_saved:
Result = process_forecasts_batch([i[0] for i in models[yr]], catalogs[yr], load_obj=result_path)
else:
Result = process_forecasts_batch([i[0] for i in models[yr]], catalogs[yr], save_obj=result_path)
Results[yr] = Result
plot_results(Result, years=yr, savepath=filepaths.get_csep_fig_path(test, yr, type))
plot_merged_batchs(merge_results_batchs(Results[years[0]], Results[years[1]]),
savepath=filepaths.get_csep_fig_path(test, "%i-%i" % tuple(years), types[0]))
print('Scores 5yr')
get_score(Results[5])
print('Scores 10yr')
get_score(Results[10])
if __name__ == '__main__':
......
......@@ -26,9 +26,9 @@ def process_forecasts_batch(Model_batch, catalog, save_obj=False, load_obj=False
for f_i, f_j in itertools.product(Model_batch, Model_batch):
T_ij = poisson.paired_t_test(f_j[0], f_i[0], catalog)
T_ij = poisson.paired_t_test(f_j, f_i, catalog)
Results_T.append(T_ij)
W_ij = poisson.w_test(f_j[0], f_i[0], catalog)
W_ij = poisson.w_test(f_j, f_i, catalog)
Results_W.append(W_ij)
if save_obj:
......@@ -42,7 +42,6 @@ def process_forecasts_batch(Model_batch, catalog, save_obj=False, load_obj=False
raise Exception('Results object not found')
return Results_T, Results_W
def merge_results_batchs(batch1, batch2):
L_merged = []
for j in batch1:
......@@ -57,7 +56,7 @@ def merge_results_batchs(batch1, batch2):
L_merged.append(Forecast_10)
return L_merged
def plot_single_results(Results, Forecasts, years, ref_model=0,savepath=False):
def plot_single_results(Results, Forecasts, years, ref_model=0,folder=False):
plot_args = {'title': 'Paired T-test & W-test\n %i years experiment\nBase Model: %s' % (years, Forecasts[ref_model][0].name) ,
'figsize':(6,8),
'xlabel': '',
......@@ -65,7 +64,7 @@ def plot_single_results(Results, Forecasts, years, ref_model=0,savepath=False):
'linewidth': 1.5,
'capsize': 0,
'markersize': 6,
'ylims': (-2,2)}
'ylims': (-3,3)}
T_Results_per_model = Results[0][ref_model*len(Forecasts):(ref_model+1)*(len(Forecasts))]
......@@ -77,12 +76,12 @@ def plot_single_results(Results, Forecasts, years, ref_model=0,savepath=False):
markerfacecolor='white', markeredgecolor='black', markersize=5)
]
ax.legend(handles=legend_elements, loc='lower right')
if savepath:
plt.savefig(os.path.join(savepath, 'TW_%i_ref%s.png' % (years, Forecasts[ref_model][0].name)), dpi=300)