Commit 9af0f80b authored by Pablo Iturrieta Rebolledo's avatar Pablo Iturrieta Rebolledo
Browse files

updated brier score and multi score figure

parent 0af670d8
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="PublishConfigData" autoUpload="Always" serverName="pciturri@rs1:22" remoteFilesAllowedToDisappearOnAutoupload="false">
<serverData>
<paths name="pciturri@rs1:22">
<serverdata>
<mappings>
<mapping deploy="/home/pciturri/pycharmprojects/italy-experiment-testing" local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
</serverData>
<component name="PublishConfigData" autoUpload="Always" remoteFilesAllowedToDisappearOnAutoupload="false">
<option name="myAutoUpload" value="ALWAYS" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (venv)" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (venv)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="PySciProjectComponent">
<option name="PY_SCI_VIEW" value="true" />
<option name="PY_SCI_VIEW_SUGGESTED" value="true" />
</component>
</project>
\ No newline at end of file
......@@ -166,7 +166,7 @@ def run(use_saved=False, catalog_name='bollettino'):
get_ref_IG(Results[10])
if __name__ == '__main__':
use_saved = False
use_saved = True
catalogs = ['bollettino', 'emrcmt', 'horus']
for cat in catalogs:
run(use_saved, cat)
......
......@@ -148,7 +148,7 @@ def run(use_saved=False, catalog_name='bollettino'):
if __name__ == '__main__':
use_saved = True
catalogs = ['bollettino', 'emrcmt', 'horus'][:1]
catalogs = ['bollettino', 'emrcmt', 'horus']
for cat in catalogs:
run(use_saved, cat)
......
......@@ -81,7 +81,7 @@ def plot_all_results(Results, years, p=0.05, order=False, savepath=False):
names = [i[0].sim_name[1] for i in Results[0]]
T_value = np.array([[T_i.observed_statistic for T_i in Ref_model] for Ref_model in Results[0]]).T
T_quantile = np.array([[T_i.test_distribution[0] for T_i in Ref_model] for Ref_model in Results[0]]).T
T_quantile = np.array([[np.abs(T_i.quantile[0]) - T_i.quantile[1] for T_i in Ref_model] for Ref_model in Results[0]]).T
W_quantile = np.array([[W_i.quantile for W_i in Ref_model] for Ref_model in Results[1]]).T
score = np.sum(T_value, axis=1)/T_value.shape[0]
......@@ -102,11 +102,15 @@ def plot_all_results(Results, years, p=0.05, order=False, savepath=False):
ax.set_yticklabels([names[i] for i in arg_ind], rotation='horizontal')
ax.set_xticklabels([names[i] for i in arg_ind],rotation='vertical')
for n, i in enumerate(data_tq):
for m, j in enumerate(i):
if j > 0:
ax.scatter(n + 0.5, m + 0.5, marker='o', s=75, facecolor='None', edgecolor='black')
for n, i in enumerate(data_w):
for m, j in enumerate(i):
if j < p:
ax.scatter(n + 0.5, m + 0.5,marker='o', s=2, color='black')
ax.scatter(n + 0.5, m + 0.5, marker='o', s=5, color='black')
# plt.text(len(score) + 2.5, -0.2, '$G_t$', fontsize=20)
# for i, j in enumerate([score[i] for i in arg_ind]):
......@@ -114,8 +118,11 @@ def plot_all_results(Results, years, p=0.05, order=False, savepath=False):
ax.set_title('Information Gain $T-$test')
legend_elements = [Line2D([0], [0], marker='o', lw=0, label='w-significant',
markerfacecolor='black', markeredgecolor='black', markersize=1)]
legend_elements = [Line2D([0], [0], marker='o', lw=0, label='$W$-significant',
markerfacecolor="black", markeredgecolor='black', markersize=4),
Line2D([0], [0], marker='o', lw=0, label='$T$-significant',
markerfacecolor="None", markeredgecolor='black', markersize=10)
]
fig.legend(handles=legend_elements, loc='lower right',
bbox_to_anchor=(0.79, 0.08, 0.2, 0.2), handletextpad=0)
plt.tight_layout()
......@@ -123,9 +130,9 @@ def plot_all_results(Results, years, p=0.05, order=False, savepath=False):
plt.show()
def run(use_saved=False, catalog_name='bollettino'):
years = [5, 10]
years = [10]
test = 'T'
types = [True, False] ## Ordered IG matrix
types = [True] ## Ordered IG matrix
models = {5: get_models.five_years(),
......@@ -152,10 +159,10 @@ def run(use_saved=False, catalog_name='bollettino'):
tw_figs_path = os.path.join(paths.csep_figs, 'tw_single')
os.makedirs(tw_figs_path, exist_ok=True)
for yr in years:
for model_i in range(len(models[yr])):
plot_single_results(Results[yr], years=yr, ref_model=model_i,
folder=tw_figs_path)
# for yr in years:
# for model_i in range(len(models[yr])):
# plot_single_results(Results[yr], years=yr, ref_model=model_i,
# folder=tw_figs_path)
for yr, typ in itertools.product(years, types):
plot_all_results(Results[yr], years=yr, order=typ,
savepath=paths.get_csep_figpath(test, yr, type=catalog_name + typ * 'order'))
......@@ -165,6 +172,6 @@ if __name__ =='__main__':
use_saved = True
catalogs = ['bollettino', 'emrcmt', 'horus']
for cat in catalogs:
for cat in catalogs[:1]:
run(use_saved, cat)
......@@ -6,16 +6,10 @@ import matplotlib.pyplot as plt
import numpy as np
import matplotlib.patches as patches
from matplotlib.lines import Line2D
import seaborn
seaborn.set_style("white", {"axes.facecolor": ".9", 'font.family': 'Ubuntu'})
def sig_exp(num_str):
parts = num_str.split('.', 2)
decimal = parts[1] if len(parts) > 1 else ''
exp = -len(decimal)
digits = parts[0].lstrip('0') + decimal
trimmed = digits.rstrip('0')
exp += len(digits) - len(trimmed)
sig = int(trimmed) if trimmed else 0
return sig, exp
def norm(array):
return (array - array.min())/(array.max() -array.min())
......@@ -49,10 +43,10 @@ def plot_axis(axis, n_results, offset, end_theta, n, min_y,
for i in yticks:
val = (i - min_y) * (array.max() - array.min()) + array.min()
if 'e' in format:
# val = format % val
print(val)
val, exp = sig_exp(str(val))
val = str(val)
val = format % val
# val, exp = sig_exp(str(val))
val, exp = str(val).split('e')
else:
val = format % val
if low_bound and i == yticks[0]:
......@@ -60,6 +54,8 @@ def plot_axis(axis, n_results, offset, end_theta, n, min_y,
axis.text(axis_angle, i - 0.03, val,
rotation=np.rad2deg(axis_angle) + 90, color=color,
ha='center', va='center', fontsize=fontsize)
axis.text(axis_angle, i + 0.05, '%s' % label,
color=color, ha='left', va='center', fontsize=10)
# axis.text(axis_angle, i + 0.15, label, color=color,
# ha='center', va='center', fontsize=14)
......@@ -96,6 +92,7 @@ def plot_scores(arrays, colors, result_labels, model_labels,
theta = np.linspace(0.0, end_theta, n_results * n_models, endpoint=False)
# Rearange results and colors
tuple_arrays = tuple([norm(i) for i in arrays])
score_total = np.vstack(tuple_arrays).T.ravel()
......@@ -111,6 +108,7 @@ def plot_scores(arrays, colors, result_labels, model_labels,
# Plot Data
ax.bar(theta + width/2., score_total, width=width, bottom=min_y,
color=colors_full, alpha=1, zorder=0)
......@@ -123,26 +121,26 @@ def plot_scores(arrays, colors, result_labels, model_labels,
yticks = plot_rticks(ax, min_y, ny)
# todo make generic
print(angle_offset)
new_labels = ['Log-likelihood', None, 'Brier score \n ($\cdot\,10^{-4}$)', None]
for k, i in enumerate([0, 2]):
plot_axis(ax, 2, offset, end_theta, k, min_y, arrays[i], yticks, lowbounds[i],
color_array[i], result_labels[i], fontsize, format[i])
color_array[i], new_labels[i], fontsize, format[i])
return ax
def plot_legends(Axes, colors, labels):
legend_elements = [Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[0]),
lw=5, label=labels[0]),
lw=4, label=labels[0]),
Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[1]),
lw=5, label=labels[1]),
lw=4, label=labels[1]),
Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[2]),
lw=5, label=labels[2]),
lw=4, label=labels[2]),
Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[3]),
lw=5, label=labels[3]),
lw=4, label=labels[3]),
Line2D([0], [0], color=colors[4],
lw=0, marker='o', markersize=15, label=labels[4])]
lw=0, marker='o', markersize=10, label=labels[4])]
Axes.get_figure().legend(handles=legend_elements, loc='lower right')
Axes.get_figure().legend(handles=legend_elements, loc='lower right', fontsize=9)
return Axes
def plot_results(years, labels, p=0.01, lowcuts=False, show=True,
......@@ -160,7 +158,6 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True,
ll_label = r'$\mathcal{L}$'
if isinstance(lowcuts[0], (float, int)):
print('a')
ll_score[ll_score < lowcuts[0]] = lowcuts[0]
......@@ -168,14 +165,13 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True,
CL = paths.get_csep_result('CL', years, catalog)
cl_score = np.array([i.observed_statistic for i in CL])
cl_label = r'$\mathcal{L}{\|N}$'
cl_label = r'$\mathcal{L}_{\|N}$'
if isinstance(lowcuts[1], (float, int)):
cl_score[cl_score < lowcuts[1]] = lowcuts[1]
## Brier score
Brier_results = paths.get_csep_result('B', years, catalog)
Brier = np.array([i.test_distribution[3] for i in Brier_results])
print(Brier)
brier_label = r'$\mathcal{B}$'
if isinstance(lowcuts[2], (float, int)):
Brier[Brier < lowcuts[2]] = lowcuts[2]
......@@ -183,7 +179,6 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True,
Brier_bin = np.array([i.test_distribution[2] for i in Brier_results])
if isinstance(lowcuts[3], (float, int)):
Brier_bin[Brier_bin < lowcuts[3]] = lowcuts[3]
## Consistency tests
Consistency_Results = []
for n, m, s in zip(paths.get_csep_result('N', years, catalog),
......@@ -198,13 +193,11 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True,
model_cons.append('S')
Consistency_Results.append(model_cons)
ref_name = 'All models'
names = [i.sim_name for i in LL]
## Order results in terms of ~average performance for visualization purpose
order_val = norm(ll_score) + norm(cl_score) + norm(Brier_bin) + norm(Brier)
order = np.flip(np.argsort(order_val))
ll_score = ll_score[order]
......@@ -215,17 +208,21 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True,
Consistency_Results = [Consistency_Results[i] for i in order]
## Complete array
colors = ['darkorange', 'orange', 'steelblue', 'blue', 'green']
colors = ['darkorange',
'darkred',
'teal',
'lightskyblue',
'olivedrab']
Axes = plot_scores([ll_score, cl_score, Brier, Brier_bin],
colors[:4],
[ll_label, cl_label, brier_label, brier_bin_label],
model_names,format=format, angle_offset=60, offset=10, min_y=0.3, ny=4,
model_names, format=format, angle_offset=60, offset=10, min_y=0.3, ny=4, fontsize=9,
lowbounds = [bool(i) for i in lowcuts])
Axes = plot_all_consistencies(Axes, Consistency_Results, color=colors[4])
Axes = plot_legends(Axes, colors, labels)
Axes.set_title('Test results - %i years (%s)' % (years, ref_name) , pad=15, fontsize=14)
# Axes.set_title('Test results - %i years' % (years) , pad=15, fontsize=14)
if savepath:
plt.savefig(savepath, dpi=300, transparent=True)
plt.show()
......@@ -262,8 +259,8 @@ if __name__ == '__main__':
# savepath=paths.get_csep_figpath('Total', 10, 'ref11_CL_%s' % cat), catalog=cat)
plot_results(10,
labels=labels,
format=['%i', '%i', '%.3e', '%.3e'],
lowcuts=[-195, -195, -0.0001086, -0.0001086],
savepath=paths.get_csep_figpath('Scores', 10, 'Total'),
format=['%i', '%i', '%.4e', '%.4e'],
lowcuts=[-200, -200, -0.00010855, -0.00010855],
savepath=paths.get_csep_figpath('Scores', 10, 'Total_%s' % cat, 'svg'),
catalog=cat)
......@@ -135,7 +135,7 @@ def get_kripley_result_path(method, year):
:return:
"""
return join(kripley_store[int(year)], '%s_%s.txt' % (method, str(year)))
return join(kripley_store[int(year)], '%s_%s' % (method, str(year)))
def get_csep_result(test, year, type=False):
......@@ -153,7 +153,7 @@ def get_csep_result(test, year, type=False):
def get_csep_figpath(test, year, type=None):
def get_csep_figpath(test, year, type=None, format='png'):
"""
Get the figure path of a particular test
:param test: Name of the test
......@@ -165,7 +165,7 @@ def get_csep_figpath(test, year, type=None):
type = '_' + str(type)
else:
type = ''
return join(csep_figs, "%s_%s%s.png" % (test, str(year), type))
return join(csep_figs, "%s_%s%s.%s" % (test, str(year), type, format))
def get_kripley_figpath(analysis, year, model):
"""
......
results/figures/csep/T_5_emrcmt.png

248 KB | W: | H:

results/figures/csep/T_5_emrcmt.png

270 KB | W: | H:

results/figures/csep/T_5_emrcmt.png
results/figures/csep/T_5_emrcmt.png
results/figures/csep/T_5_emrcmt.png
results/figures/csep/T_5_emrcmt.png
  • 2-up
  • Swipe
  • Onion skin
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment