Commit a9f3534b authored by Pablo Iturrieta Rebolledo's avatar Pablo Iturrieta Rebolledo
Browse files

Created Full rose diagram for Tests

parent acc7f9b0
......@@ -2,7 +2,7 @@
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="jdk" jdkName="Python 3.8 (venv)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
......
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (venv)" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (venv)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
......@@ -4,84 +4,226 @@ import pickle
import itertools
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.patches as patches
from matplotlib.lines import Line2D
def get_results(years):
results = {key : pickle.load(open(item, 'rb')) for key,item in filepaths.test_path.items()}
return results
return {key : pickle.load(open(item, 'rb')) for key,item in filepaths.test_path.items()}
def norm(array):
return (array - array.min())/(array.max() -array.min())
results = get_results(5)
def plot_consistency(ax, tests, model_i):
Y_offset = 1.2 * 1.24
pos = ax.get_figure().transFigure.inverted().transform(
ax.transData.transform([(i, Y_offset ) for i in ax.get_xticks()]))[model_i]
r = 0.01
sep =1.2
dh = -0.02
cx = [pos[0] - sep*r*(len(tests)-1) + 2*sep*r*(i) for i in range(len(tests))]
for i, ci in zip(tests, cx):
artist = patches.Circle((ci, pos[1] + dh), r, fc='green')
ax.get_figure().add_artist(artist)
ax.get_figure().text(ci, pos[1] + dh, i, ha='center', va='center', color='white')
return ax
def plot_axis(axis,n_results, offset, end_theta, n, min_y, array, yticks, low_bound, color, label, fontsize):
axis_angle = (end_theta + np.deg2rad(offset) + n*(2*np.pi-end_theta - 2*np.deg2rad(offset))/(n_results-1))
axis.plot([axis_angle, axis_angle], [min_y, 1. + min_y],
color=color, linewidth=2, linestyle=':', alpha=0.5)
for i in yticks:
val = (i - min_y) * (array.max() - array.min()) + array.min()
val = '%.2f' % val
if low_bound and i == yticks[0]:
val = '<'+ val
axis.text(axis_angle, i - 0.03, val,
rotation=np.rad2deg(axis_angle) + 90, color=color,
ha='center', va='center', fontsize=fontsize)
axis.text(axis_angle, i + 0.15, label, color=color,
ha='center', va='center', fontsize=14)
def plot_theta_seps(axis, name_models, end_angle, n_model, n_results, width_model,min_y=0):
for i in np.linspace(0.0, end_angle, n_model + 1): # ygrid separators
axis.plot([i , i],
[min_y, 1. + min_y], color='grey', linewidth=0.5)
axis.get_xaxis().set_ticks(np.linspace(0.0, end_angle , n_model,
endpoint=False) + n_results/2*width_model)
axis.get_xaxis().set_ticklabels(name_models)
axis.tick_params(axis='x', which='major', pad=20)
def plot_rticks(axis, min_r, n_r):
axis.get_yaxis().grid(zorder=0, label=False)
yticks = [min_r]
yticks.extend([i + min_r for i in np.linspace(0,1,n_r+1)])
axis.get_yaxis().set_ticks(yticks)
axis.get_yaxis().set_ticklabels([])
axis.set_ylim([0, 1.0 + min_r])
return yticks
def Plot(arrays, colors, result_labels, model_labels, lowbounds, angle_offset=90, offset=10, min_y=0.3,ny=4, fontsize=9):
# N plots
n_results = len(arrays)
n_models = len(arrays[0])
# Plot properties
figsize = (8,8)
end_theta = 2 * np.pi - angle_offset / 360 * 2 * np.pi
width = end_theta/n_models/n_results
theta = np.linspace(0.0, end_theta, n_results * n_models, endpoint=False)
# Rearange results and colors
tuple_arrays = tuple([norm(i) for i in arrays])
score_total = np.vstack(tuple_arrays).T.ravel()
color_array = [plt.cm.colors.to_rgb(i) for i in colors]
tuple_colors = tuple([[color for i in range(n_models)] for color in color_array])
colors_full = np.hstack(tuple_colors).reshape((n_models*n_results,3))
# Create figure
ax = plt.subplot(111,projection='polar')
fig = ax.get_figure()
fig.set_size_inches(figsize)
ax.grid(False)
# Plot Data
ax.bar(theta + width/2., score_total, width=width, bottom=min_y,
color=colors_full, alpha=1, zorder=0)
# Plot shaded region
ax.bar(np.pi+end_theta/2, 1 + min_y, width= 2*np.pi - end_theta, color='grey', alpha=0.2, bottom=min_y)
ax.bar(0, min_y, width= 2*np.pi , color='grey', alpha=0.2)
# Plot auxiliaries
plot_theta_seps(ax, model_labels, end_theta, n_models,n_results, width, min_y=min_y)
yticks = plot_rticks(ax, min_y, ny)
for i in range(n_results):
plot_axis(ax,n_results, offset, end_theta, i, min_y, arrays[i], yticks, lowbounds[i],
color_array[i], result_labels[i], fontsize)
return ax
##### Results 5
results_all = get_results(5)
n_results = 3
n_models = 19
### Absolute values plot
def norm(array):
return (array - array.min())/(array.max() -array.min())
## T_score
T = np.array([i.observed_statistic for i in results['TW_5'][0]]).reshape((n_models, n_models)).T
T = np.array([i.observed_statistic for i in results_all['TW_5'][0]]).reshape((n_models, n_models)).T
t_score = np.sum(T, axis=1) / T.shape[0]
min_val = - 1
t_score[t_score < min_val ] = min_val
t_lowcut = 0.1
t_score[t_score < t_lowcut] = t_lowcut
## PGS
pgs = np.array([i.test_distribution[0] for i in results['PGS_5']])
pgs = np.array([i.test_distribution[0] for i in results_all['PGS_5']])
## Conditional Likelihood
cl_score = np.array([i.quantile for i in results['CL_5']])
## Names and order
names = [i.sim_name for i in results['CL_5']]
cl_score = np.array([i.quantile for i in results_all['CL_5']])
## Consistency tests
alpha = 0.01
Consistency_Results = []
for n, m, s in zip(results_all['N_5'], results_all['M_5'], results_all['S_5']):
model_cons = []
if n.quantile[0] > alpha/2. and n.quantile[1] < 1-alpha/2.:
model_cons.append('N')
if m.quantile > alpha:
model_cons.append('M')
if s.quantile > alpha:
model_cons.append('S')
Consistency_Results.append(model_cons)
## Order results in terms of ~average performance for visualization purpose
names = [i.sim_name for i in results_all['CL_5']]
order = np.flip(np.argsort(norm(cl_score)+norm(pgs)+norm(t_score)))
pgs = pgs[order]
cl_score = cl_score[order]
t_score = t_score[order]
names = [names[i] for i in order]
labels = []
for i in names:
labels.extend(['', i, ''])
model_names = [names[i] for i in order]
Consistency_Results = [Consistency_Results[i] for i in order]
## Complete array
score_total = np.vstack((norm(t_score), norm(pgs), norm(cl_score))).T.ravel()
theta = np.linspace(0.0, 2 * np.pi, n_results * n_models, endpoint=False)
## Colors
colors_t = np.array([plt.cm.colors.to_rgb('orangered') for i in range(n_models)])
colors_pgs = np.array([plt.cm.colors.to_rgb('turquoise') for i in range(n_models)])
colors_cl = [plt.cm.colors.to_rgb('steelblue') for i in range(n_models)]
colors = np.hstack((colors_t, colors_pgs, colors_cl)).reshape((n_models*n_results,3))
colors = ['orangered','darkgreen','steelblue']
Axes = Plot([t_score, pgs, cl_score], colors, ['IG', r'$\Delta R_j$', r'$\delta_{CL}$'],
model_names, angle_offset=90, offset=5, min_y=0.3,ny=4, lowbounds = [True, True, False])
width = 0.115
for i, test_pass in enumerate(Consistency_Results):
plot_consistency(Axes, test_pass, i)
# fig = plt.figure(figsize=(8,8))
ax = plt.subplot(111,projection='polar')
fig = ax.get_figure()
fig.set_size_inches(8, 8)
# ax.bar(np.linspace(0.0, 2 * np.pi, n_models, endpoint=False), [i%2 for i in range(n_models)], width=0.33, color='grey', alpha=0.5, tick_label=names)
# edges
ax.grid(False)
ax.get_yaxis().grid([0, 0.4, 0.7], zorder=0, label=False)
# ax.get_yaxis().set_ticks([])
for i in np.linspace(0.0, 2 * np.pi, n_models , endpoint=False):
ax.plot([i ,i - 3*width*np.pi/n_models],[0, 1.], color='grey', linewidth=0.5)
legend_elements = [Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[0]),
lw=5, label='T-test score'),
Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[1]),
lw=5, label='PG score'),
Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[2]),
lw=5, label='CL-test score'),
Line2D([0], [0], color='green',
lw=0, marker='o', markersize=15, label='Consistency tests')]
Axes.get_figure().legend(handles=legend_elements, loc='lower right')
Axes.set_title('Combined test results', pad=15, fontsize=14)
plt.show()
ax.bar(theta, score_total, width=width, bottom=0, color=colors, alpha=1, zorder=2)
##### Results10
results_all = get_results(10)
n_results = 3
n_models = 19
ax.set_ylim([0,1])
ax.get_xaxis().set_ticks(np.linspace(0.0, 2 * np.pi, n_models, endpoint=False) + width)
ax.get_xaxis().set_ticklabels(names)
ax.tick_params(axis='x', which='major', pad=20)
### Absolute values plot
## T_score
T = np.array([i.observed_statistic for i in results_all['TW_10'][0]]).reshape((n_models, n_models)).T
t_score = np.sum(T, axis=1) / T.shape[0]
t_lowcut = 0.1
t_score[t_score < t_lowcut] = t_lowcut
## PGS
pgs = np.array([i.test_distribution[0] for i in results_all['PGS_10']])
## Conditional Likelihood
cl_score = np.array([i.quantile for i in results_all['CL_10']])
## Consistency tests
alpha = 0.01
Consistency_Results = []
for n, m, s in zip(results_all['N_10'], results_all['M_10'], results_all['S_10']):
model_cons = []
if n.quantile[0] > alpha/2. and n.quantile[1] < 1-alpha/2.:
model_cons.append('N')
if m.quantile > alpha:
model_cons.append('M')
if s.quantile > alpha:
model_cons.append('S')
Consistency_Results.append(model_cons)
## Order results in terms of ~average performance for visualization purpose
names = [i.sim_name for i in results_all['CL_10']]
order = np.flip(np.argsort(norm(cl_score)+norm(pgs)+norm(t_score)))
pgs = pgs[order]
cl_score = cl_score[order]
t_score = t_score[order]
model_names = [names[i] for i in order]
Consistency_Results = [Consistency_Results[i] for i in order]
## Complete array
colors = ['orangered','darkgreen','steelblue']
Axes = Plot([t_score, pgs, cl_score], colors, ['IG', r'$\Delta R_j$', r'$\delta_{CL}$'],
model_names, angle_offset=90, offset=5, min_y=0.3,ny=4, lowbounds = [True, True, False])
for i, test_pass in enumerate(Consistency_Results):
plot_consistency(Axes, test_pass, i)
ax.get_xticks()
#
# ax.get_xaxis().set_linewidth(2)
# ax.yaxis.grid( linewidth = 5)
# ax.yaxis.(2)
# plt.thetagrids(np.linspace(0, 360, 2))
# ax.grid( zorder=1)
legend_elements = [Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[0]),
lw=5, label='T-test score'),
Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[1]),
lw=5, label='PG score'),
Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[2]),
lw=5, label='CL-test score'),
Line2D([0], [0], color='green',
lw=0, marker='o', markersize=15, label='Consistency tests')]
Axes.get_figure().legend(handles=legend_elements, loc='lower right')
Axes.set_title('Combined test results', pad=15, fontsize=14)
plt.show()
# ref_model = 11
# t_score = T[ref_model*n_models:(ref_model+1)*n_models]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment