Commit 8286e918 authored by Leonie Pick's avatar Leonie Pick

Continued work on decision boundary and calibration curve

parent cb42391f
This diff is collapsed.
......@@ -827,12 +827,19 @@ def Assess_Model(Data,Target,N2,K2,modelTest0,n_classes,Plot,Save,SaveName):
CM_means_i = np.sum(CM,axis=1); CM_means = np.around(np.mean(CM_means_i,axis=0),0)
## Decide on decision boundary
print('best MCC:',np.nanmax(Model_Mean[19,0,:]))
print('best decision test:',thresholds[np.nanargmax(Model_Mean[19,0,:])])
print('best decision train:',thresholds[np.nanargmax(Model_Mean[19,1,:])])
decision_boundary = int(thresholds[np.nanargmax(Model_Mean[19,0,:])]*100)
decision_boundary_MCC = int(thresholds[np.nanargmax(Model_Mean[19,0,:])]*100)
decision_boundary_MCC_train = int(thresholds[np.nanargmax(Model_Mean[19,1,:])]*100)
scores_mean = np.nanmean(Model_Mean[[11,15,18,19,20],0,:],axis=0)
scores_mean_train = np.nanmean(Model_Mean[[11,15,18,19,20],1,:],axis=0)
decision_boundary_all = int(thresholds[np.nanargmax(scores_mean)]*100)
decision_boundary_all_train = int(thresholds[np.nanargmax(scores_mean_train)]*100)
#decision_boundary = 50
decision_boundary = decision_boundary_all
#print('best MCC:',np.nanmax(Model_Mean[19,0,:]))
print('best decision test:',thresholds[decision_boundary_MCC], thresholds[decision_boundary_all])
print('best decision train:',thresholds[decision_boundary_MCC_train], thresholds[decision_boundary_all_train])
C, Scores = Get_MyScores(CM_means[:,0,decision_boundary], n_classes)
## Plot mean confusion matrix and curves
......@@ -842,6 +849,6 @@ def Assess_Model(Data,Target,N2,K2,modelTest0,n_classes,Plot,Save,SaveName):
pl.Decision(thresholds, Model_Mean[:,0,:], Model_Std[:,0,:],Save,SaveName)
pl.Calibration(Calib,bin_edges,Save,SaveName)
return Model_Mean[:,:,decision_boundary], Model_Std[:,:,decision_boundary]
return Model_Mean[:,:,decision_boundary], Model_Std[:,:,decision_boundary], decision_boundary/100
###
###
......@@ -452,7 +452,7 @@ def Curves(N2,K2,Scorers,decision,Model_Mean,Model_Std,C,Save,SaveName):
#axs[1].axvspan(0,1,P/(P+N),1,color='silver',alpha=0.5,zorder=0,lw=0)
axs[0].set_xlabel(r'FPR = FP/N', fontsize=18); axs[1].set_xlabel(r'TPR = TP/P', fontsize=18)
axs[0].set_ylabel(r'TPR = TP/P', fontsize=18); axs[1].set_ylabel(r'PPV = TP/PP', fontsize=18)
axs[0].legend(loc=0, frameon=False, fontsize=16); axs[1].legend(loc=0, frameon=False, fontsize=16)
axs[0].legend(loc=0, frameon=False, fontsize=16); axs[1].legend(loc=3, frameon=False, fontsize=16)
axs[0].set_title('ROC curve',fontsize=18)
axs[0].tick_params(axis ='x',which='both',direction='inout',labelsize=16)
axs[0].tick_params(axis ='y',which='both',direction='inout',labelsize=16)
......@@ -479,11 +479,11 @@ def Decision(thresholds,Model_Mean,Model_Std,Save,SaveName):
fig.subplots_adjust(bottom=0.1,top=0.95,left=0.2,right=0.9,wspace=0.0,hspace=0.0)
axs.axis('equal')
scoreID = [11,18,19,20,21]
scoreLabel = ['ACC','HSS','MCC','J','Deltap']
linewidth=[.5,.5,1.0,.5,.5]
zorder=[0,0,1,0,0]
colors=['midnightblue','orange','maroon','purple','black']
scoreID = [11,15,18,19,20,21]
scoreLabel = ['ACC','F1','HSS','MCC','J','Deltap']
linewidth=[.5,.5,.5,1.5,.5,.5]
zorder=[0,0,0,1,0,0]
colors=['midnightblue','orange','magenta','maroon','green','purple']
for i in range(len(scoreID)):
#axs[0].plot(thresholds, Model_Mean[scoreID[i],:],label=scoreLabel[i])
axs.errorbar(thresholds, Model_Mean[scoreID[i],:], yerr=3*Model_Std[scoreID[i],:], fmt='none',elinewidth=linewidth[i], ecolor=colors[i], zorder=0)
......@@ -518,10 +518,10 @@ def Calibration(Calib,bin_edges,Save,SaveName):
Calib_outer_std = np.nanstd(Calib_inner,axis=0)
axs[0].plot(np.linspace(0,1,11),np.linspace(0,1,11),linestyle='--',color='gray',label=r'perfectly calibrated')
axs[0].errorbar(Calib_outer[1,:],Calib_outer[0,:],xerr=3*Calib_outer_std[1,:],yerr=3*Calib_outer_std[0,:],ecolor='black')
axs[0].errorbar(Calib_outer[1,:],Calib_outer[0,:],yerr=3*Calib_outer_std[0,:],ecolor='black')#xerr=3*Calib_outer_std[1,:]
axs[1].bar(bin_edges[0:-1],height=Calib_outer[2,:],width=bin_edges[1]-bin_edges[0],align='edge',yerr=3*Calib_outer_std[2,:],color='darkgray',edgecolor='black',linewidth=0.5)
axs[0].set_ylim([-0.2,1.2])
axs[0].set_ylim([-0.5,1.5])
axs[0].set_xticks(np.linspace(0,1,11)[::2])
axs[0].set_xticks(np.linspace(0,1,11),minor=True)
axs[0].set_ylabel('Fraction of positives',fontsize=18)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment