Commit 8286e918 by Leonie Pick

### Continued work on decision boundary and calibration curve

parent cb42391f
This diff is collapsed.
This diff is collapsed.
 ... ... @@ -827,12 +827,19 @@ def Assess_Model(Data,Target,N2,K2,modelTest0,n_classes,Plot,Save,SaveName): CM_means_i = np.sum(CM,axis=1); CM_means = np.around(np.mean(CM_means_i,axis=0),0) ## Decide on decision boundary print('best MCC:',np.nanmax(Model_Mean[19,0,:])) print('best decision test:',thresholds[np.nanargmax(Model_Mean[19,0,:])]) print('best decision train:',thresholds[np.nanargmax(Model_Mean[19,1,:])]) decision_boundary = int(thresholds[np.nanargmax(Model_Mean[19,0,:])]*100) decision_boundary_MCC = int(thresholds[np.nanargmax(Model_Mean[19,0,:])]*100) decision_boundary_MCC_train = int(thresholds[np.nanargmax(Model_Mean[19,1,:])]*100) scores_mean = np.nanmean(Model_Mean[[11,15,18,19,20],0,:],axis=0) scores_mean_train = np.nanmean(Model_Mean[[11,15,18,19,20],1,:],axis=0) decision_boundary_all = int(thresholds[np.nanargmax(scores_mean)]*100) decision_boundary_all_train = int(thresholds[np.nanargmax(scores_mean_train)]*100) #decision_boundary = 50 decision_boundary = decision_boundary_all #print('best MCC:',np.nanmax(Model_Mean[19,0,:])) print('best decision test:',thresholds[decision_boundary_MCC], thresholds[decision_boundary_all]) print('best decision train:',thresholds[decision_boundary_MCC_train], thresholds[decision_boundary_all_train]) C, Scores = Get_MyScores(CM_means[:,0,decision_boundary], n_classes) ## Plot mean confusion matrix and curves ... ... @@ -842,6 +849,6 @@ def Assess_Model(Data,Target,N2,K2,modelTest0,n_classes,Plot,Save,SaveName): pl.Decision(thresholds, Model_Mean[:,0,:], Model_Std[:,0,:],Save,SaveName) pl.Calibration(Calib,bin_edges,Save,SaveName) return Model_Mean[:,:,decision_boundary], Model_Std[:,:,decision_boundary] return Model_Mean[:,:,decision_boundary], Model_Std[:,:,decision_boundary], decision_boundary/100 ### ###
 ... ... @@ -452,7 +452,7 @@ def Curves(N2,K2,Scorers,decision,Model_Mean,Model_Std,C,Save,SaveName): #axs[1].axvspan(0,1,P/(P+N),1,color='silver',alpha=0.5,zorder=0,lw=0) axs[0].set_xlabel(r'FPR = FP/N', fontsize=18); axs[1].set_xlabel(r'TPR = TP/P', fontsize=18) axs[0].set_ylabel(r'TPR = TP/P', fontsize=18); axs[1].set_ylabel(r'PPV = TP/PP', fontsize=18) axs[0].legend(loc=0, frameon=False, fontsize=16); axs[1].legend(loc=0, frameon=False, fontsize=16) axs[0].legend(loc=0, frameon=False, fontsize=16); axs[1].legend(loc=3, frameon=False, fontsize=16) axs[0].set_title('ROC curve',fontsize=18) axs[0].tick_params(axis ='x',which='both',direction='inout',labelsize=16) axs[0].tick_params(axis ='y',which='both',direction='inout',labelsize=16) ... ... @@ -479,11 +479,11 @@ def Decision(thresholds,Model_Mean,Model_Std,Save,SaveName): fig.subplots_adjust(bottom=0.1,top=0.95,left=0.2,right=0.9,wspace=0.0,hspace=0.0) axs.axis('equal') scoreID = [11,18,19,20,21] scoreLabel = ['ACC','HSS','MCC','J','Deltap'] linewidth=[.5,.5,1.0,.5,.5] zorder=[0,0,1,0,0] colors=['midnightblue','orange','maroon','purple','black'] scoreID = [11,15,18,19,20,21] scoreLabel = ['ACC','F1','HSS','MCC','J','Deltap'] linewidth=[.5,.5,.5,1.5,.5,.5] zorder=[0,0,0,1,0,0] colors=['midnightblue','orange','magenta','maroon','green','purple'] for i in range(len(scoreID)): #axs[0].plot(thresholds, Model_Mean[scoreID[i],:],label=scoreLabel[i]) axs.errorbar(thresholds, Model_Mean[scoreID[i],:], yerr=3*Model_Std[scoreID[i],:], fmt='none',elinewidth=linewidth[i], ecolor=colors[i], zorder=0) ... ... @@ -518,10 +518,10 @@ def Calibration(Calib,bin_edges,Save,SaveName): Calib_outer_std = np.nanstd(Calib_inner,axis=0) axs[0].plot(np.linspace(0,1,11),np.linspace(0,1,11),linestyle='--',color='gray',label=r'perfectly calibrated') axs[0].errorbar(Calib_outer[1,:],Calib_outer[0,:],xerr=3*Calib_outer_std[1,:],yerr=3*Calib_outer_std[0,:],ecolor='black') axs[0].errorbar(Calib_outer[1,:],Calib_outer[0,:],yerr=3*Calib_outer_std[0,:],ecolor='black')#xerr=3*Calib_outer_std[1,:] axs[1].bar(bin_edges[0:-1],height=Calib_outer[2,:],width=bin_edges[1]-bin_edges[0],align='edge',yerr=3*Calib_outer_std[2,:],color='darkgray',edgecolor='black',linewidth=0.5) axs[0].set_ylim([-0.2,1.2]) axs[0].set_ylim([-0.5,1.5]) axs[0].set_xticks(np.linspace(0,1,11)[::2]) axs[0].set_xticks(np.linspace(0,1,11),minor=True) axs[0].set_ylabel('Fraction of positives',fontsize=18) ... ...
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!