Commit beee3fe6 authored by Leonie Pick's avatar Leonie Pick

Correction to calculation of calibration curve

parent 8286e918
......@@ -730,7 +730,18 @@ def Assess_Model(Data,Target,N2,K2,modelTest0,n_classes,Plot,Save,SaveName):
# Calibration
n_bins = 10
Calib = np.zeros((N2,K2,3,n_bins))
Calib = np.zeros((N2,K2,3,n_bins))*np.nan
## Calibration all
# Calib_all = np.zeros((3,n_bins))*np.nan
# modelTest0.fit(Data,Target)
# proba_all = modelTest0.predict_proba(Data)
# Calib_all[2,:], bin_edges = np.histogram(proba_all[:,1], bins=np.arange(0,1.1,0.1))
# Prob_true, Prob_pred = calibration_curve(Target, proba_all[:,1], normalize=False, n_bins=n_bins)
# bins_present = np.where(Calib_all[2,:]!=0)[0]
# Calib_all[0,bins_present]=Prob_true
# Calib_all[1,bins_present]=Prob_pred
#modelTest0 = CalibratedClassifierCV(modelTest, method='sigmoid', cv=4)
......@@ -801,16 +812,30 @@ def Assess_Model(Data,Target,N2,K2,modelTest0,n_classes,Plot,Save,SaveName):
## Calibration curve
Calib[i,ii,2,:], bin_edges = np.histogram(proba_test[:,1], bins=np.arange(0,1.1,0.1))
Prob_true, Prob_pred = calibration_curve(T_test2, proba_test[:,1], normalize=False, n_bins=n_bins)
bins_missing = np.where(Calib[i,ii,2,:]==0)[0]
if len(bins_missing) > 0:
Calib[i,ii,0,:]=np.insert(Prob_true,bins_missing,np.zeros(len(bins_missing))+np.nan)
Calib[i,ii,1,:]=np.insert(Prob_pred,bins_missing,np.zeros(len(bins_missing))+np.nan)
else:
Calib[i,ii,0,:]=Prob_true
Calib[i,ii,1,:]=Prob_pred
# Calib[i,ii,0,:], Calib[i,ii,1,:] = calibration_curve(T_test2, proba_test[:,1], normalize=False, n_bins=n_bins)
# My way for calibration curve:
# for iii in range(len(bin_edges)-1):
# if iii == len(bin_edges)-2:
# indices = np.logical_and(proba_test[:,1] >= bin_edges[iii],proba_test[:,1] <= bin_edges[iii+1])
# count = len(np.where(indices==True)[0])
# else:
# indices = np.logical_and(proba_test[:,1] >= bin_edges[iii],proba_test[:,1] < bin_edges[iii+1])
# count = len(np.where(indices==True)[0])
# #countAll += count
# #Calib[i,ii,0,iii] = countAll/len(T_test2)
# Positives = len(np.where(T_test2[indices]==1)[0])
# if count == 0:
# Calib[i,ii,0,iii] = np.nan
# else:
# Calib[i,ii,0,iii] = Positives/count
# Calib[i,ii,1,iii] = np.nanmean(proba_test[indices,1])
# sklearn function:
Prob_true, Prob_pred = calibration_curve(T_test2, proba_test[:,1], normalize=False, n_bins=n_bins)
bins_present = np.where(Calib[i,ii,2,:]!=0)[0]
Calib[i,ii,0,bins_present]=Prob_true
Calib[i,ii,1,bins_present]=Prob_pred
## AUC scores
Scorers[i,ii,6,0,np.isnan(Scorers[i,ii,6,0,:])] = 1; Scorers[i,ii,6,1,np.isnan(Scorers[i,ii,6,1,:])] = 1
Scorers[i,ii,0,0,:] = metrics.auc(Scorers[i,ii,4,0,:],Scorers[i,ii,2,0,:]); Scorers[i,ii,0,1,:] = metrics.auc(Scorers[i,ii,4,1,:],Scorers[i,ii,2,1,:]) #ROC
......
......@@ -476,7 +476,7 @@ def Decision(thresholds,Model_Mean,Model_Std,Save,SaveName):
fig, axs = plt.subplots(1,1,facecolor='w',edgecolor='k',sharey=True)
fig.set_size_inches([6.5,6], forward=True)
fig.subplots_adjust(bottom=0.1,top=0.95,left=0.2,right=0.9,wspace=0.0,hspace=0.0)
fig.subplots_adjust(bottom=0.1,top=0.95,left=0.15,right=0.95,wspace=0.0,hspace=0.0)
axs.axis('equal')
scoreID = [11,15,18,19,20,21]
......@@ -508,36 +508,40 @@ def Decision(thresholds,Model_Mean,Model_Std,Save,SaveName):
def Calibration(Calib,bin_edges,Save,SaveName):
fig, axs = plt.subplots(2,1,facecolor='w',edgecolor='k',sharex=True)
fig.set_size_inches([6.5,10], forward=True)
fig.subplots_adjust(bottom=0.1,top=0.95,left=0.2,right=0.9,wspace=0.0,hspace=0.02)
fig.set_size_inches([6,10], forward=True)
fig.subplots_adjust(bottom=0.08,top=0.99,left=0.15,right=0.99,wspace=0.0,hspace=0.02)
axs = axs.ravel()
axs[0].axis('equal')
Calib_inner = np.nanmean(Calib,axis=1)
Calib_outer = np.nanmean(Calib_inner,axis=0)
Calib_outer_std = np.nanstd(Calib_inner,axis=0)
axs[0].plot(np.linspace(0,1,11),np.linspace(0,1,11),linestyle='--',color='gray',label=r'perfectly calibrated')
axs[0].errorbar(Calib_outer[1,:],Calib_outer[0,:],yerr=3*Calib_outer_std[0,:],ecolor='black')#xerr=3*Calib_outer_std[1,:]
axs[1].bar(bin_edges[0:-1],height=Calib_outer[2,:],width=bin_edges[1]-bin_edges[0],align='edge',yerr=3*Calib_outer_std[2,:],color='darkgray',edgecolor='black',linewidth=0.5)
axs[0].errorbar(Calib_outer[1,:],Calib_outer[0,:],xerr=Calib_outer_std[1,:],yerr=Calib_outer_std[0,:],ecolor='black', label=r'reliability curve $\pm$ std')
#axs[0].plot(Calib_all[1,:],Calib_all[0,:],color='red')
axs[1].bar(bin_edges[0:-1],height=Calib_outer[2,:],width=bin_edges[1]-bin_edges[0],align='edge',yerr=Calib_outer_std[2,:],color='darkgray',edgecolor='black',linewidth=0.5)
#axs[1].bar(bin_edges[0:-1],height=Calib_all[2,:],width=bin_edges[1]-bin_edges[0],align='edge',color='red',zorder=0)
axs[0].set_ylim([-0.5,1.5])
axs[0].set_xticks(np.linspace(0,1,11)[::2])
axs[0].set_xticks(np.linspace(0,1,11),minor=True)
axs[0].set_xlim([-0.1,1.1])
axs[0].set_ylim([-0.1,1.1])
axs[0].set_xticklabels([])
axs[0].set_yticks(np.linspace(0,1,11)[::2])
axs[0].set_yticks(np.linspace(0,1,11),minor=True)
axs[0].set_ylabel('Fraction of positives',fontsize=18)
axs[0].tick_params(axis ='x',which='both',direction='inout',labelsize=16)
axs[0].tick_params(axis ='y',which='both',direction='inout',labelsize=16)
axs[0].legend(loc=0,frameon=False,fontsize=16)
axs[0].set_xticklabels([])
#axs[1].set_xlim([0,1])
axs[1].set_ylim([0,np.nanmax(Calib_outer[2,:]+2.5)])
axs[1].set_yticks(np.arange(0,40,5))
axs[1].set_ylabel('Count',fontsize=18)
axs[1].tick_params(axis ='x',which='both',direction='inout',labelsize=16)
axs[1].tick_params(axis ='y',which='both',direction='inout',labelsize=16)
axs[1].set_xticks(np.linspace(0,1,11)[::2])
axs[1].set_xticks(np.linspace(0,1,11),minor=True)
axs[1].set_xticklabels(['0.0','0.2','0.4','0.6','0.8','1.0'])
axs[1].set_xlabel('Mean predicted probability',fontsize=18)
axs[1].set_xlabel('Predicted probability',fontsize=18)
if Save == True:
fig.savefig('./Dump/Fig/development/Calibration_'+SaveName+'.pdf',format='pdf',dpi=300,transparent=True)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment