Print all the mean differences of accuracy of the 10 runs 10-fold cross-validation for nbc versus aode on each dataset



In [2]:

    
ClassID = readdlm("Data/ClassifierID.dat", ',')
ClassNames = readdlm("Data/ClassifierNames.dat", ',')
DatasetID = readdlm("Data/DatasetID.dat", ',');
DatasetNames = readdlm("Data/DatasetNames.dat", ',');
Percent_correct = readdlm("Data/Percent_correct.dat", ',');

cl1=1 #nbc
cl2=2 #aode
println("Comparison of ", ClassNames[cl1,1], " vs. ", ClassNames[cl2,1])
println()

#Compute mean accuracy
indi=find(x->x==cl1,ClassID)
indj=find(x->x==cl2,ClassID)
acci=Float64[]
accj=Float64[]
for d=1:Int32(maximum(DatasetID))
    indd=find(x->x==d,DatasetID)
    indid=intersect(indi,indd)
    indjd=intersect(indj,indd)
    push!(acci,mean(Percent_correct[indid])/100)
    push!(accj,mean(Percent_correct[indjd])/100)
    if mod(d,3)==0
        @printf "%s & %1.3f & %s & %1.3f & %s & %1.3f \\\\\n" DatasetNames[d-2] (acci[d-2]-accj[d-2])*100 DatasetNames[d-1] (acci[d-1]-accj[d-1])*100 DatasetNames[d] (acci[d]-accj[d])*100   
    end
end









    



Comparison of nbc vs. aode

anneal & -1.939 & audiology & -0.261 & wisconsin-breast-cancer & 0.467 \\
cmc & -0.719 & contact-lenses & 2.000 & credit & -0.464 \\
german-credit & -1.014 & pima-diabetes & -0.151 & ecoli & -7.269 \\
eucalyptus & -0.790 & glass & -2.600 & grub-damage & 4.362 \\
haberman & -0.614 & hayes-roth & 0.000 & cleeland-14 & -0.625 \\
hungarian-14 & -0.069 & hepatitis & -0.212 & hypothyroid & -1.683 \\
ionosphere & 0.267 & iris & -3.242 & kr-s-kp & -0.833 \\
labor & 0.000 & lier-disorders & -1.762 & lymphography & -1.863 \\
monks1 & -10.002 & monks3 & -0.343 & monks & -4.190 \\
mushroom & -2.434 & nursery & -4.747 & optdigits & -3.548 \\
page-blocks & 0.583 & pasture-production & -10.043 & pendigits & -0.443 \\
postoperatie & 1.333 & primary-tumor & -0.674 & segment & -3.922 \\
solar-flare-C & -2.776 & solar-flare-m & -0.688 & solar-flare-X & -3.996 \\
sonar & -0.338 & soybean & -1.112 & spambase & -3.284 \\
spect-reordered & -1.684 & splice & -0.699 & squash-stored & -0.367 \\
squash-unstored & -5.600 & tae & -0.400 & credit & -16.909 \\
owel & -5.040 & waveform & -1.809 & white-clover & 0.500 \\
wine & 0.143 & yeast & -0.202 & zoo & -0.682 \\



In [4]:

    
using HypothesisTests
using Gadfly
using DataFrames
include("Plots/plot_data.jl")

diff=acci-accj

#Density plot of difference of accuracies
p=plot_data(cl1,cl2,[],diff,-0.03,0.02)
display(p)

#Compute p-value Sign-Rank Test
pval_SignRankTest=pvalue(SignedRankTest(acci,accj))
println("p-value=$pval_SignRankTest")

#Compute p-value SignTest
#pvalSignTest=pvalue(SignTest(acci-accj), tail=:left)
#println("p-value=$pvalSignTest")









    













    



p-value=1.628439911664703e-6



In [ ]:

Bayesian sign-test



In [8]:

    
using  Distributions
using DataFrames
using Gadfly
using Compose
include("Tests/Bsigntest.jl")
include("Plots/plot_simplex.jl")


#Bayesian Sign Test without rope
rope=0
data=Bsigntest(acci,accj,rope)

#Plot
df = DataFrame(Pleft=data[1,:][:])
p=plot(df, x=:Pleft, Geom.histogram,Theme(major_label_font_size=13pt,minor_label_font_size=12pt,key_label_font_size=11pt))
display(p)
draw(PDF("Plots/postSignTestnorope$cl1$cl2.pdf", 6inch, 3inch),p)

#Bayesian Sign Test with rope
rope=0.01
data=Bsigntest(acci,accj,rope)
#Plot
ptriangle=plot_simplex(data, ClassNames[cl1],ClassNames[cl2])
display(ptriangle)
draw(PDF("Plots/plotSignSimplex$cl1$cl2.pdf", 5inch, 5inch), ptriangle)

#Plot Marginals
df = DataFrame(Pleft=data[1,:][:], Prope=data[2,:][:])
p=plot(df, x=:Pleft, y=:Prope, Geom.histogram2d,Guide.xlabel(ClassNames[cl2]),Guide.ylabel("rope"),Theme(major_label_font_size=13pt,minor_label_font_size=12pt,key_label_font_size=11pt))
display(p)
draw(PDF("Plots/postSignTestA$cl1$cl2.pdf", 6inch, 3inch), p)

df = DataFrame(Pright=data[3,:][:], Prope=data[2,:][:])
p=plot(df, x=:Pright, y=:Prope, Geom.histogram2d,Guide.xlabel(ClassNames[cl1]),Guide.ylabel("rope"),Theme(major_label_font_size=13pt,minor_label_font_size=12pt,key_label_font_size=11pt))
display(p)
draw(PDF("Plots/postSignTestB$cl1$cl2.pdf", 6inch, 3inch), p)