Notebook5 compares two classifiers on multiple datasets


Print all the mean differences of accuracy of the 10 runs 10-fold cross-validation for nbc versus aode on each dataset


In [2]:
ClassID = readdlm("Data/ClassifierID.dat", ',')
ClassNames = readdlm("Data/ClassifierNames.dat", ',')
DatasetID = readdlm("Data/DatasetID.dat", ',');
DatasetNames = readdlm("Data/DatasetNames.dat", ',');
Percent_correct = readdlm("Data/Percent_correct.dat", ',');

cl1=1 #nbc
cl2=2 #aode
println("Comparison of ", ClassNames[cl1,1], " vs. ", ClassNames[cl2,1])
println()

#Compute mean accuracy
indi=find(x->x==cl1,ClassID)
indj=find(x->x==cl2,ClassID)
acci=Float64[]
accj=Float64[]
for d=1:Int32(maximum(DatasetID))
    indd=find(x->x==d,DatasetID)
    indid=intersect(indi,indd)
    indjd=intersect(indj,indd)
    push!(acci,mean(Percent_correct[indid])/100)
    push!(accj,mean(Percent_correct[indjd])/100)
    if mod(d,3)==0
        @printf "%s & %1.3f & %s & %1.3f & %s & %1.3f \\\\\n" DatasetNames[d-2] (acci[d-2]-accj[d-2])*100 DatasetNames[d-1] (acci[d-1]-accj[d-1])*100 DatasetNames[d] (acci[d]-accj[d])*100   
    end
end


Comparison of nbc vs. aode

anneal & -1.939 & audiology & -0.261 & wisconsin-breast-cancer & 0.467 \\
cmc & -0.719 & contact-lenses & 2.000 & credit & -0.464 \\
german-credit & -1.014 & pima-diabetes & -0.151 & ecoli & -7.269 \\
eucalyptus & -0.790 & glass & -2.600 & grub-damage & 4.362 \\
haberman & -0.614 & hayes-roth & 0.000 & cleeland-14 & -0.625 \\
hungarian-14 & -0.069 & hepatitis & -0.212 & hypothyroid & -1.683 \\
ionosphere & 0.267 & iris & -3.242 & kr-s-kp & -0.833 \\
labor & 0.000 & lier-disorders & -1.762 & lymphography & -1.863 \\
monks1 & -10.002 & monks3 & -0.343 & monks & -4.190 \\
mushroom & -2.434 & nursery & -4.747 & optdigits & -3.548 \\
page-blocks & 0.583 & pasture-production & -10.043 & pendigits & -0.443 \\
postoperatie & 1.333 & primary-tumor & -0.674 & segment & -3.922 \\
solar-flare-C & -2.776 & solar-flare-m & -0.688 & solar-flare-X & -3.996 \\
sonar & -0.338 & soybean & -1.112 & spambase & -3.284 \\
spect-reordered & -1.684 & splice & -0.699 & squash-stored & -0.367 \\
squash-unstored & -5.600 & tae & -0.400 & credit & -16.909 \\
owel & -5.040 & waveform & -1.809 & white-clover & 0.500 \\
wine & 0.143 & yeast & -0.202 & zoo & -0.682 \\

In [4]:
using HypothesisTests
using Gadfly
using DataFrames
include("Plots/plot_data.jl")

diff=acci-accj

#Density plot of difference of accuracies
p=plot_data(cl1,cl2,[],diff,-0.03,0.02)
display(p)

#Compute p-value Sign-Rank Test
pval_SignRankTest=pvalue(SignedRankTest(acci,accj))
println("p-value=$pval_SignRankTest")

#Compute p-value SignTest
#pvalSignTest=pvalue(SignTest(acci-accj), tail=:left)
#println("p-value=$pvalSignTest")


DeltaAcc -0.09 -0.08 -0.07 -0.06 -0.05 -0.04 -0.03 -0.02 -0.01 0.00 0.01 0.02 0.03 0.04 0.05 0.06 0.07 0.08 -0.080 -0.078 -0.076 -0.074 -0.072 -0.070 -0.068 -0.066 -0.064 -0.062 -0.060 -0.058 -0.056 -0.054 -0.052 -0.050 -0.048 -0.046 -0.044 -0.042 -0.040 -0.038 -0.036 -0.034 -0.032 -0.030 -0.028 -0.026 -0.024 -0.022 -0.020 -0.018 -0.016 -0.014 -0.012 -0.010 -0.008 -0.006 -0.004 -0.002 0.000 0.002 0.004 0.006 0.008 0.010 0.012 0.014 0.016 0.018 0.020 0.022 0.024 0.026 0.028 0.030 0.032 0.034 0.036 0.038 0.040 0.042 0.044 0.046 0.048 0.050 0.052 0.054 0.056 0.058 0.060 0.062 0.064 0.066 0.068 0.070 0.072 -0.10 -0.05 0.00 0.05 0.10 -0.080 -0.075 -0.070 -0.065 -0.060 -0.055 -0.050 -0.045 -0.040 -0.035 -0.030 -0.025 -0.020 -0.015 -0.010 -0.005 0.000 0.005 0.010 0.015 0.020 0.025 0.030 0.035 0.040 0.045 0.050 0.055 0.060 0.065 0.070 0.075 -40 -30 -20 -10 0 10 20 30 40 50 60 70 -30 -29 -28 -27 -26 -25 -24 -23 -22 -21 -20 -19 -18 -17 -16 -15 -14 -13 -12 -11 -10 -9 -8 -7 -6 -5 -4 -3 -2 -1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 -30 0 30 60 -30 -28 -26 -24 -22 -20 -18 -16 -14 -12 -10 -8 -6 -4 -2 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 48 50 52 54 56 58 60
p-value=1.628439911664703e-6

In [ ]:

Bayesian sign-test


In [8]:
using  Distributions
using DataFrames
using Gadfly
using Compose
include("Tests/Bsigntest.jl")
include("Plots/plot_simplex.jl")


#Bayesian Sign Test without rope
rope=0
data=Bsigntest(acci,accj,rope)

#Plot
df = DataFrame(Pleft=data[1,:][:])
p=plot(df, x=:Pleft, Geom.histogram,Theme(major_label_font_size=13pt,minor_label_font_size=12pt,key_label_font_size=11pt))
display(p)
draw(PDF("Plots/postSignTestnorope$cl1$cl2.pdf", 6inch, 3inch),p)

#Bayesian Sign Test with rope
rope=0.01
data=Bsigntest(acci,accj,rope)
#Plot
ptriangle=plot_simplex(data, ClassNames[cl1],ClassNames[cl2])
display(ptriangle)
draw(PDF("Plots/plotSignSimplex$cl1$cl2.pdf", 5inch, 5inch), ptriangle)

#Plot Marginals
df = DataFrame(Pleft=data[1,:][:], Prope=data[2,:][:])
p=plot(df, x=:Pleft, y=:Prope, Geom.histogram2d,Guide.xlabel(ClassNames[cl2]),Guide.ylabel("rope"),Theme(major_label_font_size=13pt,minor_label_font_size=12pt,key_label_font_size=11pt))
display(p)
draw(PDF("Plots/postSignTestA$cl1$cl2.pdf", 6inch, 3inch), p)

df = DataFrame(Pright=data[3,:][:], Prope=data[2,:][:])
p=plot(df, x=:Pright, y=:Prope, Geom.histogram2d,Guide.xlabel(ClassNames[cl1]),Guide.ylabel("rope"),Theme(major_label_font_size=13pt,minor_label_font_size=12pt,key_label_font_size=11pt))
display(p)
draw(PDF("Plots/postSignTestB$cl1$cl2.pdf", 6inch, 3inch), p)


Pleft -0.1 0.0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0 1.1 1.2 1.3 1.4 1.5 1.6 0.00 0.02 0.04 0.06 0.08 0.10 0.12 0.14 0.16 0.18 0.20 0.22 0.24 0.26 0.28 0.30 0.32 0.34 0.36 0.38 0.40 0.42 0.44 0.46 0.48 0.50 0.52 0.54 0.56 0.58 0.60 0.62 0.64 0.66 0.68 0.70 0.72 0.74 0.76 0.78 0.80 0.82 0.84 0.86 0.88 0.90 0.92 0.94 0.96 0.98 1.00 1.02 1.04 1.06 1.08 1.10 1.12 1.14 1.16 1.18 1.20 1.22 1.24 1.26 1.28 1.30 1.32 1.34 1.36 1.38 1.40 1.42 1.44 1.46 1.48 1.50 0.0 0.5 1.0 1.5 0.00 0.05 0.10 0.15 0.20 0.25 0.30 0.35 0.40 0.45 0.50 0.55 0.60 0.65 0.70 0.75 0.80 0.85 0.90 0.95 1.00 1.05 1.10 1.15 1.20 1.25 1.30 1.35 1.40 1.45 1.50 -3000 -2500 -2000 -1500 -1000 -500 0 500 1000 1500 2000 2500 3000 3500 4000 4500 5000 5500 -2500 -2400 -2300 -2200 -2100 -2000 -1900 -1800 -1700 -1600 -1500 -1400 -1300 -1200 -1100 -1000 -900 -800 -700 -600 -500 -400 -300 -200 -100 0 100 200 300 400 500 600 700 800 900 1000 1100 1200 1300 1400 1500 1600 1700 1800 1900 2000 2100 2200 2300 2400 2500 2600 2700 2800 2900 3000 3100 3200 3300 3400 3500 3600 3700 3800 3900 4000 4100 4200 4300 4400 4500 4600 4700 4800 4900 5000 -2500 0 2500 5000 -2600 -2400 -2200 -2000 -1800 -1600 -1400 -1200 -1000 -800 -600 -400 -200 0 200 400 600 800 1000 1200 1400 1600 1800 2000 2200 2400 2600 2800 3000 3200 3400 3600 3800 4000 4200 4400 4600 4800 5000
150 100 1 50 Count