In [2]:
ClassID = readdlm("Data/ClassifierID.dat", ',')
ClassNames = readdlm("Data/ClassifierNames.dat", ',')
DatasetID = readdlm("Data/DatasetID.dat", ',');
DatasetNames = readdlm("Data/DatasetNames.dat", ',');
Percent_correct = readdlm("Data/Percent_correct.dat", ',');
cl1=1 #nbc
cl2=2 #aode
println("Comparison of ", ClassNames[cl1,1], " vs. ", ClassNames[cl2,1])
println()
#Compute mean accuracy
indi=find(x->x==cl1,ClassID)
indj=find(x->x==cl2,ClassID)
acci=Float64[]
accj=Float64[]
for d=1:Int32(maximum(DatasetID))
indd=find(x->x==d,DatasetID)
indid=intersect(indi,indd)
indjd=intersect(indj,indd)
push!(acci,mean(Percent_correct[indid])/100)
push!(accj,mean(Percent_correct[indjd])/100)
if mod(d,3)==0
@printf "%s & %1.3f & %s & %1.3f & %s & %1.3f \\\\\n" DatasetNames[d-2] (acci[d-2]-accj[d-2])*100 DatasetNames[d-1] (acci[d-1]-accj[d-1])*100 DatasetNames[d] (acci[d]-accj[d])*100
end
end
In [4]:
using HypothesisTests
using Gadfly
using DataFrames
include("Plots/plot_data.jl")
diff=acci-accj
#Density plot of difference of accuracies
p=plot_data(cl1,cl2,[],diff,-0.03,0.02)
display(p)
#Compute p-value Sign-Rank Test
pval_SignRankTest=pvalue(SignedRankTest(acci,accj))
println("p-value=$pval_SignRankTest")
#Compute p-value SignTest
#pvalSignTest=pvalue(SignTest(acci-accj), tail=:left)
#println("p-value=$pvalSignTest")
In [ ]:
In [8]:
using Distributions
using DataFrames
using Gadfly
using Compose
include("Tests/Bsigntest.jl")
include("Plots/plot_simplex.jl")
#Bayesian Sign Test without rope
rope=0
data=Bsigntest(acci,accj,rope)
#Plot
df = DataFrame(Pleft=data[1,:][:])
p=plot(df, x=:Pleft, Geom.histogram,Theme(major_label_font_size=13pt,minor_label_font_size=12pt,key_label_font_size=11pt))
display(p)
draw(PDF("Plots/postSignTestnorope$cl1$cl2.pdf", 6inch, 3inch),p)
#Bayesian Sign Test with rope
rope=0.01
data=Bsigntest(acci,accj,rope)
#Plot
ptriangle=plot_simplex(data, ClassNames[cl1],ClassNames[cl2])
display(ptriangle)
draw(PDF("Plots/plotSignSimplex$cl1$cl2.pdf", 5inch, 5inch), ptriangle)
#Plot Marginals
df = DataFrame(Pleft=data[1,:][:], Prope=data[2,:][:])
p=plot(df, x=:Pleft, y=:Prope, Geom.histogram2d,Guide.xlabel(ClassNames[cl2]),Guide.ylabel("rope"),Theme(major_label_font_size=13pt,minor_label_font_size=12pt,key_label_font_size=11pt))
display(p)
draw(PDF("Plots/postSignTestA$cl1$cl2.pdf", 6inch, 3inch), p)
df = DataFrame(Pright=data[3,:][:], Prope=data[2,:][:])
p=plot(df, x=:Pright, y=:Prope, Geom.histogram2d,Guide.xlabel(ClassNames[cl1]),Guide.ylabel("rope"),Theme(major_label_font_size=13pt,minor_label_font_size=12pt,key_label_font_size=11pt))
display(p)
draw(PDF("Plots/postSignTestB$cl1$cl2.pdf", 6inch, 3inch), p)