In [1]:

    

%load_ext autoreload
%autoreload 2
%matplotlib inline

import numpy as np
import scipy as sp
import six
from matplotlib.pyplot import *


    

In [14]:

    

'''
Analyzing results of student2 with 2 skills, with training length 2 and testing length 2.
'''
data11 = np.load('experiments/test2_model2_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/stats-runA.npz')

vloss = data11['vloss']
#scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]


    

In [20]:

    

six.print_(vloss.shape)
figure()
x_avg = np.mean(vloss,axis=0)
x_serr = np.std(vloss,axis=0) / vloss.shape[0] * 1.98
plot(x_avg, color='#0000ff')
plot(x_avg + x_serr, color='#ddddff')
plot(x_avg - x_serr, color='#ddddff')

x_end = np.mean(x_avg[-2:])
x_thres = 0.0148
six.print_('last {} threshold {}'.format(x_end, x_thres))
plot([0,60],[x_end,x_end],color='#ff0000')
plot([0,60],[x_thres,x_thres],color='#ff0000')

xlim(30,40)
ylim(0.013,0.015)

# looks like epoch 54 is good for 0.0139
# looks like epoch 30 is good for 0.0148


    

    




(50, 55)
last 0.0138820153256 threshold 0.0148






    
Out[20]:





(0.013, 0.015)






    

In [9]:

    

'''
Analyzing results of student2 with 2 skills, with training length 2 and testing length 2.
'''
data11 = np.load('experiments/test2_model2simple_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/stats-runA.npz')

vloss = data11['vloss']
#scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]


    

In [13]:

    

six.print_(vloss.shape)
figure()
x_avg = np.mean(vloss,axis=0)
x_serr = np.std(vloss,axis=0) / vloss.shape[0] * 1.98
plot(x_avg, color='#0000ff')
plot(x_avg + x_serr, color='#ddddff')
plot(x_avg - x_serr, color='#ddddff')

x_end = np.mean(x_avg[-2:])
x_thres = 0.001
six.print_('last {} threshold {}'.format(x_end, x_thres))
plot([0,60],[x_end,x_end],color='#ff0000')
plot([0,60],[x_thres,x_thres],color='#ff0000')

#xlim(15,30)
#ylim(0.0,0.002)

# looks like epoch 46 is good for 0.0001
# looks like epoch 20 is good for 0.001


    

    




(50, 47)
last 0.000100714282317 threshold 0.001






    
Out[13]:





[<matplotlib.lines.Line2D at 0x7faa75a24fd0>]






    

In [2]:

    

'''
Analyzing results of student2 with 2 skills, with training length 2 and testing length 2.
'''
data11 = np.load('experiments/test2_model2gru_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/stats-runA.npz')

vloss = data11['vloss']
#scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]


    

In [8]:

    

six.print_(vloss.shape)
figure()
x_avg = np.mean(vloss,axis=0)
x_serr = np.std(vloss,axis=0) / vloss.shape[0] * 1.98
plot(x_avg, color='#0000ff')
plot(x_avg + x_serr, color='#ddddff')
plot(x_avg - x_serr, color='#ddddff')

x_end = np.mean(x_avg[-2:])
x_thres = 0.001
six.print_('last {} threshold {}'.format(x_end, x_thres))
plot([0,60],[x_end,x_end],color='#ff0000')
plot([0,60],[x_thres,x_thres],color='#ff0000')

xlim(15,20)
ylim(0.0,0.002)

# looks like epoch 43 is good for 0.0001
# looks like epoch 20 is good for 0.001


    

    




(20, 61)
last 2.86438517734e-05 threshold 0.001






    
Out[8]:





(0.0, 0.002)






    

In [2]:

    

'''
Analyzing results of student2 with 2 skills, with training length 2 and testing length 2.
'''
data11 = np.load('experiments/test2_model2_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/stats-runA.npz')
data21 = np.load('experiments/test2_model2_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/mcts-rtype2-rollouts1000-trajectories100-real1-runA.npz')

vloss = data11['vloss']
scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]


    

In [3]:

    

six.print_('vloss shape {}'.format(vloss.shape))
six.print_(vloss)
six.print_('scores shape {}'.format(scores.shape))
six.print_(scores)


    

    




vloss shape (50, 55)
[[ 0.09441637  0.0821061   0.06651635 ...,  0.01403089  0.01401687
   0.01400377]
 [ 0.09528936  0.0877762   0.07626525 ...,  0.01350397  0.01348733
   0.01347177]
 [ 0.09361646  0.0820771   0.06622386 ...,  0.01369305  0.01367851
   0.01366495]
 ..., 
 [ 0.09256435  0.0779258   0.06302304 ...,  0.01383118  0.01381435
   0.01379868]
 [ 0.09070774  0.07706053  0.06221853 ...,  0.01358032  0.01356731
   0.01355519]
 [ 0.09316752  0.07823799  0.06175289 ...,  0.01400177  0.0139882
   0.01397556]]
scores shape (50,)
[ 1.       0.       1.       1.       1.       0.84375  0.       1.       1.
  1.       1.       1.       1.       1.       1.       1.       1.       1.
  1.       1.       0.       1.       1.       1.       1.       1.       1.
  1.       1.       1.       1.       1.       1.       1.       1.       1.
  1.       1.       1.       1.       1.       1.       1.       1.       1.
  1.       1.       1.       1.       1.     ]

In [5]:

    

xs = np.sum(vloss,axis=1)
ys = scores
title('Training Loss AUC versus Actual Performance')
xlabel('Training Loss AUC')
ylabel('Posttest Score')
plot(xs,ys,'.')


    

    
Out[5]:





[<matplotlib.lines.Line2D at 0x7f36517afd10>]






    

In [3]:

    

'''
Analyzing results of student2 with 2 skills, with training length 2 and testing length 2. With the simpler model
'''
data11 = np.load('experiments/test2_model2simple_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/stats-runA.npz')
data21 = np.load('experiments/test2_model2simple_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/mcts-rtype2-rollouts1000-trajectories100-real1-runA.npz')

vloss = data11['vloss']
scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]


    

In [4]:

    

six.print_('vloss shape {}'.format(vloss.shape))
six.print_(vloss)
six.print_('scores shape {}'.format(scores.shape))
six.print_(scores)


    

    




vloss shape (50, 47)
[[  5.60354423e-02   0.00000000e+00   3.45983439e-02 ...,   1.07799362e-04
    9.98436733e-05   9.25175496e-05]
 [  5.61487071e-02   4.63366595e-02   3.68188272e-02 ...,   9.92860048e-05
    9.18873619e-05   8.50844727e-05]
 [  5.51633144e-02   4.41024173e-02   3.34944126e-02 ...,   1.16312240e-04
    1.07566351e-04   9.95353181e-05]
 ..., 
 [  5.55556385e-02   4.53986116e-02   3.51159690e-02 ...,   1.21000005e-04
    1.12096910e-04   1.03894910e-04]
 [  5.49687211e-02   4.39238912e-02   3.30713904e-02 ...,   1.29236685e-04
    1.19730405e-04   1.10972159e-04]
 [  5.55256202e-02   4.51044651e-02   3.49070457e-02 ...,   1.15971312e-04
    1.07432622e-04   9.95662725e-05]]
scores shape (50,)
[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  0.  1.  1.  1.  1.  1.  1.  1.  1.
  1.  1.  1.  1.  0.  1.  1.  1.  1.  1.  1.  1.  1.  1.  0.  1.  1.  1.
  1.  1.  0.  1.  1.  1.  1.  0.  0.  0.  1.  1.  1.  1.]

In [5]:

    

xs = np.sum(vloss,axis=1)
ys = scores
title('Training Loss AUC versus Actual Performance')
xlabel('Training Loss AUC')
ylabel('Posttest Score')
plot(xs,ys,'.')


    

    
Out[5]:





[<matplotlib.lines.Line2D at 0x7fae11912310>]






    

In [2]:

    

'''
Analyzing results of student2 with 2 skills, with training length 2 and testing length 2. With the gru model
'''
data11 = np.load('experiments/test2_model2gru_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/stats-runA.npz')
data21 = np.load('experiments/test2_model2gru_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/mcts-rtype2-rollouts1000-trajectories100-real1-runA.npz')

vloss = data11['vloss']
scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]


    

In [4]:

    

six.print_('vloss shape {}'.format(vloss.shape))
six.print_(vloss)
six.print_('scores shape {}'.format(scores.shape))
six.print_(scores)


    

    




vloss shape (50, 44)
[[  1.24660005e-01   9.74883808e-02   6.46064837e-02 ...,   1.90614390e-04
    1.73033830e-04   1.57141048e-04]
 [  1.26972781e-01   1.00285856e-01   6.67210233e-02 ...,   9.77403937e-05
    9.03463540e-05   8.35759406e-05]
 [  1.18413929e-01   8.01361284e-02   4.64666935e-02 ...,   8.62135920e-05
    7.91471312e-05   7.27472454e-05]
 ..., 
 [  1.22993569e-01   9.04669706e-02   5.20611677e-02 ...,   1.20912361e-04
    1.11792422e-04   1.03415427e-04]
 [  1.31132155e-01   1.02064132e-01   6.27080469e-02 ...,   1.24872571e-04
    1.15411477e-04   1.06727092e-04]
 [  1.26553115e-01   1.02315676e-01   6.59910226e-02 ...,   1.15525539e-04
    1.06497150e-04   9.82490103e-05]]
scores shape (50,)
[ 0.          0.          1.          0.          0.          0.          1.
  0.          1.          0.          0.          0.          0.          0.
  0.          1.          0.          0.          1.          0.          0.
  0.          1.          0.92708333  1.          0.          1.          0.
  0.          0.          0.          0.          0.          0.          1.
  0.          0.          0.          0.83333333  0.          0.          1.
  0.          0.          0.          1.          0.          1.          1.
  0.        ]

In [3]:

    

xs = np.sum(vloss,axis=1)
ys = scores
title('Training Loss AUC versus Actual Performance')
xlabel('Training Loss AUC')
ylabel('Posttest Score')
plot(xs,ys,'.')


    

    
Out[3]:





[<matplotlib.lines.Line2D at 0x7f3fa586bb50>]






    

In [3]:

    

'''
Analyzing results of student2 with 2 skills, with training length 2 and testing length 2. With fewer training epochs.
'''
data11 = np.load('experiments/test2_model2_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/stats-runB.npz')
data21 = np.load('experiments/test2_model2_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/mcts-rtype2-rollouts1000-trajectories100-real1-runB.npz')

vloss = data11['vloss']
scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]

six.print_('vloss shape {}'.format(vloss.shape))
six.print_(vloss)
six.print_('scores shape {}'.format(scores.shape))
six.print_(scores)

xs = np.sum(vloss,axis=1)
ys = scores
title('Training Loss AUC versus Actual Performance')
xlabel('Training Loss AUC')
ylabel('Posttest Score')
plot(xs,ys,'.')


    

    




vloss shape (50, 31)
[[ 0.08936516  0.0754681   0.06283438 ...,  0.01514173  0.01501562
   0.01490227]
 [ 0.09201896  0.07820546  0.06347355 ...,  0.01518269  0.01508005
   0.01498646]
 [ 0.0960233   0.08372398  0.07006697 ...,  0.015364    0.0152562
   0.01515773]
 ..., 
 [ 0.08906623  0.07618408  0.06147273 ...,  0.01456837  0.01447097
   0.01438174]
 [ 0.0943704   0.0821831   0.06846968 ...,  0.0154057   0.01529267
   0.01518957]
 [ 0.08907868  0.07548768  0.06275198 ...,  0.01502329  0.01490513
   0.01479774]]
scores shape (50,)
[ 1.          1.          1.          1.          1.          1.          1.
  1.          1.          0.          1.          1.          1.          1.
  1.          0.01041667  1.          1.          1.          1.          1.
  1.          1.          1.          1.          1.          1.          1.
  1.          1.          1.          1.          1.          1.          1.
  1.          1.          1.          1.          1.          1.          1.
  1.          1.          1.          1.          1.          1.          0.
  1.        ]






    
Out[3]:





[<matplotlib.lines.Line2D at 0x7ff244d22b90>]






    

In [4]:

    

'''
Analyzing results of student2 with 2 skills, with training length 2 and testing length 2. Simpler model with fewer training epochs.
'''
data11 = np.load('experiments/test2_model2simple_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/stats-runB.npz')
data21 = np.load('experiments/test2_model2simple_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/mcts-rtype2-rollouts1000-trajectories100-real1-runB.npz')

vloss = data11['vloss']
scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]

six.print_('vloss shape {}'.format(vloss.shape))
six.print_(vloss)
six.print_('scores shape {}'.format(scores.shape))
six.print_(scores)

xs = np.sum(vloss,axis=1)
ys = scores
title('Training Loss AUC versus Actual Performance')
xlabel('Training Loss AUC')
ylabel('Posttest Score')
plot(xs,ys,'.')


    

    




vloss shape (50, 21)
[[ 0.05621311  0.04638603  0.03557363 ...,  0.00123461  0.00109884
   0.00098196]
 [ 0.05526491  0.0446969   0.03436239 ...,  0.00126352  0.00110778
   0.00097658]
 [ 0.05540264  0.04436184  0.03365188 ...,  0.00126147  0.00111623
   0.00099174]
 ..., 
 [ 0.05540877  0.0454841   0.0356551  ...,  0.00119104  0.00105837
   0.00094481]
 [ 0.05590601  0.0456485   0.03532783 ...,  0.00121171  0.00108016
   0.00096674]
 [ 0.05541185  0.04434226  0.03342759 ...,  0.00134842  0.00119605
   0.00106492]]
scores shape (50,)
[ 1.          1.          1.          1.          1.          1.          1.
  0.92708333  1.          0.          1.          1.          1.          1.
  1.          1.          1.          1.          0.98958333  1.          1.
  0.96875     0.          1.          1.          1.          1.          1.
  0.          1.          1.          1.          1.          1.          1.
  1.          0.          1.          1.          1.          1.          1.
  1.          1.          1.          1.          1.          1.          1.
  1.        ]






    
Out[4]:





[<matplotlib.lines.Line2D at 0x7ff2455bec50>]






    

In [5]:

    

'''
Analyzing results of student2 with 2 skills, with training length 2 and testing length 2. GRU model with fewer training epochs.
'''
data11 = np.load('experiments/test2_model2gru_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/stats-runB.npz')
data21 = np.load('experiments/test2_model2gru_tiny-dropout10-shuffle0-data-test2-n10000-l2-random.pickle/mcts-rtype2-rollouts1000-trajectories100-real1-runB.npz')

vloss = data11['vloss']
scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]

six.print_('vloss shape {}'.format(vloss.shape))
six.print_(vloss)
six.print_('scores shape {}'.format(scores.shape))
six.print_(scores)

xs = np.sum(vloss,axis=1)
ys = scores
title('Training Loss AUC versus Actual Performance')
xlabel('Training Loss AUC')
ylabel('Posttest Score')
plot(xs,ys,'.')


    

    




vloss shape (50, 21)
[[ 0.1333752   0.11080759  0.07659189 ...,  0.00111848  0.00095859
   0.0008297 ]
 [ 0.12815169  0.09953441  0.06451445 ...,  0.00127968  0.00112063
   0.00098746]
 [ 0.13361075  0.1095274   0.07571035 ...,  0.00124767  0.00107896
   0.00094073]
 ..., 
 [ 0.125384    0.09691066  0.06426334 ...,  0.00125629  0.00109703
   0.00096482]
 [ 0.12904907  0.10344235  0.0708679  ...,  0.001012    0.00088422
   0.00077725]
 [ 0.12629925  0.10087278  0.06796972 ...,  0.00175584  0.00152009
   0.0013258 ]]
scores shape (50,)
[ 0.94791667  0.          1.          1.          0.          1.          1.
  0.          0.          0.          0.          0.          0.          0.
  1.          0.          1.          1.          0.          0.
  0.83333333  1.          1.          0.          0.25        1.          1.
  1.          1.          0.          1.          0.          0.          1.
  1.          1.          0.          0.91666667  0.35416667  1.          0.
  1.          0.          0.          1.          0.          1.          1.
  1.          1.        ]






    
Out[5]:





[<matplotlib.lines.Line2D at 0x7ff244d22c90>]






    

In [5]:

    

'''
Analyzing results of student2 with 2 skills, with training length 3 and testing length 2.
Double LSTM
Trying to find a training epoch to stop training.
'''
data11 = np.load('experiments/test2_model2_tiny-dropout10-shuffle0-data-test2-n10000-l3-random.pickle/stats-runA.npz')

vloss = data11['vloss']
#scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]

six.print_(vloss.shape)
figure()
x_avg = np.mean(vloss,axis=0)
x_serr = np.std(vloss,axis=0) / vloss.shape[0] * 1.98
plot(x_avg, color='#0000ff')
plot(x_avg + x_serr, color='#ddddff')
plot(x_avg - x_serr, color='#ddddff')

x_end = np.mean(x_avg[-2:])
x_thres = 0.012
six.print_('last {} threshold {}'.format(x_end, x_thres))
plot([0,60],[x_end,x_end],color='#ff0000')
plot([0,60],[x_thres,x_thres],color='#ff0000')

#xlim(30,40)
ylim(0.011,0.013)

# looks like epoch 40 is good for 0.012


    

    




(20, 61)
last 0.0113193088135 threshold 0.012






    
Out[5]:





(0.011, 0.013)






    

In [9]:

    

'''
Analyzing results of student2 with 2 skills, with training length 3 and testing length 2.
Single LSTM
Trying to find a training epoch to stop training.
'''
data11 = np.load('experiments/test2_model2simple_tiny-dropout10-shuffle0-data-test2-n10000-l3-random.pickle/stats-runA.npz')

vloss = data11['vloss']
#scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]

six.print_(vloss.shape)
figure()
x_avg = np.mean(vloss,axis=0)
x_serr = np.std(vloss,axis=0) / vloss.shape[0] * 1.98
plot(x_avg, color='#0000ff')
plot(x_avg + x_serr, color='#ddddff')
plot(x_avg - x_serr, color='#ddddff')

x_end = np.mean(x_avg[-2:])
x_thres = 0.001
six.print_('last {} threshold {}'.format(x_end, x_thres))
plot([0,60],[x_end,x_end],color='#ff0000')
plot([0,60],[x_thres,x_thres],color='#ff0000')

#xlim(30,40)
#ylim(0.0,0.002)

# looks like epoch 25 is good for 0.001


    

    




(20, 61)
last 4.15097025594e-05 threshold 0.001






    
Out[9]:





[<matplotlib.lines.Line2D at 0x7f885ec45f10>]






    

In [11]:

    

'''
Analyzing results of student2 with 2 skills, with training length 3 and testing length 2.
Double GRU
Trying to find a training epoch to stop training.
'''
data11 = np.load('experiments/test2_model2gru_tiny-dropout10-shuffle0-data-test2-n10000-l3-random.pickle/stats-runA.npz')

vloss = data11['vloss']
#scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]

six.print_(vloss.shape)
figure()
x_avg = np.mean(vloss,axis=0)
x_serr = np.std(vloss,axis=0) / vloss.shape[0] * 1.98
plot(x_avg, color='#0000ff')
plot(x_avg + x_serr, color='#ddddff')
plot(x_avg - x_serr, color='#ddddff')

x_end = np.mean(x_avg[-2:])
x_thres = 0.001
six.print_('last {} threshold {}'.format(x_end, x_thres))
plot([0,60],[x_end,x_end],color='#ff0000')
plot([0,60],[x_thres,x_thres],color='#ff0000')

#xlim(30,40)
ylim(0.0,0.002)

# looks like epoch 25 is good for 0.001


    

    




(20, 61)
last 3.51745995667e-05 threshold 0.001






    
Out[11]:





(0.0, 0.002)






    

In [2]:

    

'''
Analyzing results of student2 with 2 skills, with training length 3 and testing length 2.
Double LSTM model.
Looking at correlation between training AUC and actual performance
'''
data11 = np.load('experiments/test2_model2_tiny-dropout10-shuffle0-data-test2-n10000-l3-random.pickle/stats-runB.npz')
data21 = np.load('experiments/test2_model2_tiny-dropout10-shuffle0-data-test2-n10000-l3-random.pickle/mcts-rtype2-rollouts1000-trajectories100-real1-runB.npz')

vloss = data11['vloss']
scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]

six.print_('vloss shape {}'.format(vloss.shape))
six.print_(vloss)
six.print_('scores shape {}'.format(scores.shape))
six.print_(scores)

xs = np.sum(vloss,axis=1)
ys = scores
title('Training Loss AUC versus Actual Performance')
xlabel('Training Loss AUC')
ylabel('Posttest Score')
plot(xs,ys,'.')


    

    




vloss shape (50, 41)
[[ 0.11973911  0.08873725  0.062083   ...,  0.01210291  0.01204618
   0.0119935 ]
 [ 0.12763282  0.10679469  0.07714722 ...,  0.01184324  0.01179229
   0.01174514]
 [ 0.12275101  0.10102007  0.07785704 ...,  0.01197197  0.01191723
   0.01186645]
 ..., 
 [ 0.11925535  0.08805766  0.06705227 ...,  0.01219413  0.01213706
   0.01208434]
 [ 0.12563111  0.09619637  0.0692479  ...,  0.01184236  0.0117885
   0.01173862]
 [ 0.11884481  0.0890677   0.06535447 ...,  0.01169011  0.01163844
   0.01159064]]
scores shape (50,)
[ 1.       1.       1.       1.       1.       1.       1.       0.       1.
  1.       1.       1.       1.       1.       1.       1.       1.       1.
  1.       1.       1.       1.       1.       1.       1.       1.       1.
  1.       1.       1.       1.       1.       1.       1.       1.       1.
  1.       1.       1.       1.       0.21875  1.       0.       1.       1.
  1.       1.       1.       1.       1.     ]






    
Out[2]:





[<matplotlib.lines.Line2D at 0x7f99b5d914d0>]






    

In [4]:

    

'''
Analyzing results of student2 with 2 skills, with training length 3 and testing length 2.
Single LSTM model.
Looking at correlation between training AUC and actual performance
'''
data11 = np.load('experiments/test2_model2simple_tiny-dropout10-shuffle0-data-test2-n10000-l3-random.pickle/stats-runB.npz')
data21 = np.load('experiments/test2_model2simple_tiny-dropout10-shuffle0-data-test2-n10000-l3-random.pickle/mcts-rtype2-rollouts1000-trajectories100-real1-runB.npz')

vloss = data11['vloss']
scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]

six.print_('vloss shape {}'.format(vloss.shape))
six.print_(vloss)
six.print_('scores shape {}'.format(scores.shape))
six.print_(scores)

xs = np.sum(vloss,axis=1)
ys = scores
title('Training Loss AUC versus Actual Performance')
xlabel('Training Loss AUC')
ylabel('Posttest Score')
plot(xs,ys,'.')


    

    




vloss shape (50, 26)
[[ 0.07038667  0.05296684  0.04162331 ...,  0.00086331  0.00077806
   0.0007032 ]
 [ 0.0744532   0.06084078  0.04826403 ...,  0.00063225  0.00057206
   0.00051893]
 [ 0.07082326  0.05457835  0.04520417 ...,  0.00087264  0.00078487
   0.00070826]
 ..., 
 [ 0.07223552  0.05409869  0.04126726 ...,  0.00088338  0.00079667
   0.00072036]
 [ 0.07215533  0.05550894  0.0463388  ...,  0.00082692  0.00073836
   0.00066132]
 [ 0.07042608  0.05536843  0.04656875 ...,  0.00083239  0.00074515
   0.00066937]]
scores shape (50,)
[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.
  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.
  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.]






    
Out[4]:





[<matplotlib.lines.Line2D at 0x7f99b65f7a90>]






    

In [5]:

    

'''
Analyzing results of student2 with 2 skills, with training length 3 and testing length 2.
Double GRU model
Looking at correlation between training AUC and actual performance
'''
data11 = np.load('experiments/test2_model2gru_tiny-dropout10-shuffle0-data-test2-n10000-l3-random.pickle/stats-runB.npz')
data21 = np.load('experiments/test2_model2gru_tiny-dropout10-shuffle0-data-test2-n10000-l3-random.pickle/mcts-rtype2-rollouts1000-trajectories100-real1-runB.npz')

vloss = data11['vloss']
scores = data21['scores'][:,0]
#initialq = data51['qvals'][:,0]
#opts = data61['opts']
#qfuncs = data61['qs'][:,0,:,:]

six.print_('vloss shape {}'.format(vloss.shape))
six.print_(vloss)
six.print_('scores shape {}'.format(scores.shape))
six.print_(scores)

xs = np.sum(vloss,axis=1)
ys = scores
title('Training Loss AUC versus Actual Performance')
xlabel('Training Loss AUC')
ylabel('Posttest Score')
plot(xs,ys,'.')


    

    




vloss shape (50, 26)
[[ 0.16523354  0.13730846  0.08717519 ...,  0.00076391  0.00067873
   0.0006053 ]
 [ 0.15543124  0.11512258  0.07533636 ...,  0.00139091  0.00122447
   0.0010836 ]
 [ 0.14960701  0.10779101  0.06741117 ...,  0.0006807   0.00061247
   0.00055265]
 ..., 
 [ 0.16040278  0.11989488  0.06712871 ...,  0.00071317  0.0006422
   0.00057992]
 [ 0.1577603   0.11754873  0.06949968 ...,  0.00063443  0.00056828
   0.00051057]
 [ 0.1591086   0.10939669  0.06788692 ...,  0.00104207  0.00093177
   0.0008365 ]]
scores shape (50,)
[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.
  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.
  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.]






    
Out[5]:





[<matplotlib.lines.Line2D at 0x7f99b37d2550>]






    

In [ ]: