In [1]:
import pandas as pd
import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt
import time
from scipy import stats
from scipy.optimize import minimize
In [2]:
stud_learning = pd.read_csv('student_learning_final.csv')
stud_learning.drop(['Unnamed: 0'], axis=1, inplace=True)
cluster_index = pd.read_csv("cluster_index.csv", header=None)
stud_learning['cluster_index'] = cluster_index[1]
stud_learning['frac_incorrect_atts'] = stud_learning['number of incorrect attempts'] / stud_learning['number of attempts']
stud_learning.head()
Out[2]:
In [3]:
stud_learning.columns
Out[3]:
In [4]:
stud_data = pd.read_hdf('stud_data.hdf','test')
stud_data.head()
Out[4]:
In [5]:
stud_data = stud_data.join(stud_learning)
stud_data.head()
Out[5]:
Determine what clusters more successful in learning in terms of fraction of correct attempts:
In [6]:
stud_data_sum = stud_data.groupby('cluster_index').agg(np.sum).copy()
stud_data_sum['frac_incorrect_atts'] = stud_data_sum['number of incorrect attempts'] / stud_data_sum['number of attempts']
stud_data_sum
Out[6]:
Interestingly, group 3 has the smallest fraction of incorrect attempts (~36.5%). Also, not surprisingly, 'frac_incorrect_atts' in group 1 (with large 'frac_1s_hints') is significantly (p-value = 1.75e-8) smaller than in group 2 (with small 'frac_1s_hints' and large 'frac_3s_atts'):
In [7]:
arr1 = np.array(stud_data[stud_data['cluster_index'] == 1]['frac_incorrect_atts'])
arr2 = np.array(stud_data[stud_data['cluster_index'] == 2]['frac_incorrect_atts'])
arr1 = arr1[~np.isnan(arr1)]
arr2 = arr2[~np.isnan(arr2)]
stats.ttest_ind(arr1,arr2, equal_var = False)
Out[7]:
However, the difference of 'frac_incorrect_atts' between students with "gaming" and non-gaming behaviour is not significant (p-value = 0.83):
In [8]:
arr_gam = np.array(stud_data[stud_data['cluster_index'] <= 2]['frac_incorrect_atts'])
arr_nongam = np.array(stud_data[stud_data['cluster_index'] > 2]['frac_incorrect_atts'])
arr_gam = arr_gam[~np.isnan(arr_gam)]
arr_nongam = arr_nongam[~np.isnan(arr_nongam)]
stats.ttest_ind(arr_gam,arr_nongam, equal_var = False)
Out[8]:
Notably, group 5 (students with medium 'num_sess' and 'num_probs') has significantly smaller 'frac_incorrect_atts' than group 4 (students with small 'num_sess' and 'num_probs') but significantly smaller 'frac_incorrect_atts' than in group 6 (students with large 'num_sess' and 'num_probs'):
In [9]:
arr4 = np.array(stud_data[stud_data['cluster_index'] == 4]['frac_incorrect_atts'])
arr5 = np.array(stud_data[stud_data['cluster_index'] == 5]['frac_incorrect_atts'])
arr6 = np.array(stud_data[stud_data['cluster_index'] == 6]['frac_incorrect_atts'])
arr4 = arr4[~np.isnan(arr4)]
arr5 = arr5[~np.isnan(arr5)]
arr6 = arr6[~np.isnan(arr6)]
print(stats.ttest_ind(arr5,arr4, equal_var = False))
print(stats.ttest_ind(arr6,arr5, equal_var = False))
In other words, for students with non-gaming behaviour 'frac_incorrect_atts' steadily decreases with learning experience.
Other differences between groups:
In [10]:
stud_data_mean = stud_data.groupby('cluster_index').agg(np.mean).copy()
stud_data_mean
Out[10]:
There is a significant increase of 'frac_3s_atts' for groups 4-5-6:
In [11]:
arr4 = np.array(stud_data[stud_data['cluster_index'] == 4]['frac_3s_atts'])
arr5 = np.array(stud_data[stud_data['cluster_index'] == 5]['frac_3s_atts'])
arr6 = np.array(stud_data[stud_data['cluster_index'] == 6]['frac_3s_atts'])
arr4 = arr4[~np.isnan(arr4)]
arr5 = arr5[~np.isnan(arr5)]
arr6 = arr6[~np.isnan(arr6)]
print(stats.ttest_ind(arr5,arr4, equal_var = False))
print(stats.ttest_ind(arr6,arr5, equal_var = False))
, a significant increase of 'frac_1s_hints' for groups 4-5-6:
In [12]:
arr4 = np.array(stud_data[stud_data['cluster_index'] == 4]['frac_1s_hints'])
arr5 = np.array(stud_data[stud_data['cluster_index'] == 5]['frac_1s_hints'])
arr6 = np.array(stud_data[stud_data['cluster_index'] == 6]['frac_1s_hints'])
arr4 = arr4[~np.isnan(arr4)]
arr5 = arr5[~np.isnan(arr5)]
arr6 = arr6[~np.isnan(arr6)]
print(stats.ttest_ind(arr5,arr4, equal_var = False))
print(stats.ttest_ind(arr6,arr5, equal_var = False))
, and a significant decrease of 'max_atts' for groups 4-5-6:
In [13]:
arr4 = np.array(stud_data[stud_data['cluster_index'] == 4]['max_atts'])
arr5 = np.array(stud_data[stud_data['cluster_index'] == 5]['max_atts'])
arr6 = np.array(stud_data[stud_data['cluster_index'] == 6]['max_atts'])
arr4 = arr4[~np.isnan(arr4)]
arr5 = arr5[~np.isnan(arr5)]
arr6 = arr6[~np.isnan(arr6)]
print(stats.ttest_ind(arr5,arr4, equal_var = False))
print(stats.ttest_ind(arr6,arr5, equal_var = False))
Increase of max_probl_views is significant between groups 5 and 6:
In [14]:
arr4 = np.array(stud_data[stud_data['cluster_index'] == 4]['max_probl_views'])
arr5 = np.array(stud_data[stud_data['cluster_index'] == 5]['max_probl_views'])
arr6 = np.array(stud_data[stud_data['cluster_index'] == 6]['max_probl_views'])
arr4 = arr4[~np.isnan(arr4)]
arr5 = arr5[~np.isnan(arr5)]
arr6 = arr6[~np.isnan(arr6)]
print(stats.ttest_ind(arr5,arr4, equal_var = False))
print(stats.ttest_ind(arr6,arr5, equal_var = False))
As we see, increasing "experience" (in group sequence 4-5-6) also leads to:
'frac_3s_atts' and 'frac_1s_hints';'max_probl_views' (so the problems are viewed in more details);'max_atts' (so there are smaller attempts per problem).Because learning parameter is determined from the fit of the learning curve, it is essential to analyse learning curves starting with some reasonable minimum number of attempts.
As we see, there is a very large spread in learning parameters: between -9.97 and 24.9:
In [15]:
stud_data['learning_parameter'].describe()
Out[15]:
Moreover, for ~1/3 of students learning parameter is negative:
In [16]:
stud_data[stud_data['learning_parameter'] < 0].shape[0]
Out[16]:
What best describes "negative learners"? First, look on extreme examples. Take "extreme negative learners" (students with learning rate < -0.5, and compare them with "extreme positive learnens" (students with learning rate > 0.5):
In [17]:
stud_data[stud_data['learning_parameter'] < -0.5].describe()
Out[17]:
In [18]:
stud_data[stud_data['learning_parameter'] >= 0.5].describe()
Out[18]:
As we see, "extreme negative learners" are very similar to "extreme positive learners" in terms of 'frac_corr_atts'. However, they opened much more sessions ('num_sess'), tried to solve more problems ('num_probs'), made more attempts to solve the problems ('num_atts') and spent more time (time_atts) for solving them.
In [19]:
stud_data[stud_data['learning_parameter'] < -0.5].groupby('cluster_index').agg(len)['num_sess']
Out[19]:
In [20]:
stud_data[stud_data['learning_parameter'] >= 0.5].groupby('cluster_index').agg(len)['num_sess']
Out[20]:
Notably, most of "extreme learners" belong to groups 3 and 4 that have the smallest 'num_atts':
In [21]:
stud_data.groupby('cluster_index').agg(np.mean)
Out[21]:
Together with the smallest group 2, groups 3 and 4 also correspond to the largest 'learning_parameter' variation across their members:
In [22]:
stud_data.groupby('cluster_index').agg(np.std)['learning_parameter']
Out[22]:
Note that both absolute average value and standard deviation come to zero with increasing 'num_atts': a possible manifestation of the Plateau effect.
Finally, I try to explain the observed increase of 'frac_incorrect_atts' for 5 and more attempts.
The group with the largest 'frac_incorrect_atts' is group 4 that also has the largest 'max_atts' (number of maximal attempts averaged for all assessed problems). Consequently, students from two groups with the smallest 'frac_incorrect_atts' (groups 3 and 6) also have the smallest 'max_atts':
In [23]:
stud_data.groupby('cluster_index').agg(np.mean)[['frac_incorrect_atts', 'max_atts']].corr()
Out[23]:
In [24]:
stud_data.corr()['frac_incorrect_atts']['max_atts']
Out[24]:
As a result, students from groups 3 and 6 contribute less (and students from group 4 contribute more) to problems with large number of attempts and distort the averaged learning curve towards larger 'frac_incorrect_atts'.
In [25]:
stud_data['gaming_index'] = stud_data['cluster_index'].replace({1: 0, 2: 0, 3: 1, 4: 1, 5: 1, 6:1})
In [26]:
stud_data.groupby('gaming_index').agg(np.mean)
Out[26]:
In [27]:
data = pd.read_hdf('data.hdf','test')
data.head()
Out[27]:
In [28]:
stud_list = data['Anon Student Id'].unique()
#print(stud_list[:5])
stud_dict = {stud: cluster_index.loc[i, 1] for i, stud in enumerate(stud_list)}
In [29]:
stud_list[0]
Out[29]:
In [30]:
cluster_index.loc[0, 1]
Out[30]:
In [31]:
stud_dict['Stu_001d187b1b375fe98b88696b250177f0']
Out[31]:
In [32]:
stud_df = pd.DataFrame()
for item in stud_dict:
#print(item, stud_dict[item])
stud_df.loc[item, 'cluster_index'] = int(stud_dict[item])
stud_df.head()
Out[32]:
In [33]:
data_125 = data[(data['Anon Student Id'].isin(stud_df[stud_df['cluster_index'] == 1].index)) | \
(data['Anon Student Id'].isin(stud_df[stud_df['cluster_index'] == 2].index)) | \
(data['Anon Student Id'].isin(stud_df[stud_df['cluster_index'] == 5].index))]
data_125.head()
Out[33]:
In [34]:
data_2 = data[(data['Anon Student Id'].isin(stud_df[stud_df['cluster_index'] == 2].index))]
In [35]:
s1 = data[data['Outcome'] <= 1].groupby(['x']).agg(len)['Problem Name']
s2 = data[data['Outcome'] == 1].groupby(['x']).agg(len)['Problem Name']
s1[8] = s1.loc[8:].sum()
for i in range(9, int(s1.index.max()+1)):
try:
s1.drop(i, inplace=True)
except ValueError:
pass
s2[8] = s2.loc[8:].sum()
for i in range(9, int(s2.index.max()+1)):
try:
s2.drop(i, inplace=True)
except ValueError:
pass
In [36]:
data_1 = data[(data['Anon Student Id'].isin(stud_df[stud_df['cluster_index'] == 1].index))]
data_2 = data[(data['Anon Student Id'].isin(stud_df[stud_df['cluster_index'] == 2].index))]
data_3 = data[(data['Anon Student Id'].isin(stud_df[stud_df['cluster_index'] == 3].index))]
data_4 = data[(data['Anon Student Id'].isin(stud_df[stud_df['cluster_index'] == 4].index))]
data_5 = data[(data['Anon Student Id'].isin(stud_df[stud_df['cluster_index'] == 5].index))]
data_6 = data[(data['Anon Student Id'].isin(stud_df[stud_df['cluster_index'] == 6].index))]
s1_1 = data_1[data_1['Outcome'] <= 1].groupby(['x']).agg(len)['Problem Name']
s2_1 = data_1[data_1['Outcome'] == 1].groupby(['x']).agg(len)['Problem Name']
s1_1[8] = s1_1.loc[8:].sum()
for i in range(9, int(s1_1.index.max()+1)):
try:
s1_1.drop(i, inplace=True)
except ValueError:
pass
s2_1[8] = s2_1.loc[8:].sum()
for i in range(9, int(s2_1.index.max()+1)):
try:
s2_1.drop(i, inplace=True)
except ValueError:
pass
s1_2 = data_2[data_2['Outcome'] <= 1].groupby(['x']).agg(len)['Problem Name']
s2_2 = data_2[data_2['Outcome'] == 1].groupby(['x']).agg(len)['Problem Name']
s1_2[8] = s1_2.loc[8:].sum()
for i in range(9, int(s1_2.index.max()+1)):
try:
s1_2.drop(i, inplace=True)
except ValueError:
pass
s2_2[8] = s2_2.loc[8:].sum()
for i in range(9, int(s2_2.index.max()+1)):
try:
s2_2.drop(i, inplace=True)
except ValueError:
pass
s1_3 = data_3[data_3['Outcome'] <= 1].groupby(['x']).agg(len)['Problem Name']
s2_3 = data_3[data_3['Outcome'] == 1].groupby(['x']).agg(len)['Problem Name']
s1_3[8] = s1_3.loc[8:].sum()
for i in range(9, int(s1_3.index.max()+1)):
try:
s1_3.drop(i, inplace=True)
except ValueError:
pass
s2_3[8] = s2_3.loc[8:].sum()
for i in range(9, int(s2_3.index.max()+1)):
try:
s2_3.drop(i, inplace=True)
except ValueError:
pass
s1_4 = data_4[data_4['Outcome'] <= 1].groupby(['x']).agg(len)['Problem Name']
s2_4 = data_4[data_4['Outcome'] == 1].groupby(['x']).agg(len)['Problem Name']
s1_4[8] = s1_4.loc[8:].sum()
for i in range(9, int(s1_4.index.max()+1)):
try:
s1_4.drop(i, inplace=True)
except ValueError:
pass
s2_4[8] = s2_4.loc[8:].sum()
for i in range(9, int(s2_4.index.max()+1)):
try:
s2_4.drop(i, inplace=True)
except ValueError:
pass
s1_5 = data_5[data_5['Outcome'] <= 1].groupby(['x']).agg(len)['Problem Name']
s2_5 = data_5[data_5['Outcome'] == 1].groupby(['x']).agg(len)['Problem Name']
s1_5[8] = s1_5.loc[8:].sum()
for i in range(9, int(s1_5.index.max()+1)):
try:
s1_5.drop(i, inplace=True)
except ValueError:
pass
s2_5[8] = s2_5.loc[8:].sum()
for i in range(9, int(s2_5.index.max()+1)):
try:
s2_5.drop(i, inplace=True)
except ValueError:
pass
s1_6 = data_6[data_6['Outcome'] <= 1].groupby(['x']).agg(len)['Problem Name']
s2_6 = data_6[data_6['Outcome'] == 1].groupby(['x']).agg(len)['Problem Name']
s1_6[8] = s1_6.loc[8:].sum()
for i in range(9, int(s1_6.index.max()+1)):
try:
s1_6.drop(i, inplace=True)
except ValueError:
pass
s2_6[8] = s2_6.loc[8:].sum()
for i in range(9, int(s2_6.index.max()+1)):
try:
s2_6.drop(i, inplace=True)
except ValueError:
pass
In [37]:
fig, ax1 = plt.subplots()
fig_size = plt.rcParams["figure.figsize"]
fig_size[0] = 8.3
fig_size[1] = 4.7
plt.rcParams["figure.figsize"] = fig_size
plt.xlim(0.5,8.5)
plt.bar(s1.index, s1, width=0.9)
#plt.bar(s2.index, s2, width=0.9)
#plt.legend(['CORRECT', 'INCORRECT'])
plt.xlabel("Attempt number", size=14)
plt.ylabel("Number of attempts", size=14)
ax1.tick_params(axis ='both', which='major', length=0, labelsize =14, color='black')
ax1.tick_params(axis ='both', which='minor', length=0)
labels = [item.get_text() for item in ax1.get_xticklabels()]
labels = ['0', '1', '2', '3', '4', '5', '6', '7', '8+']
#labels = ['2', '4', '6', '8+']
#print(labels)
ax2 = ax1.twinx()
ax2.plot(s1.index, s2/s1, 'r-o', linewidth=4, label='Average')
ax2.plot(s1_1.index, s2_1/s1_1, 'c-+', label='group 1')
ax2.plot(s1_2.index, s2_2/s1_2, 'b-+', label='group 2')
ax2.plot(s1_3.index, s2_3/s1_3, 'c-.', label='group 3')
ax2.plot(s1_4.index, s2_4/s1_4, 'b-.', label='group 4')
ax2.plot(s1_5.index, s2_5/s1_5, 'c-x', label='group 5')
ax2.plot(s1_6.index, s2_6/s1_6, 'b-x', label='group 6')
ax2.legend()
ax2.set_ylabel('Fraction of incorrect attempts', size=14, color='r')
ax2.tick_params('y', colors='r')
ax2.tick_params(axis ='both', which='minor', length=0)
ax2.tick_params(axis ='both', which='major', length=0, labelsize =14, color='red')
ax1.set_xticklabels(labels)
plt.show()
In [ ]: