In [ ]:
from __future__ import division
import random
import math
import copy
import os
import pickle
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns
sns.set_style('darkgrid')
from lentil import datatools
from lentil import datasynth
from lentil import evaluate
from lentil import models
from lentil import est
%matplotlib inline
In [ ]:
import logging
logging.getLogger().setLevel(logging.DEBUG)
Generate a synthetic 1PL/2PL IRT model and sample an interaction history from it
In [ ]:
num_students = 2000
num_assessments = 3000
num_ixns_per_student = 1000
USING_2PL = False # False => using 1PL
In [ ]:
proficiencies = np.random.normal(0, 1, num_students)
difficulties = np.random.normal(0, 1, num_assessments)
if USING_2PL:
discriminabilities = np.random.normal(0, 1, num_assessments)
else:
discriminabilities = np.ones(num_assessments)
student_ids = ['S'+str(x) for x in xrange(num_students)]
assessment_ids = ['A'+str(x) for x in xrange(num_assessments)]
In [ ]:
ixns = [None] * (num_students * num_ixns_per_student)
assessment_idxes = range(num_assessments)
for student_idx, student_id in enumerate(student_ids):
for t in xrange(num_ixns_per_student):
module_idx = random.choice(assessment_idxes)
pass_likelihood = 1 / (1 + math.exp(-(discriminabilities[module_idx]*proficiencies[student_idx] + difficulties[module_idx])))
ixns[student_idx * num_ixns_per_student + t] = {
'student_id' : student_id,
'module_id' : assessment_ids[module_idx],
'module_type' : datatools.AssessmentInteraction.MODULETYPE,
'outcome' : np.random.random() < pass_likelihood,
'timestep' : t+1
}
history = datatools.InteractionHistory(pd.DataFrame(ixns))
history.idx_of_student_id = lambda x: int(x[1:])
history.idx_of_assessment_id = lambda x: int(x[1:])
In [ ]:
mirt_model = models.MIRTModel(history, dims=1, using_assessment_factors=USING_2PL)
estimator = est.MIRTMAPEstimator(
regularization_constant=1e-3,
ftol=1e-5,
debug_mode_on=True)
mirt_model.fit(estimator)
In [ ]:
onepl_model = models.OneParameterLogisticModel(
history.data, select_regularization_constant=True)
onepl_model.fit()
In [ ]:
twopl_model = models.TwoParameterLogisticModel(
history.data, select_regularization_constant=True)
twopl_model.fit()
In [ ]:
student_idxes = [int(k[1:]) for k in history.data['student_id'].unique()]
assessment_idxes = [int(k[1:]) for k in history.data['module_id'].unique()]
Verify that models.OneParameterLogisticModel can recover parameters. We would only expect this to be possible when USING_2PL = False.
In [ ]:
plt.xlabel('True difficulties')
plt.ylabel('Estimated difficulties')
plt.scatter(difficulties[assessment_idxes], onepl_model.model.coef_[0, num_students:])
plt.show()
plt.xlabel('Estimated difficulty - true difficulty')
plt.ylabel('Frequency (number of assessments)')
plt.hist(onepl_model.model.coef_[0, num_students:] - difficulties[assessment_idxes], bins=20)
plt.show()
In [ ]:
plt.xlabel('True proficiencies')
plt.ylabel('Estimated proficiencies')
plt.scatter(proficiencies[student_idxes], onepl_model.model.coef_[0, :num_students])
plt.show()
plt.xlabel('Estimated proficiency - true proficiency')
plt.ylabel('Frequency (number of students)')
plt.hist(onepl_model.model.coef_[0, :num_students] - proficiencies[student_idxes], bins=20)
plt.show()
Verify that models.TwoParameterLogisticModel can recover parameters. We would only expect this to be possible when USING_2PL = True.
In [ ]:
plt.xlabel('True difficulties')
plt.ylabel('Estimated difficulties')
plt.scatter(difficulties[assessment_idxes], twopl_model.model.coef_[0, (num_students*num_assessments):])
plt.show()
plt.xlabel('Estimated difficulty - true difficulty')
plt.ylabel('Frequency (number of assessments)')
plt.hist(twopl_model.model.coef_[0, (num_students*num_assessments):] - difficulties[assessment_idxes], bins=20)
plt.show()
In [ ]:
est_params = twopl_model.model.coef_[0, :(num_students*num_assessments)]
true_params = discriminabilities[:, None].dot(proficiencies[:, None].T).ravel()
plt.xlabel('True proficiency*discriminability')
plt.ylabel('Estimated proficiency*discriminability')
plt.scatter(true_params, est_params)
plt.show()
plt.xlabel('Estimated proficiency*discriminability - true proficiency*discriminability')
plt.ylabel('Frequency (number of student-assessment pairs)')
plt.hist(est_params - true_params, bins=20)
plt.show()
Verify that models.MIRTModel can recover parameters
In [ ]:
plt.xlabel('True difficulties')
plt.ylabel('Estimated difficulties')
plt.scatter(difficulties, mirt_model.assessment_offsets)
plt.show()
plt.xlabel('Estimated difficulty - true difficulty')
plt.ylabel('Frequency (number of assessments)')
plt.hist(mirt_model.assessment_offsets - difficulties, bins=20)
plt.show()
In [ ]:
plt.xlabel('True proficiencies')
plt.ylabel('Estimated proficiencies')
plt.scatter(proficiencies, mirt_model.student_factors[:, 0])
plt.show()
plt.xlabel('Estimated proficiency - true proficiency')
plt.ylabel('Frequency (number of students)')
plt.hist(mirt_model.student_factors[:, 0] - proficiencies, bins=20)
plt.show()
In [ ]:
plt.xlabel('True discriminabilities')
plt.ylabel('Estimated discriminabilities')
plt.scatter(discriminabilities, mirt_model.assessment_factors[:, 0])
plt.show()
plt.xlabel('Estimated discriminability - true discriminability')
plt.ylabel('Frequency (number of assessments)')
plt.hist(mirt_model.assessment_factors[:, 0] - discriminabilities, bins=20)
plt.show()
Verify that all models achieve similar training AUCs
In [ ]:
# models.OneParameterLogisticModel
evaluate.training_auc(onepl_model, history, plot_roc_curve=True)
In [ ]:
# models.TwoParameterLogisticModel
evaluate.training_auc(twopl_model, history, plot_roc_curve=True)
In [ ]:
# models.MIRTModel
evaluate.training_auc(mirt_model, history, plot_roc_curve=True)
In [ ]:
# true model
true_model = copy.deepcopy(mirt_model)
true_model.student_factors[:, 0] = proficiencies
true_model.assessment_factors[:, 0] = discriminabilities
true_model.assessment_offsets = difficulties
evaluate.training_auc(true_model, history, plot_roc_curve=True)
Construct a synthetic embedding
In [ ]:
num_students = 10000
num_assessment_interactions_per_step = 100
grid_size = 5
embedding_dimension = 2
num_assessments = grid_size ** 2
num_lessons = 2 * grid_size * (grid_size - 1)
num_lesson_interactions_per_student = 2 * (grid_size - 1) + 2
In [ ]:
S = np.zeros((num_students, embedding_dimension, num_lesson_interactions_per_student))
A = np.zeros((num_assessments, embedding_dimension))
L = np.zeros((num_lessons, embedding_dimension))
Q = np.zeros((num_lessons, embedding_dimension))
lesson_idx_of_loc = {}
assessment_idx_of_loc = {}
cell_size = 10 / (grid_size - 1)
lesson_count = 0
for i in xrange(grid_size):
for j in xrange(grid_size):
A[grid_size * i + j, :] = [i, j]
assessment_idx_of_loc[(i, j)] = grid_size * i + j
if j < grid_size - 1:
Q[lesson_count, :] = [i, j]
L[lesson_count, :] = [0, 1]
lesson_idx_of_loc[(i, j, 0, 1)] = lesson_count
lesson_count += 1
if i < grid_size - 1:
Q[lesson_count, :] = [i, j]
L[lesson_count, :] = [1, 0]
lesson_idx_of_loc[(i, j, 1, 0)] = lesson_count
lesson_count += 1
A *= cell_size
Q *= cell_size
L *= cell_size
A = np.maximum(1e-3, A)
Q = np.maximum(1e-3, Q)
Sample interactions from the synthetic embedding
In [ ]:
id_of_loc = lambda x: '-'.join(str(z) for z in x)
In [ ]:
data = []
for student_idx in xrange(num_students):
student_id = 'S' + str(student_idx)
steps = ([(0, 1)] * (grid_size - 1)) + ([(1, 0)] * (grid_size - 1))
random.shuffle(steps)
x, y = 0, 0
t = 1
assessment_idx = assessment_idx_of_loc[(0, 0)]
assessment_id = id_of_loc(assessment_loc_of_idx[assessment_idx])
pass_likelihood = 1 / (1 + math.exp(-(np.dot(S[student_idx, :, t], A[assessment_idx, :]) / np.linalg.norm(A[assessment_idx, :]) - np.linalg.norm(A[assessment_idx, :]))))
outcome = random.random() < pass_likelihood
data.append({
'student_id' : student_id,
'module_id' : assessment_id,
'module_type' : datatools.AssessmentInteraction.MODULETYPE,
'timestep' : t,
'outcome' : outcome})
for i, j in steps:
lesson_idx = lesson_idx_of_loc[(x, y, i, j)]
lesson_id = id_of_loc(lesson_loc_of_idx[lesson_idx])
data.append({
'student_id' : student_id,
'module_id' : lesson_id,
'module_type' : datatools.LessonInteraction.MODULETYPE,
'timestep' : t,
'outcome' : None})
x += i
y += j
# DEBUG
S[student_idx, :, t+1] = S[student_idx, :, t] + L[lesson_idx, :]# / (1 + math.exp(-(np.dot(S[student_idx, :, t], Q[lesson_idx, :]) / np.linalg.norm(Q[lesson_idx, :]) - np.linalg.norm(Q[lesson_idx, :]))))
t += 1
for _ in xrange(num_assessment_interactions_per_step):
assessment_idx = random.randint(0, num_assessments - 1)
assessment_id = id_of_loc(assessment_loc_of_idx[assessment_idx])
pass_likelihood = 1 / (1 + math.exp(-(np.dot(S[student_idx, :, t], A[assessment_idx, :]) / np.linalg.norm(A[assessment_idx, :]) - np.linalg.norm(A[assessment_idx, :]))))
outcome = random.random() < pass_likelihood
# BEGIN DEBUG
if assessment_idx_of_loc[(0, 0)] == assessment_idx:
outcome = random.random() < 0.1
# END DEBUG
data.append({
'student_id' : student_id,
'module_id' : assessment_id,
'module_type' : datatools.AssessmentInteraction.MODULETYPE,
'timestep' : t,
'outcome' : outcome})
In [ ]:
history = datatools.InteractionHistory(pd.DataFrame(data))
assessment_idx_map = {id_of_loc(loc): idx for idx, loc in assessment_loc_of_idx.iteritems()}
lesson_idx_map = {id_of_loc(loc): idx for idx, loc in lesson_loc_of_idx.iteritems()}
history.compute_idx_maps(assessment_idx=assessment_idx_map, lesson_idx=lesson_idx_map)
In [ ]:
len(history.data)
In [ ]:
history_path = os.path.join('data', 'lse_synthetic_history.pkl')
In [ ]:
with open(history_path, 'wb') as f:
pickle.dump(history, f, pickle.HIGHEST_PROTOCOL)
Estimate an embedding from the sampled interactions
In [ ]:
model = models.EmbeddingModel(
history, embedding_dimension=2,
using_lessons=True, using_prereqs=False, using_bias=True,
learning_update_variance_constant=0.5)
estimator = est.EmbeddingMAPEstimator(
regularization_constant=1e-3, using_scipy=True,
debug_mode_on=True, ftol=1e-4)
model.fit(estimator)
In [ ]:
model = models.OneParameterLogisticModel(history.data, select_regularization_constant=True)
model.fit()
In [ ]:
evaluate.training_auc(model, history, plot_roc_curve=True)
Visualize the estimated embedding vs. the true embedding
In [ ]:
plt.scatter(A[:, 0], A[:, 1])
for assessment_idx in xrange(num_assessments):
plt.annotate(id_of_assessment_idx(assessment_idx), (A[assessment_idx, 0], A[assessment_idx, 1]))
"""
for i in xrange(grid_size):
for j in xrange(grid_size):
if j < grid_size - 1:
assessment_idxes = [assessment_idx_of_loc[(i, j)], assessment_idx_of_loc[(i, j + 1)]]
plt.plot(A[assessment_idxes, 0], A[assessment_idxes, 1], c='black')
if i < grid_size - 1:
assessment_idxes = [assessment_idx_of_loc[(i, j)], assessment_idx_of_loc[(i + 1, j)]]
plt.plot(A[assessment_idxes, 0], A[assessment_idxes, 1], c='black')
"""
plt.show()
In [ ]:
plt.scatter(model.assessment_embeddings[:, 0], model.assessment_embeddings[:, 1])
for assessment_idx in xrange(num_assessments):
plt.annotate(id_of_assessment_idx(assessment_idx), (model.assessment_embeddings[assessment_idx, 0], model.assessment_embeddings[assessment_idx, 1]))
"""
for i in xrange(grid_size):
for j in xrange(grid_size):
if j < grid_size - 1:
assessment_idxes = [assessment_idx_of_loc[(i, j)], assessment_idx_of_loc[(i, j + 1)]]
plt.plot(model.assessment_embeddings[assessment_idxes, 0], model.assessment_embeddings[assessment_idxes, 1], c='black')
if i < grid_size - 1:
assessment_idxes = [assessment_idx_of_loc[(i, j)], assessment_idx_of_loc[(i + 1, j)]]
plt.plot(model.assessment_embeddings[assessment_idxes, 0], model.assessment_embeddings[assessment_idxes, 1], c='black')
"""
plt.show()
In [ ]:
plt.quiver(Q[:, 0], Q[:, 1], L[:, 0], L[:, 1], pivot='tail', color='black')
"""
for i in xrange(grid_size):
for j in xrange(grid_size):
if j < grid_size - 1:
lesson_idxes = [lesson_idx_of_loc[(i, j)], lesson_idx_of_loc[(i, j + 1)]]
plt.plot(Q[lesson_idxes, 0], Q[lesson_idxes, 1], c='black')
if i < grid_size - 1:
lesson_idxes = [lesson_idx_of_loc[(i, j)], lesson_idx_of_loc[(i + 1, j)]]
plt.plot(Q[lesson_idxes, 0], Q[lesson_idxes, 1], c='black')
"""
plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.xlim([-1, 11])
plt.ylim([-1, 11])
plt.show()
In [ ]:
plt.quiver(model.prereq_embeddings[:, 0], model.prereq_embeddings[:, 1], model.lesson_embeddings[:, 0], model.lesson_embeddings[:, 1], pivot='tail', color='black')
"""
for i in xrange(grid_size):
for j in xrange(grid_size):
if j < grid_size - 1:
lesson_idxes = [lesson_idx_of_loc[(i, j)], lesson_idx_of_loc[(i, j + 1)]]
plt.plot(model.prereq_embeddings[lesson_idxes, 0], model.prereq_embeddings[lesson_idxes, 1], c='black')
if i < grid_size - 1:
lesson_idxes = [lesson_idx_of_loc[(i, j)], lesson_idx_of_loc[(i + 1, j)]]
plt.plot(model.prereq_embeddings[lesson_idxes, 0], model.prereq_embeddings[lesson_idxes, 1], c='black')
"""
plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.xlim([-1, 11])
plt.ylim([-1, 11])
plt.show()
In [ ]:
right_lesson_idxes = [lesson_idx_of_loc[(i, j, 1, 0)] for i in xrange(grid_size) for j in xrange(grid_size) if (i, j, 1, 0) in lesson_idx_of_loc]
up_lesson_idxes = [lesson_idx_of_loc[(i, j, 0, 1)] for i in xrange(grid_size) for j in xrange(grid_size) if (i, j, 0, 1) in lesson_idx_of_loc]
In [ ]:
plt.quiver(0, 0, L[right_lesson_idxes, 0], L[right_lesson_idxes, 1], pivot='tail', color='red', alpha=0.25)
plt.quiver(0, 0, L[up_lesson_idxes, 0], L[up_lesson_idxes, 1], pivot='tail', color='blue', alpha=0.25)
plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.xlim([-1, 11])
plt.ylim([-1, 11])
plt.show()
In [ ]:
plt.quiver(0, 0, model.lesson_embeddings[right_lesson_idxes, 0], model.lesson_embeddings[right_lesson_idxes, 1], pivot='tail', color='red', alpha=0.25)
plt.quiver(0, 0, model.lesson_embeddings[up_lesson_idxes, 0], model.lesson_embeddings[up_lesson_idxes, 1], pivot='tail', color='blue', alpha=0.25)
plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.xlim([-1, 11])
plt.ylim([-1, 11])
plt.show()
In [ ]:
plt.scatter(L[right_lesson_idxes, 0], L[right_lesson_idxes, 1], color='red', label='1-0')
plt.scatter(L[up_lesson_idxes, 0], L[up_lesson_idxes, 1], color='blue', label='0-1')
plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.legend(loc='best')
plt.show()
In [ ]:
plt.scatter(model.lesson_embeddings[right_lesson_idxes, 0], model.lesson_embeddings[right_lesson_idxes, 1], color='red', label='1-0')
plt.scatter(model.lesson_embeddings[up_lesson_idxes, 0], model.lesson_embeddings[up_lesson_idxes, 1], color='blue', label='0-1')
plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.legend(loc='best')
plt.show()
In [ ]:
student_idxes = random.sample(range(num_students), 10)
In [ ]:
for student_idx in student_idxes:
plt.scatter(S[student_idx, 0, :], S[student_idx, 1, :], c='black')
for i in xrange(num_lesson_interactions_per_student):
plt.plot(S[student_idx, 0, i:(i+2)], S[student_idx, 1, i:(i+2)], c='black')
plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.title('student_id = %s' % history.id_of_student_idx(student_idx))
plt.show()
In [ ]:
for student_idx in student_idxes:
plt.scatter(model.student_embeddings[student_idx, 0, :], model.student_embeddings[student_idx, 1, :], c='black')
for i in xrange(num_lesson_interactions_per_student):
plt.plot(model.student_embeddings[student_idx, 0, i:(i+2)], model.student_embeddings[student_idx, 1, i:(i+2)], c='black')
plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.title('student_id = %s' % history.id_of_student_idx(student_idx))
plt.show()
In [ ]:
for student_idx in student_idxes:
for i in xrange(embedding_dimension):
plt.plot(S[student_idx, i, :], '-s', label='Skill 1')
plt.xlabel('Timestep')
plt.ylabel('Skill')
plt.title('student_id = %s' % history.id_of_student_idx(student_idx))
plt.legend(loc='best')
plt.show()
In [ ]:
for student_idx in student_idxes:
for i in xrange(embedding_dimension):
plt.plot(model.student_embeddings[student_idx, i, :], '-s', label='Skill 1')
plt.xlabel('Timestep')
plt.ylabel('Skill')
plt.title('student_id = %s' % history.id_of_student_idx(student_idx))
plt.legend(loc='best')
plt.show()
In [ ]: