In [178]:
import pickle
from sklearn.linear_model import SGDRegressor
from _collections import defaultdict
import time
import timeit
from numpy.linalg import norm
import scipy.optimize
import random
from sets import Set
import numpy as np
In [179]:
def parseData(fname):
for l in open(fname):
yield eval(l)
def parseTxt(fname):
for l in open(fname):
yield l.strip().split(" ")
print "Reading train..."
train = list(parseData("/home/iizhaki/oasis/CSE255/Project2/assignment2/train.json"))
N = len(train)
#test = train[int(-0.7 * N) : ]
#train = train[: int(0.7 * N)]
In [179]:
In [179]:
In [180]:
print "done"
def parseTxt(fname):
for l in open(fname):
yield l.strip().split(" ")
train2 = list(parseTxt("/home/iizhaki/oasis/CSE255/Project2/assignment2/ref.txt"))
allXs = []
allYs = []
for l in train:
user, item, rating = l['reviewerID'], l['itemID'], l['rating']
allXs.append([user, item])
allYs.append(float(rating))
'''for l in train2[1:]:
user, rest = l[0].split("-")
item, rating = rest.split(",")
#user, item, rating = l['reviewerID'], l['itemID'], l['rating']
allXs.append([user, item])
allYs.append(float(rating))
'''
print "done"
allXt = []
allYt = []
for l in test:
user, item, rating = l['reviewerID'], l['itemID'], l['rating']
allXt.append([user, item])
allYt.append(float(rating))
In [180]:
In [180]:
In [181]:
lambd = 1.0
alpha = 0
X = allXs
y = allYs
Ntrain = len(y)
data = zip(X, y)
numU = 0
numI = 0
allUs = {}
allUrs = {}
allIs = {}
allIrs = {}
fastData = []
for [u, i], Rui in data:
if u not in allUs:
allUs[u] = numU
allUrs[numU] = u
numU += 1
if i not in allIs:
allIs[i] = numI
allIrs[numI] = i
numI += 1
fastData.append((allUs[u], allIs[i], Rui))
numUt = 0
numIt = 0
allUt = {}
allUrt = {}
allIt = {}
allIrt = {}
datat = zip(allXt, allYt)
fastTest = []
for [u, i], Rui in datat:
if u not in allUt:
allUt[u] = numUt
allUrt[numUt] = u
numUt += 1
if i not in allIt:
allIt[i] = numIt
allIrt[numIt] = i
numIt += 1
fastTest.append((allUt[u], allIt[i], Rui))
Iu = np.zeros(numU)
Ii = np.zeros(numI)
uToRI = [[]] * numU
iToRU = [[]] * numI
for (u, i, Rui) in fastData:
Iu[u] += 1
Ii[i] += 1
if uToRI[u] == []:
uToRI[u] = [(i, Rui)]
else:
uToRI[u].append((i, Rui))
if iToRU[i] == []:
iToRU[i] = [(u, Rui)]
else:
iToRU[i].append((u, Rui))
In [182]:
print "Done"
In [183]:
# Objective
def func(theta, X, y, lam):
diff = numpy.dot(X,theta) - y
diffSq = numpy.dot(diff,diff)
diffSqReg = diffSq + lam * numpy.dot(theta,theta)
#print "offset =", diffSqReg
return diffSqReg
# Derivative
def fprime(theta, X, y, lam):
diff = numpy.dot(X,theta) - y
res = 2 * numpy.dot(X.T,diff) + 2 * lam * theta
#print "gradient =", res
return res
In [184]:
def miniFunc(Data, Alpha, BetaU, BetaI, GammaU, GammaI, Lambd):
part1 = 0
for (u, i, Rui) in Data:
part1 += ((Alpha + BetaU[u] + BetaI[i] + np.dot(GammaU[u], GammaI[i]) - Rui) ** 2)
if Lambd:
part2 = sum(BetaU * BetaU) + sum(BetaI * BetaI)
part3 = sum(GammaU * GammaU) + sum(GammaI * GammaI)
else:
part2 = 0
part3 = 0
print part1, part2, part3
return part1 + Lambd * (part2 + part3)
In [185]:
oldVal = 0
betaU = np.zeros(numU)
betaI = np.zeros(numI)
gammaU = np.zeros((numU, 3))
gammaI = np.zeros((numI, 3))
for u in range(numU):
gammaU[u] = np.array([0.] * 3)
for i in range(numI):
gammaI[i] = np.array([1.] * 3)
In [ ]:
alpha = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_a_BEST500.pck", "rb"))
betaU = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_bu_BEST500.pck", "rb"))
betaI = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_bi_BEST500.pck", "rb"))
gammaU = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_gu_BEST500.pck", "rb"))
gammaI = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_gi_BEST500.pck", "rb"))
oldVal = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_val_BEST500.pck", "rb"))
In [142]:
#alpha = 3.70361669988
#betaUD = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/Bu.pck", "r"))
#betaID = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/Bi.pck", "rb"))
#gammaUD = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/Yu.pck", "rb"))
#gammaID = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/Yi.pck", "rb"))
#oldVal = 0
In [11]:
for u in betaUD:
betaU[allUs[u]] = betaUD[u]
gammaU[allUs[u]] = gammaUD[u]
for i in betaID:
betaI[allIs[i]] = betaID[i]
gammaI[allIs[i]] = gammaID[i]
print betaU[0]
print gammaU[0]
print betaI[0]
print gammaI[0]
In [186]:
betaU = np.array(betaU)
betaI = np.array(betaI)
gammaU = list(gammaU)
gammaI = list(gammaI)
for u in range(numU):
gammaU[u] = np.append(gammaU[u], [0] * 1)
for i in range(numI):
gammaI[i] = np.append(gammaI[i], [1] * 1)
gammaU = np.array(gammaU)
gammaI = np.array(gammaI)
oldVal = 0
In [ ]:
def checkTest(Data, alpha, betaU, betaI, gammaU, gammaI):
error = miniFunc(Data, alpha, betaU, betaI, gammaU, gammaI, 0)
return math.sqrt(error)
In [103]:
# Adjust to less features
betaU = betaU[:471404]
betaI = betaI[:102549]
In [ ]:
# HERE I RUN WITH 50
oldVal = 0
error = 50000000
fstPlus = 0
lambd = 1
lambd2 = 1
while True:
lastAlpha = alpha
lastBetaU = betaU.copy()
lastBetaI = betaI.copy()
lastGammaU = gammaU.copy()
lastGammaI = gammaI.copy()
#----------------------
start = time.time()
#----------------------
# Alpha stage
alpha = 0
for (u, i, Rui) in fastData:
gu = gammaU[u]
gi = gammaI[i]
alpha += Rui - (betaU[u] + betaI[i] + np.dot(gu, gi))
alpha = alpha / Ntrain
#----------------------
#end = time.time()
#finished = end - start
#print "Alpha time: ", finished
#----------------------
#----------------------
#start = time.time()
#----------------------
# BetaU stage
betaU.fill(0)
for (u, i, Rui) in fastData:
gu = gammaU[u]
gi = gammaI[i]
betaU[u] += (Rui - (alpha + betaI[i] + np.dot(gu, gi)))
betaU = betaU / (lambd + Iu)
#----------------------
#end = time.time()
#finished = end - start
#print "BetaU time: ", finished
#----------------------
#----------------------
#start = time.time()
#----------------------
# BetaI stage
betaI.fill(0)
for (u, i, Rui) in fastData:
gu = gammaU[u]
gi = gammaI[i]
betaI[i] += (Rui - (alpha + betaU[u] + np.dot(gu, gi)))
betaI = betaI / (lambd + Ii)
#----------------------
#end = time.time()
#finished = end - start
#print "BetaI time: ", finished
#----------------------
#----------------------
#start = time.time()
#----------------------
# GammaU stage
for u in range(numU):
gi = []
y = []
for (i, Rui) in uToRI[u]:
gi.append(gammaI[i])
y.append(Rui - (alpha + betaU[u] + betaI[i]))
#clf = SGDRegressor(n_iter = 5, alpha = 0.1)
#clf.fit(gi, y)
#thetas = clf.coef_
#thetas, _, _, _ = np.linalg.lstsq(gi, y)
thetas, _, _ = scipy.optimize.fmin_l_bfgs_b(func, np.array(gammaU[u]).T, fprime, args = (np.array(gi), np.array(y).T, lambd2))
#thetas, _, _ = scipy.optimize.fmin_cg(func, np.array(gammaU[u]).T, fprime, args = (np.array(gi), np.array(y).T, lambd))
gammaU[u] = thetas
#----------------------
#end = time.time()
#finished = end - start
#print "GammaU time: ", finished
#----------------------
#----------------------
#start = time.time()
#----------------------
# GammaI stage
for i in range(numI):
gu = []
y = []
for (u, Rui) in iToRU[i]:
gu.append(gammaU[u])
y.append(Rui - (alpha + betaU[u] + betaI[i]))
#clf = SGDRegressor(n_iter = 5, alpha = 0.1)
#clf.fit(gu, y)
#thetas = clf.coef_
#thetas, _, _, _ = np.linalg.lstsq(gu, y)
thetas, _, _ = scipy.optimize.fmin_l_bfgs_b(func, np.array(gammaI[i]).T, fprime, args = (np.array(gu), np.array(y).T, lambd))
#thetas, _, _ = scipy.optimize.fmin_cg(func, np.array(gammaI[i]).T, fprime, args = (np.array(gu), np.array(y).T, lambd))
gammaI[i] = thetas
#----------------------
#end = time.time()
#finished = end - start
#print "GammaI time: ", finished
#----------------------
#----------------------
#start = time.time()
#----------------------
newVal = miniFunc(fastData, alpha, betaU, betaI, gammaU, gammaI, lambd)
#----------------------
end = time.time()
finished = end - start
print "miniFunc time: ", finished, " --> Diff: ", (oldVal - newVal), newVal
#----------------------
#err = checkTest(fastData, alpha, betaU, betaI, gammaU, gammaI)
'''if newVal <= oldVal:
oldVal = newVal
else:
alpha = lastAlpha
betaU = lastBetaU
betaI = lastBetaI
gammaU = lastGammaU
gammaI = lastGammaI
if fstPlus == 0:
fstPlus += 1
lambd -= 0.1
if lambd < 0:
lambd = 0
elif fstPlus == 1:
fstPlus += 1
lambd += 0.1
if lambd > 2:
lambd = 2
else:
break
'''
if oldVal > 0 and oldVal <= newVal:
alpha = lastAlpha
betaU = lastBetaU
betaI = lastBetaI
gammaU = lastGammaU
gammaI = lastGammaI
break
fstPlus = 0
oldVal = newVal
print alpha
In [160]:
print miniFunc(fastData, alpha, betaU, betaI, gammaU, gammaI, lambd)
print len(gammaU[0])
In [42]:
pickle.dump(alpha, open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_a_BEST50.pck", "wb"))
pickle.dump(betaU, open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_bu_BEST50.pck", "wb"))
pickle.dump(betaI, open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_bi_BEST50.pck", "wb"))
pickle.dump(gammaU, open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_gu_BEST50.pck", "wb"))
pickle.dump(gammaI, open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_gi_BEST50.pck", "wb"))
pickle.dump(oldVal, open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_val_BEST50.pck", "wb"))
In [161]:
print "Done"
In [189]:
print (oldVal - newVal), newVal, err
In [109]:
In [191]:
testRest = np.array(list(parseTxt("/home/iizhaki/oasis/CSE255/Project2/assignment2/pairs_Rating.txt")))
myPredictions = open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_" + str(lambd) + "_" + str(alpha) + "_" + str(oldVal) + "_BEST50_AVG.txt", 'w')
myPredictions.write(str(testRest[0][0]) + '\n')
avgBu = np.mean(betaU)
avgBi = np.mean(betaI)
avgGu = np.mean(gammaU, axis = 0)
avgGi = np.mean(gammaI, axis = 0)
mse = 0
for currLine in testRest[1:]:
u, i = currLine[0].split("-")
if u in allUs:
bu = betaU[allUs[u]]
gu = gammaU[allUs[u]]
else:
bu = avgBu
gu = avgGu
if i in allIs:
bi = betaI[allIs[i]]
gi = gammaI[allIs[i]]
else:
bi = avgBi
gi = avgGi
p = alpha + bu + bi + np.dot(gu, gi)
if p > 5.0:
p = 4.8
if p < 1.0:
p = 1.2
myPredictions.write(u + '-' + i + ',' + str(p) + '\n')
myPredictions.flush()
myPredictions.close()
In [ ]:
In [ ]:
In [47]:
In [59]:
In [60]:
In [ ]: