In [2]:
from _collections import defaultdict
import time
import timeit
from numpy.linalg import norm
import scipy.optimize
import random
import numpy as np
def parseData(fname):
for l in open(fname):
yield eval(l)
def parseTxt(fname):
for l in open(fname):
yield l.strip().split(" ")
print "Reading train..."
train = list(parseData("/home/iizhaki/oasis/CSE255/Project2/assignment2/train.json"))
print "Reading test..."
test = np.array(list(parseTxt("/home/iizhaki/oasis/CSE255/Project2/assignment2/pairs_Purchase.txt")))
print "done"
In [ ]:
trainBu = np.array(train, copy=True)
N = len(train)
myRange = range(N)
random.shuffle(myRange)
forTest = myRange[-int(0.7 * N) : ]
forTrain = myRange[: int(0.7 * N)]
In [13]:
test = [train[i] for i in forTest]
train = [train[i] for i in forTrain]
print "done"
In [15]:
from sets import Set
items = Set()
purchases = defaultdict(Set)
for l in train:
user, item = l['reviewerID'], l['itemID']
purchases[user].add(item)
items.add(item)
In [ ]:
def miniFunc(Alpha, BetaU, BetaI, Lambd):
part1 = 0
for u in purchases.keys():
purch = purchases[u]
for i in items:
if i in purch:
part1 += ((Alpha + BetaU[u] + BetaI[i] - 1.) ** 2)
else:
part1 += ((Alpha + BetaU[u] + BetaI[i]) ** 2)
part2 = 0
for u in BetaU:
part2 += (BetaU[u] ** 2)
for i in BetaI:
part2 += (BetaI[i] ** 2)
return part1 + Lambd * part2
oldVal = 0
alpha = 0
Ntrain = len(purchases)
bestMse = 0
rouni = 1
for lam in range(10, 11):
print "Round ", rouni
rouni += 1
betaU = defaultdict(int)
betaI = defaultdict(int)
lambd = lam / 10.0
while True:
lastAlpha = alpha
lastBetaU = betaU
lastBetaI = betaI
# Alpha stage
alpha = 0.
for u in purchases.keys():
purch = purchases[u]
for i in items:
bu = betaU[u]
bi = betaI[i]
if i in purch:
alpha += 1. - (bu + bi)
else:
alpha -= (bu + bi)
alpha = alpha / Ntrain
# BetaU stage
betaU = defaultdict(int)
for u in purchases.keys():
purch = purchases[u]
for i in items:
if i in purch:
betaU[u] += (1. - (alpha + betaI[i]))
else:
betaU[u] -= (alpha + betaI[i])
for u in betaU:
betaU[u] = betaU[u] / (lambd + len(purchases[u]))
# BetaI stage
Ii = defaultdict(int)
betaI = defaultdict(int)
for u in purchases.keys():
purch = purchases[u]
for i in items:
if i in purch:
betaI[i] += (1. - (alpha + betaU[u]))
Ii[i] += 1
else:
betaI[i] -= (alpha + betaU[u])
for i in betaI:
betaI[i] = betaI[i] / (lambd + Ii[i])
newVal = miniFunc(alpha, betaU, betaI, lambd)
if lastAlpha != 0 and oldVal < newVal:
alpha = lastAlpha
betaU = lastBetaU
betaI = lastBetaI
break
oldVal = newVal
mse = 0
for currLine in test[1:]:
u, rest = currLine[0].split("-")
i, Rui = rest.split(",")
p = alpha + betaU[u] + betaI[i]
mse += (p - float(Rui)) ** 2
mse = mse / len(test)
#print "Mse: ", mse
if mse >= bestMse:
storedAlpha = alpha
storedBetaU = betaU
storedBetaI = betaI
print "Improved MSE from ", bestMse, " to ",mse, " using lambda ", lambd
bestMse = mse
myPredictions = open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Purchase_" + str(lambd) + "_" + str(alpha) + ".txt", 'w')
myPredictions.write(str(testRest[0][0]) + '\n')
mse = 0
for currLine in testRest[1:]:
u, i = currLine[0].split("-")
p = alpha + betaU[u] + betaI[i]
if p >= 0.5:
p = "1"
else:
p = "0"
myPredictions.write(u + '-' + i + ',' + p + '\n')
myPredictions.flush()
myPredictions.close()
In [3]:
item = 'I102776733'
user = 'U566105319'
In [4]:
print "Alpha: ", alpha
print "BetaI of ", item, ": ", betaI[item]
print "BetaU of ", user, ": ", betaU[user]
In [20]:
In [30]:
print "Reading test..."
testRest = np.array(list(parseTxt("/home/iizhaki/oasis/CSE255/Project2/assignment2/pairs_Rating.txt")))
myPredictions = open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating.txt", 'w')
print "done"
In [31]:
myPredictions.write(str(testRest[0][0]) + '\n')
mse = 0
for currLine in testRest[1:]:
u, i = currLine[0].split("-")
p = alpha + betaU[u] + betaI[i]
myPredictions.write(u + '-' + i + ',' + str(p) + '\n')
myPredictions.flush()
myPredictions.close()
In [6]:
user1 = 'U229891973'
user2 = 'U622491081'
A = set([i for (u, i) in allXs if u == user1])
B = set([i for (u, i) in allXs if u == user2])
In [7]:
print len(A), len(B)
In [8]:
Jaccard1 = len(A.intersection(B)) * 1.0 / len((A.union(B)))
print Jaccard1
In [9]:
user3 = 'U622491081'
dictU = defaultdict(list)
for u, i in allXs:
dictU[u].append(i)
In [10]:
A = set(dictU[user3])
bestJac = 0
bestU = []
for u in dictU:
if u == user3:
continue
B = set(dictU[u])
jacc = len(A.intersection(B)) * 1.0 / len(A.union(B))
if jacc > bestJac:
bestU = [u]
bestJac = jacc
elif jacc == bestJac:
bestU.append(u)
print bestU, bestJac
In [10]:
In [11]:
user4 = 'U639726733'
dictI = defaultdict(list)
for u, i in allXs:
dictI[i].append(u)
In [12]:
items = dictU[user4]
for item in items:
A = set(dictI[item])
bestJac = 0
bestI = []
for i in dictI:
if i in items:
continue
B = set(dictI[i])
jacc = len(A.intersection(B)) * 1.0 / len(A.union(B))
if jacc > bestJac:
bestI = [i]
bestJac = jacc
elif jacc == bestJac:
#bestI += [i]
bestI = [i]
print item, bestI, bestJac
print "-------------"
In [12]:
In [ ]: