In [31]:

    

import pickle
from sklearn.linear_model import SGDRegressor
from _collections import defaultdict
import time
import timeit

from numpy.linalg import norm
import scipy.optimize
import random
from sets import Set

import numpy as np


    

In [2]:

    

def parseData(fname):
  for l in open(fname):
    yield eval(l)
    
def parseTxt(fname):
  for l in open(fname):
    yield l.strip().split(" ")

print "Reading train..."
train = list(parseData("/home/iizhaki/oasis/CSE255/Project2/assignment2/train.json"))


    

    




Reading train...

In [2]:

    




    

In [2]:

    




    

In [3]:

    

print "done"

allXs = []
allYs = []
for l in train:
  user, item, rating = l['reviewerID'], l['itemID'], l['rating']
  allXs.append([user, item])
  allYs.append(float(rating))


    

    




done

In [3]:

    




    

In [3]:

    




    

In [4]:

    

lambd = 1.0
alpha = 0
X = allXs
y = allYs
Ntrain = len(y)
data = zip(X, y)

numU = 0
numI = 0
allUs = {}
allUrs = {}
allIs = {}
allIrs = {}
fastData = []
for [u, i], Rui in data:
    if u not in allUs:
        allUs[u] = numU
        allUrs[numU] = u
        numU += 1
    if i not in allIs:
        allIs[i] = numI
        allIrs[numI] = i
        numI += 1
    fastData.append((allUs[u], allIs[i], Rui))


Iu = np.zeros(numU)
Ii = np.zeros(numI)
uToRI = [[]] * numU
iToRU = [[]] * numI
for (u, i, Rui) in fastData:
    Iu[u] += 1
    Ii[i] += 1
    if uToRI[u] == []:
        uToRI[u] = [(i, Rui)]
    else:
        uToRI[u].append((i, Rui))
    if iToRU[i] == []:
        iToRU[i] = [(u, Rui)]
    else:
        iToRU[i].append((u, Rui))


    

In [5]:

    

print "Done"


    

    




Done

In [ ]:

    

# Objective
def func(theta, X, y, lam):
    diff = numpy.dot(X,theta) - y
    diffSq = numpy.dot(diff,diff) 
    diffSqReg = diffSq  + lam * numpy.dot(theta,theta)
    #print "offset =", diffSqReg
    return diffSqReg

# Derivative
def fprime(theta, X, y, lam):
    diff = numpy.dot(X,theta) - y
    res = 2 * numpy.dot(X.T,diff)    + 2 * lam * theta
    #print "gradient =", res
    return res


    

In [ ]:

    

def miniFunc(Data, Alpha, BetaU, BetaI, GammaU, GammaI, Lambd):
    part1 = 0
    for (u, i, Rui) in Data:
        part1 += ((Alpha + BetaU[u] + BetaI[i] + np.dot(GammaU[u], GammaI[i]) - Rui) ** 2)
    
    part2 = sum(BetaU * BetaU) + sum(BetaI * BetaI)
    part3 = sum(GammaU * GammaU) + sum(GammaI * GammaI)
    
    print part1, part2, part3
    return part1 + Lambd * (part2 + part3)


    

In [38]:

    

oldVal = 0
betaU = np.zeros(numU)
betaI = np.zeros(numI)
gammaU = [[]] * numU
gammaI = [[]] * numI
for u in range(numU):
    gammaU[u] = [0, 0, 0]
for i in range(numI):
    gammaI[i] = [5.0, 5.0, 5.0]


    

In [39]:

    

alpha = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_a_BEST3.pck", "rb"))
betaU = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_bu_BEST3.pck", "rb"))
betaI = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_bi_BEST3.pck", "rb"))
gammaU = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_gu_BEST3.pck", "rb"))
gammaI = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_gi_BEST3.pck", "rb"))
oldVal = pickle.load(open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_val_BEST3.pck", "rb"))


    

In [40]:

    

while True:
    lastAlpha = alpha
    lastBetaU = list(betaU)
    lastBetaI = list(betaI)
    lastGammaU = list(gammaU)
    lastGammaI = list(gammaI)
    
    #----------------------
    start = time.time()
    #----------------------
    
    # Alpha stage
    alpha = 0
    for (u, i, Rui) in fastData:
        gu = gammaU[u]
        gi = gammaI[i]
        alpha += Rui - (betaU[u] + betaI[i] + np.dot(gu, gi))
    alpha = alpha / Ntrain
    
    #----------------------
    #end = time.time()
    #finished = end - start
    #print "Alpha time: ", finished
    #----------------------
    
    #----------------------
    #start = time.time()
    #----------------------

    # BetaU stage 
    betaU.fill(0)
    for (u, i, Rui) in fastData:
        gu = gammaU[u]
        gi = gammaI[i]
        betaU[u] += (Rui - (alpha + betaI[i] + np.dot(gu, gi)))
    betaU = betaU / (lambd + Iu)
        
    #----------------------
    #end = time.time()
    #finished = end - start
    #print "BetaU time: ", finished
    #----------------------
        
    #----------------------
    #start = time.time()
    #----------------------
        
    # BetaI stage 
    betaI.fill(0)
    for (u, i, Rui) in fastData:
        gu = gammaU[u]
        gi = gammaI[i]
        betaI[i] += (Rui - (alpha + betaU[u] + np.dot(gu, gi)))
    betaI = betaI / (lambd + Ii)
        
    #----------------------
    #end = time.time()
    #finished = end - start
    #print "BetaI time: ", finished
    #----------------------
    
    #----------------------
    #start = time.time()
    #----------------------

    # GammaU stage 
    for u in range(numU):
        gi = []
        y = []
        for (i, Rui) in uToRI[u]:
            gi.append(gammaI[i])
            y.append(Rui - (alpha + betaU[u] + betaI[i]))
                    
        #clf = SGDRegressor(n_iter = 5, alpha = 0.1)
        #clf.fit(gi, y)
        #thetas = clf.coef_

        #thetas, _, _, _ = np.linalg.lstsq(gi, y)
        #thetas, _, _ = scipy.optimize.fmin_l_bfgs_b(func, np.array(gammaU[u]).T, fprime, args = (np.array(gi), np.array(y).T, lambd))
        thetas, _, _ = scipy.optimize.fmin_tnc(func, np.array(gammaU[u]).T, fprime, args = (np.array(gi), np.array(y).T, lambd))
        gammaU[u] = thetas
            
    #----------------------
    #end = time.time()
    #finished = end - start
    #print "GammaU time: ", finished
    #----------------------
        
    #----------------------
    #start = time.time()
    #----------------------
        
    # GammaI stage 
    for i in range(numI):
        gu = []
        y = []
        for (u, Rui) in iToRU[i]:
            gu.append(gammaU[u])
            y.append(Rui - (alpha + betaU[u] + betaI[i]))
            
        #clf = SGDRegressor(n_iter = 5, alpha = 0.1)
        #clf.fit(gu, y)
        #thetas = clf.coef_
            
        #thetas, _, _, _ = np.linalg.lstsq(gu, y)
        #thetas, _, _ = scipy.optimize.fmin_l_bfgs_b(func, np.array(gammaI[i]).T, fprime, args = (np.array(gu), np.array(y).T, lambd))
        thetas, _, _ = scipy.optimize.fmin_tnc(func, np.array(gammaI[i]).T, fprime, args = (np.array(gu), np.array(y).T, lambd))
        gammaI[i] = thetas
    
    #----------------------
    #end = time.time()
    #finished = end - start
    #print "GammaI time: ", finished
    #----------------------
    
    #----------------------
    #start = time.time()
    #----------------------
    newVal = miniFunc(fastData, alpha, betaU, betaI, np.array(gammaU), np.array(gammaI), lambd)
    #----------------------
    end = time.time()
    finished = end - start
    print "miniFunc time: ", finished,  " --> Diff: ", (oldVal - newVal), newVal
    #----------------------

    if oldVal > 0 and oldVal < newVal:
        alpha = lastAlpha
        betaU = lastBetaU
        betaI = lastBetaI
        gammaU = lastGammaU
        gammaI = lastGammaI
        break
        
    oldVal = newVal
    
print alpha


    

    




58737.675964 92109.8194207 228284.179523
miniFunc time:  262.268394947  --> Diff:  62.7674105238 379131.674908
58742.7480683 92659.2915681 227705.806147
miniFunc time:  274.529338837  --> Diff:  23.8291246124 379107.845783
58735.5814311 92985.2626411 227372.795073
miniFunc time:  335.463057041  --> Diff:  14.2066383667 379093.639145
58740.5324374 93175.3015668 227168.328418
miniFunc time:  268.743221998  --> Diff:  9.4767222057 379084.162423
58749.2902732 93292.3602736 227035.644053
miniFunc time:  309.002724171  --> Diff:  6.86782251729 379077.2946
58759.3059159 93367.8350682 226944.864703
miniFunc time:  253.324931145  --> Diff:  5.2889129867 379072.005687
58769.314265 93418.6682016 226879.750626
miniFunc time:  259.778446913  --> Diff:  4.27259449655 379067.733093
58778.702135 93454.2794939 226831.164781
miniFunc time:  251.097245932  --> Diff:  3.58668271877 379064.14641
58787.3024796 93480.0810669 226793.656721
miniFunc time:  254.635754108  --> Diff:  3.1061429301 379061.040267
58795.0615844 93499.3490259 226763.86775
miniFunc time:  244.758531094  --> Diff:  2.76190651575 379058.278361
58801.9782799 93514.1252067 226739.664591
miniFunc time:  283.604290962  --> Diff:  2.51028265845 379055.768078
58808.1609949 93525.7040606 226719.582049
miniFunc time:  239.867660999  --> Diff:  2.32097354066 379053.447104
58813.6638372 93534.9738492 226702.642126
miniFunc time:  246.009880066  --> Diff:  2.16729186103 379051.279812
58818.6155558 93542.5238693 226688.108171
miniFunc time:  237.46310401  --> Diff:  2.03221626848 379049.247596
58823.0671349 93548.7801791 226675.490455
miniFunc time:  244.491731882  --> Diff:  1.90982720244 379047.337769
58827.0910779 93554.0363607 226664.406922
miniFunc time:  234.416419983  --> Diff:  1.80340882641 379045.53436
58830.7506036 93558.5078263 226654.563159
miniFunc time:  243.785156012  --> Diff:  1.71277096472 379043.821589
58834.0644503 93562.362145 226645.757377
miniFunc time:  232.450376034  --> Diff:  1.63761707826 379042.183972
58837.0910973 93565.7103544 226637.807301
miniFunc time:  240.071408987  --> Diff:  1.57521925092 379040.608753
58839.8492849 93568.6594145 226630.576609
miniFunc time:  232.029131889  --> Diff:  1.52344432409 379039.085309
58842.3566012 93571.2629843 226623.985306
miniFunc time:  239.83244586  --> Diff:  1.48041704844 379037.604892
58844.6693016 93573.5831146 226617.907787
miniFunc time:  230.075997114  --> Diff:  1.44468790962 379036.160204
58846.7767494 93575.670862 226612.297165
miniFunc time:  238.716980934  --> Diff:  1.41542701685 379034.744777
58848.7063236 93577.54776 226607.099105
miniFunc time:  228.957467079  --> Diff:  1.39158814296 379033.353188
58850.4577093 93579.2418795 226602.281473
miniFunc time:  237.157579899  --> Diff:  1.37212692585 379031.981062
58852.0816869 93580.7737054 226597.767453
miniFunc time:  228.475773096  --> Diff:  1.35821589688 379030.622846
58853.5717246 93582.1850422 226593.517458
miniFunc time:  234.712717056  --> Diff:  1.34862132778 379029.274224
58854.8968008 93583.4853179 226589.547324
miniFunc time:  226.786530972  --> Diff:  1.34478175204 379027.929443
58856.0937897 93584.6752823 226585.812599
miniFunc time:  234.82873106  --> Diff:  1.34777130018 379026.581671
58857.1624304 93585.7706021 226582.291526
miniFunc time:  226.23832202  --> Diff:  1.35711294954 379025.224558
58858.0891389 93586.7936657 226578.968285
miniFunc time:  234.462721109  --> Diff:  1.37346856954 379023.85109
58858.8539774 93587.7538544 226575.850216
miniFunc time:  226.163551092  --> Diff:  1.39304208226 379022.458048
58859.4661836 93588.6356584 226572.942982
miniFunc time:  232.940192938  --> Diff:  1.41322356078 379021.044824
58859.998087 93589.4532778 226570.158147
miniFunc time:  224.858077049  --> Diff:  1.43531267263 379019.609511
58860.3721439 93590.2189183 226567.556344
miniFunc time:  231.666915178  --> Diff:  1.46210562222 379018.147406






    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-40-ddd0a15ebafa> in <module>()
     78         #thetas, _, _, _ = np.linalg.lstsq(gi, y)
     79         #thetas, _, _ = scipy.optimize.fmin_l_bfgs_b(func, np.array(gammaU[u]).T, fprime, args = (np.array(gi), np.array(y).T, lambd))
---> 80         thetas, _, _ = scipy.optimize.fmin_tnc(func, np.array(gammaU[u]).T, fprime, args = (np.array(gi), np.array(y).T, lambd))
     81         gammaU[u] = thetas
     82 

/oasis/scratch/iizhaki/temp_project/PV/python-virtualEnv3/lib/python2.7/site-packages/scipy/optimize/tnc.pyc in fmin_tnc(func, x0, fprime, args, approx_grad, bounds, epsilon, scale, offset, messages, maxCGit, maxfun, eta, stepmx, accuracy, fmin, ftol, xtol, pgtol, rescale, disp, callback)
    261             'disp': False}
    262 
--> 263     res = _minimize_tnc(fun, x0, args, jac, bounds, callback=callback, **opts)
    264 
    265     return res['x'], res['nfev'], res['status']

/oasis/scratch/iizhaki/temp_project/PV/python-virtualEnv3/lib/python2.7/site-packages/scipy/optimize/tnc.pyc in _minimize_tnc(fun, x0, args, jac, bounds, eps, scale, offset, mesg_num, maxCGit, maxiter, eta, stepmx, accuracy, minfev, ftol, xtol, gtol, rescale, disp, callback, **unknown_options)
    396                                         offset, messages, maxCGit, maxfun,
    397                                         eta, stepmx, accuracy, fmin, ftol,
--> 398                                         xtol, pgtol, rescale, callback)
    399 
    400     funv, jacv = func_and_grad(x)

/oasis/scratch/iizhaki/temp_project/PV/python-virtualEnv3/lib/python2.7/site-packages/scipy/optimize/tnc.pyc in func_and_grad(x)
    358     else:
    359         def func_and_grad(x):
--> 360             f = fun(x, *args)
    361             g = jac(x, *args)
    362             return f, g

<ipython-input-6-d3046b08a4dd> in func(theta, X, y, lam)
      2 def func(theta, X, y, lam):
      3     diff = numpy.dot(X,theta) - y
----> 4     diffSq = numpy.dot(diff,diff)
      5     diffSqReg = diffSq  + lam * numpy.dot(theta,theta)
      6     #print "offset =", diffSqReg

KeyboardInterrupt: 

In [32]:

    

pickle.dump(alpha, open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_a_LBFGS.pck", "wb"))
pickle.dump(betaU, open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_bu_LBFGS.pck", "wb"))
pickle.dump(betaI, open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_bi_LBFGS.pck", "wb"))
pickle.dump(gammaU, open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_gu_LBFGS.pck", "wb"))
pickle.dump(gammaI, open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_gi_LBFGS.pck", "wb"))
pickle.dump(oldVal, open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_val_LBFGS.pck", "wb"))


    

In [33]:

    

print "Done"


    

    




Done

In [34]:

    

print (oldVal - newVal), newVal


    

    




0.0 450797.985242

In [41]:

    

testRest = np.array(list(parseTxt("/home/iizhaki/oasis/CSE255/Project2/assignment2/pairs_Rating.txt")))
myPredictions = open("/home/iizhaki/oasis/CSE255/Project2/assignment2/idan_predictions_Rating_" + str(lambd) + "_" + str(alpha) + "_" + str(oldVal) + "_LBFGS.txt", 'w')
myPredictions.write(str(testRest[0][0]) + '\n')

mse = 0
for currLine in testRest[1:]:
    u, i = currLine[0].split("-")
    if u in allUs:
        bu = betaU[allUs[u]]
        gu = gammaU[allUs[u]]
    else:
        bu = 0
        gu = [0, 0, 0]
    if i in allIs:
        bi = betaI[allIs[i]]
        gi = gammaI[allIs[i]]
    else:
        bi = 0
        gi = [0, 0, 0]
    p = alpha + bu + bi + np.dot(gu, gi)
    if p > 5.0:
        p = 5.0
    if p < 1.0:
        p = 1.0
    myPredictions.write(u + '-' + i + ',' + str(p) + '\n')

myPredictions.flush()
myPredictions.close()


    

In [ ]:

    




    

In [ ]:

    




    

In [18]:

    




    

    




[[ 1  1  1]
 [ 4  4  4]
 [ 9  9  9]
 [16 16 16]]

In [ ]: