In [1]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
In [2]:
# Import data
path = "../data/petdata_1000_100.csv"
raw_data = pd.read_csv(path, index_col="doc_uri")
assert raw_data.shape == (1000,100), "Import error, df has false shape"
In [3]:
# Convert df
data = raw_data.unstack().to_frame().reset_index()
data.columns = ["user", "doc_uri", "rating"]
# Missing value handling
data.fillna(0, inplace=True)
assert data.shape == (raw_data.shape[0] * raw_data.shape[1], 3), "Conversion error, df has false shape"
assert data.rating.max() <= 5., "Value error, max rating over upper bound"
assert data.rating.min() >= 0., "Value error, min rating under lower bound"
data.head()
Out[3]:
In [4]:
data.rating.describe().to_frame().T
Out[4]:
In [5]:
# Plot distribution of (random) ratings
plt.rcParams['figure.figsize'] = [9, 3]
plt.subplot(1, 2, 1)
hist = data.rating.plot(kind="hist", grid=True,
bins=[-0.1,0.1,0.9,1.1,1.9,2.1,2.9,3.1,3.9,4.1,4.9,5.1])
hist.set(xlabel= "rating")
plt.subplot(1, 2, 2)
box = data.rating.plot(kind="box")
plt.tight_layout()
plt.savefig("plots/ratings.png", orientation="landscape", dpi=120)
In [6]:
from surprise import SVD, Dataset, Reader, NMF, accuracy
from surprise.model_selection import cross_validate
from surprise.prediction_algorithms.random_pred import NormalPredictor
reader = Reader(rating_scale=(1, 5))
ds = Dataset.load_from_df(data[["user", "doc_uri", "rating"]], reader)
baseline_model = NormalPredictor() # Baseline model, predicts labels based on distribution of ratings
In [7]:
# Models - tune parameters, if you'd like ;)
svd = SVD() # Singular Value Decomposition
nmf = NMF() # Non-negative Matrix factorization
In [8]:
for algo in [baseline_model, svd, nmf]:
cross_validate(algo, ds, measures=["RMSE", "MAE"], cv=5, verbose=True)
In [9]:
from surprise.prediction_algorithms.knns import KNNBasic
sim_options = {"name": "pearson", # pearson's r
"user_based": False # item-based
}
knn = KNNBasic(sim_options=sim_options)
In [10]:
cross_validate(knn, ds, measures=["RMSE", "MAE"], cv=5, verbose=True)
Out[10]:
In [11]:
# Train final model
trainset = ds.build_full_trainset()
knn.fit(trainset)
# RMSE of final model
testset = trainset.build_testset()
test_pred = knn.test(testset)
accuracy.rmse(test_pred, verbose=True) # should be very bad ;)
Out[11]:
In [12]:
combinations_to_predict = [("Aaron Keith III", "http://gregory.com/"),
("Abigail Wong", "http://hicks.com/"),
("Julie Bullock", "https://www.garcia.com/"),
("Victoria Perez", "http://lee-phillips.org/register/")]
In [13]:
# Predictions
for combination in combinations_to_predict:
user = combination[0]
doc = combination[1]
pred = knn.predict(user, doc)
print(pred[0], "should rate", pred[1], "with", int(round(pred[3])), "stars")
In [ ]: