Movie ranking example

this example is a EclairJS (JavaScript) implementation of movie recommending

Create the spark context and globals



In [1]:

    
var SparkContext = require('eclairjs/SparkContext');
var sc = new SparkContext("local[*]", "myapp");

var Tuple = require('eclairjs/Tuple');
var List = require('eclairjs/List');
var ALS = require('eclairjs/mllib/recommendation/ALS');
var Rating = require('eclairjs/mllib/recommendation/Rating');

load the small rating dataset



In [2]:

    
var small_ratings_raw_data = sc.textFile('../data/mllib/ml-latest-small/ratings.csv');
var small_ratings_raw_data_header = small_ratings_raw_data.take(1)[0];
var small_ratings_data = small_ratings_raw_data.filter(function(line, small_ratings_raw_data_header) {
    // filters out the header
    return line != small_ratings_raw_data_header;
}, [small_ratings_raw_data_header])
.map(function(line) {
    return line.split(",");
})
.map(function(tokens, Rating) {
    // tokenes are userId,movieId,rating,timestamp
    return new Rating(tokens[0],tokens[1],tokens[2]);
}, [Rating]).cache()
JSON.stringify(small_ratings_data.take(3));









    Out[2]:





["{Rating: [1,16,4]}","{Rating: [1,24,1.5]}","{Rating: [1,32,4]}"]

load the small moving dataset



In [3]:

    
var small_movies_raw_data = sc.textFile('../data/mllib/ml-latest-small/movies.csv');
var small_movies_raw_data_header = small_movies_raw_data.take(1)[0];
var small_movies_data = small_movies_raw_data.filter(function(line, small_movies_raw_data_header) {
        // filters out the header
        return line != small_movies_raw_data_header;
    }, [small_movies_raw_data_header])

    .map(function(line, Tuple) {
        var fields = line.split(",");
        return new Tuple(parseInt(fields[0]), fields[1]);

    }, [Tuple]).cache();

var small_movies_titles = small_movies_data.mapToPair(
    function( tuple2, Tuple) { // Tuple2
        return new Tuple(tuple2[0], tuple2[1]);

    }, [Tuple]);
JSON.stringify(small_movies_data.take(3));









    Out[3]:





[{"0":1,"1":"Toy Story (1995)","length":2},{"0":2,"1":"Jumanji (1995)","length":2},{"0":3,"1":"Grumpier Old Men (1995)","length":2}]

Selecting ALS parameters using the small dataset

In order to determine the best ALS parameters, we will use the small dataset. We need first to split it into train, validation, and test datasets.



In [4]:

    
var seed = 0;
var split = small_ratings_data.randomSplit([0.6, 0.2, 0.2], seed)
var training_RDD = split[0];
var validation_RDD = split[1];
var test_RDD = split[2];

var  validation_for_predict_RDD = validation_RDD.map(function(rating, Tuple) {
        return new Tuple(rating.user(), rating.product());

}, [Tuple]);
JSON.stringify(validation_for_predict_RDD.take(3));









    Out[4]:





[{"0":1,"1":50,"length":2},{"0":1,"1":161,"length":2},{"0":1,"1":356,"length":2}]



In [5]:

    
var test_for_predict_RDD = test_RDD.map(function(rating, Tuple) {
        return new Tuple(rating.user(), rating.product());

}, [Tuple]);

JSON.stringify(test_for_predict_RDD.take(3));









    Out[5]:





[{"0":1,"1":16,"length":2},{"0":1,"1":47,"length":2},{"0":1,"1":110,"length":2}]

Proceed with the training phase.



In [6]:

    
seed = 5
var iterations = 10
var regularization_parameter = 0.1
var ranks = [4, 8, 12];
var errors = [0, 0, 0];
var err = 0
var tolerance = 0.02

var min_error = Number.POSITIVE_INFINITY
var best_rank = -1
var best_iteration = -1
var blocks = -1;
var lambda = regularization_parameter;

ranks.forEach(function(rank) {
    var model = ALS.train(training_RDD, rank, iterations, regularization_parameter, blocks, seed);
    var predictions = model.predict(validation_for_predict_RDD)
        .mapToPair(function(rating, Tuple) {
            return new Tuple(new Tuple(rating.user(), rating.product()), rating.rating());
        }, [Tuple]
    );

    var rates_and_preds = validation_RDD.mapToPair( function(rating, Tuple) {
            return new Tuple(new Tuple(rating.user(), rating.product()), rating.rating());

    }, [Tuple])
    .join(predictions);

    var t = rates_and_preds.mapToFloat(function(tuple) {
            // Rating object
            var x1 = tuple[0]; // Tuple2
            var x2 = tuple[1]; // Tuple2
            var a =  x2[0];
            var y =x2[0] - x2[1]; // ([1][0] - r[1][1])**2
            return Math.pow(y, 2);

    });
    var error = Math.sqrt(t.mean());
    errors[err] = error;
    err += 1;
    if (error < min_error) {
        min_error = error;
        best_rank = rank;
    }

});

"The best model was trained with rank " +best_rank;









    



[Stage 1457:==========================================>             (6 + 2) / 8]





    Out[6]:





The best model was trained with rank 4

To build our recommender model, we will use the complete dataset.



In [7]:

    
var complete_ratings_raw_data = 
    sc.textFile("../data/mllib/ml-latest-small/ratings.csv");
var complete_ratings_raw_data_header = complete_ratings_raw_data.take(1)[0];

var complete_ratings_data = complete_ratings_raw_data.filter(function (line, complete_ratings_raw_data_header) {
    return line != complete_ratings_raw_data_header;
}, [complete_ratings_raw_data_header])
.map(function( line, Rating) {
    var fields = line.split(",");
    var userId = parseInt(fields[0]);
    var movieId = parseInt(fields[1]);
    var rating = parseFloat(fields[2]);
    return new Rating(userId, movieId, rating);

}, [Rating])
.cache();

JSON.stringify("There are recommendations in the complete dataset:  " + complete_ratings_data.count());









    



[Stage 1459:>                                                       (0 + 2) / 2]





    Out[7]:





"There are recommendations in the complete dataset:  105339"

We test on our testing set.



In [8]:

    
var splits2 = complete_ratings_data.randomSplit([0.7, 0.3], 0);
training_RDD = splits2[0];
test_RDD = splits2[1];

var complete_model = ALS.train(training_RDD, best_rank, iterations, regularization_parameter, blocks, seed);

test_for_predict_RDD = test_RDD.map(function (rating, Tuple) {
        return new Tuple(rating.user(), rating.product());

}, [Tuple]);

var predictions = complete_model.predict(test_for_predict_RDD)
    .mapToPair(function( rating, Tuple) {
        return new Tuple(new Tuple(rating.user(), rating.product()), rating.rating());

    }, [Tuple]);

var rates_and_preds = test_RDD.mapToPair(function( rating, Tuple) {
        return new Tuple(new Tuple(rating.user(), rating.product()), rating.rating());

}, [Tuple])
.join(predictions);

var t = rates_and_preds.mapToFloat( function( x) {
        // Rating object
        /*System.out.println("x._1 " + x._1);
         System.out.println("x._2 " + x._2);*/
        var x1 = x[0];
        var x2 = x[1];
        var a =  x2[0];
        var y = x2[0] - x2[1]; // ([1][0] - r[1][1])**2
        //return new Tuple2(x.user(), x.product());
        return Math.pow(y, 2);


});
var error = Math.sqrt(t.mean());
JSON.stringify("For testing data the RMSE is " + error);









    



[Stage 1621:>                                                       (0 + 8) / 8]





    Out[8]:





"For testing data the RMSE is 0.9066538818812506"

How to make recommendations

So let's first load the movies complete file for later use.



In [9]:

    
var complete_movies_raw_data = 
    sc.textFile("../data/mllib/ml-latest-small/movies.csv");
var complete_movies_raw_data_header = complete_movies_raw_data.take(1)[0];
var complete_movies_data = complete_movies_raw_data.filter(function(line, complete_movies_raw_data_header) {
        // filters out the header
        return line != complete_movies_raw_data_header;
    }, [complete_movies_raw_data_header])

    .map(function(line, Tuple) {
        var fields = line.split(",");
        return new Tuple(parseInt(fields[0]), fields[1]);

    }, [Tuple]).cache();

var complete_movies_titles = complete_movies_data.mapToPair(
    function( tuple2, Tuple) { // Tuple2
        return new Tuple(tuple2[0], tuple2[1]);

    }, [Tuple]);

JSON.stringify("There are movies in the complete dataset " + complete_movies_titles.count());









    



[Stage 1623:>                                                       (0 + 2) / 2]





    Out[9]:





"There are movies in the complete dataset 10329"

Give recommendations of movies

Another thing we want to do, is give recommendations of movies with a certain minimum number of ratings. For that, we need to count the number of ratings per movie.



In [10]:

    
var movie_ID_with_ratings_RDD = complete_ratings_data.mapToPair(function( rating, Tuple) {
        return new Tuple(rating.product(), rating.rating());

}, [Tuple])
.groupByKey();

var movie_ID_with_avg_ratings_RDD = movie_ID_with_ratings_RDD.mapToPair(function( ID_and_ratings_tuple, Tuple) {

        var w = ID_and_ratings_tuple[1];
    var count = 0;
    var sum = 0;
    for (var i = 0; i < w.length; i++) {
        var r = w[i];
        sum += r;
        count++;
    }

        var avgRating = sum / count;
        return new Tuple(ID_and_ratings_tuple[0], new Tuple(count, avgRating));

    }, [Tuple]);

var movie_rating_counts_RDD = movie_ID_with_avg_ratings_RDD.mapToPair(function( ID_with_avg_ratings, Tuple) {
        var x =  ID_with_avg_ratings;
        var coutAvg = ID_with_avg_ratings[1];
        return new Tuple(ID_with_avg_ratings[0], coutAvg[0]); // movieID, rating count


}, [Tuple]);

JSON.stringify("movie_ID_with_avg_ratings_RDD  " + movie_ID_with_avg_ratings_RDD.take(10));









    



[Stage 1624:============================>                           (1 + 1) / 2]





    Out[10]:





"movie_ID_with_avg_ratings_RDD  (1084,(45,3.966666666666667)),(32196,(2,4)),(1490,(1,0.5)),(91902,(1,3)),(68522,(1,5)),(3702,(43,3.5348837209302326)),(6754,(26,3.4423076923076925)),(68482,(1,2.5)),(6308,(4,3)),(91622,(3,3.3333333333333335))"

Rate some movies for the new user.



In [11]:

    
var new_user_ID = 0;

// The format of each line is (userID, movieID, rating)
var new_user_ratings = [
    new Rating(0, 260, 9), // Star Wars (1977)
    new Rating(0, 1, 8), // Toy Story (1995)
    new Rating(0, 16, 7), // Casino (1995)
    new Rating(0, 25, 8), // Leaving Las Vegas (1995)
    new Rating(0, 32, 9), // Twelve Monkeys (a.k.a. 12 Monkeys) (1995)
    new Rating(0, 335, 4), // Flintstones, The (1994)
    new Rating(0, 379, 3), // Timecop (1994)
    new Rating(0, 296, 7), // Pulp Fiction (1994)
    new Rating(0, 858, 10), // Godfather, The (1972)
    new Rating(0, 50, 8) // Usual Suspects, The (1995)
];
var new_user_ratings_RDD = sc.parallelize(new_user_ratings);
JSON.stringify("New user ratings: " + new_user_ratings_RDD.take(10));









    Out[11]:





"New user ratings: {Rating: [0,260,9]},{Rating: [0,1,8]},{Rating: [0,16,7]},{Rating: [0,25,8]},{Rating: [0,32,9]},{Rating: [0,335,4]},{Rating: [0,379,3]},{Rating: [0,296,7]},{Rating: [0,858,10]},{Rating: [0,50,8]}"

Add them to the data we will use to train our recommender model.



In [12]:

    
var complete_data_with_new_ratings_RDD = complete_ratings_data.union(new_user_ratings_RDD);


var new_ratings_model = 
    ALS.train(complete_data_with_new_ratings_RDD, best_rank, 
              iterations, regularization_parameter, blocks, seed);

/*
 Let's now get some recommendations
 */

//  get just movie IDs
var new_user_ratings_ids = [];
for (var i = 0; i < new_user_ratings.length; i++) {
    new_user_ratings_ids.push(new_user_ratings[i].product());
}

// keep just those not on the ID list
var new_user_unrated_movies_RDD = complete_movies_data.filter(function( tuple, new_user_ratings_ids) {
        if (new_user_ratings_ids.indexOf(tuple[0]) < 0) {
            return true;
        } else {
            return false;
        }
}, [new_user_ratings_ids])
.map(function( tuple, new_user_ID, Tuple) {
        return new Tuple(new_user_ID, tuple[0]);

}, [new_user_ID, Tuple]);
// Use the input RDD, new_user_unrated_movies_RDD, 
//with new_ratings_model.predictAll() to predict new ratings for the movies
var new_user_recommendations_RDD = new_ratings_model.predict(new_user_unrated_movies_RDD);

// Transform new_user_recommendations_RDD into pairs of the form (Movie ID, Predicted Rating)
var new_user_recommendations_rating_RDD = new_user_recommendations_RDD.mapToPair( function( rating, Tuple) {
        return new Tuple(rating.product(), rating.rating());

}, [Tuple]);
var aRDD = new_user_recommendations_rating_RDD.join(complete_movies_titles);

var new_user_recommendations_rating_title_and_count_RDD =
    /*new_user_recommendations_rating_RDD.join(complete_movies_titles)*/ aRDD.join(movie_rating_counts_RDD);

"new_user_recommendations_rating_title_and_count_RDD " + new_user_recommendations_rating_title_and_count_RDD.count();









    



[Stage 1739:>                                                       (0 + 2) / 2]





    Out[12]:





new_user_recommendations_rating_title_and_count_RDD 10315

Flatten the RDD

we need to flat this down a bit in order to have (Title, Rating, Ratings Count).



In [13]:

    
var new_user_recommendations_rating_title_and_count_RDD2 =
    new_user_recommendations_rating_title_and_count_RDD.map(function( t, Tuple) {
        // ( 27456,( (7.553736917670094,Shackleton's Antarctic Adventure (2001) ),1) )
        // a = (27456, b)
        var a = /*(Tuple2)*/ t;
        // b = ( c ,1)
        var b = /*(Tuple2)*/ a[1];
        // c = (7.553736917670094,Shackleton's Antarctic Adventure (2001) )
        var c = /*(Tuple2)*/ b[0];
        var x = new Tuple(c[1], c[0], b[1]);
        return x;

}, [Tuple]);

JSON.stringify("new_user_recommendations_rating_title_and_count_RDD2" +
               new_user_recommendations_rating_title_and_count_RDD2.take(3));









    Out[13]:





"new_user_recommendations_rating_title_and_count_RDD2(Shackleton's Antarctic Adventure (2001),7.553736917670094,1),(\"Call of Cthulhu,8.860291348428946,2),(RV (2006),2.476218985783453,4)"

Get highest rated recommendations

Finally, get the highest rated recommendations for the new user, filtering out movies with less than 25 ratings.



In [14]:

    
var new_user_recommendations_rating_title_and_count_RDD2_filtered =
    new_user_recommendations_rating_title_and_count_RDD2.filter(function( tuple3) {
        if (tuple3[2] < 25) {
            return false;
        } else {
            return true;
        }
    });

/*
list top 25
 */

var top_movies = new_user_recommendations_rating_title_and_count_RDD2_filtered.takeOrdered(25,
   function(tuple3_a, tuple3_b){

        var aRate = tuple3_a[1];
        var bRate = tuple3_b[1];
        return aRate > bRate ? -1 : aRate == bRate? 0 : 1;

});
var str = "TOP recommended movies (with more than 25 reviews):\n\n";

for (var i = 0; i < top_movies.length; i++) {
    str += top_movies[i][0] + " average rating " + top_movies[i][1] + " number of ratings " + top_movies[i][2] + "\n";
}









    



[Stage 1902:============================>                           (4 + 4) / 8]





    Out[14]:





TOP recommended movies (with more than 25 reviews):

Being There (1979) average rating 8.628200481841834 number of ratings 26
Annie Hall (1977) average rating 8.546276091351059 number of ratings 68
My Neighbor Totoro (Tonari no Totoro) (1988) average rating 8.519469203967729 number of ratings 28
Citizen Kane (1941) average rating 8.496776961299442 number of ratings 77
Cinema Paradiso (Nuovo cinema Paradiso) (1989) average rating 8.440391886714995 number of ratings 37
All About Eve (1950) average rating 8.42478972179922 number of ratings 27
Princess Mononoke (Mononoke-hime) (1997) average rating 8.42020502325419 number of ratings 52
Manhattan (1979) average rating 8.400647804793403 number of ratings 40
Ran (1985) average rating 8.369219421907811 number of ratings 31
Spirited Away (Sen to Chihiro no kamikakushi) (2001) average rating 8.367017216556984 number of ratings 72
Lone Star (1996) average rating 8.313256584773576 number of ratings 35
Monty Python and the Holy Grail (1975) average rating 8.310843186811322 number of ratings 154
Old Boy (2003) average rating 8.310831456431455 number of ratings 38
Band of Brothers (2001) average rating 8.294547580780362 number of ratings 26
One Flew Over the Cuckoo's Nest (1975) average rating 8.2834222602241 number of ratings 143
Goodfellas (1990) average rating 8.282319190627655 number of ratings 135
"Sting average rating 8.2696152337938 number of ratings 77
Fargo (1996) average rating 8.268114373240525 number of ratings 201
Rear Window (1954) average rating 8.26193105355311 number of ratings 74
Up (2009) average rating 8.256685003489332 number of ratings 62
Sunset Blvd. (a.k.a. Sunset Boulevard) (1950) average rating 8.245400319453282 number of ratings 29
Harold and Maude (1971) average rating 8.229893025284042 number of ratings 33
"Departed average rating 8.228682614639016 number of ratings 87
12 Angry Men (1957) average rating 8.214880043481715 number of ratings 63
Into the Wild (2007) average rating 8.212655662767194 number of ratings 32

Predicted rating for a particular movie for a given user.



In [17]:

    
var my_movie = sc.parallelizePairs([new Tuple(0, 500)]); // Quiz Show (1994)
var individual_movie_rating_RDD = new_ratings_model.predict(my_movie);

"Predicted rating for movie " + individual_movie_rating_RDD.take(1);









    Out[17]:





Predicted rating for movie {Rating: [0,500,5.534481054561288]}



In [ ]: