Ranking Metrics: DCG, NDCG, and MAP

In [43]:

    

import numpy as np


    

In [55]:

    

y1 = [True, True, True, True, True]
y2 = [False, True, True, True, True]
y3 = [True, False, True, True, True]
y4 = [True, True, False, True, True]
y5 = [True, True, False, False, True]
y6 = [True, True, False, True, False]


    

In [67]:

    

def ap(rel):
    k = len(rel)
    return 1 / k * sum([
        (1 / (i + 1)) * sum(rel[:i+1])
        for i in range(k)
    ])

def ndcg(rel):
    k = len(rel)
    best = dcg([True] * k, k)
    dcg_ = dcg(rel, k)
    return dcg_ / best

def dcg(rel, k):
    return sum([rel[r] / np.log(r + 1 + 1)
               for r in range(k)])

def compare(y):
    print('NDCG: {:.3f}, AP: {:.3f}'.format(ndcg(y), ap(y)))


    

In [68]:

    

compare(y1)


    

    




NDCG: 1.000, AP: 1.000

In [69]:

    

compare(y2)


    

    




NDCG: 0.661, AP: 0.543

In [70]:

    

compare(y3)


    

    




NDCG: 0.786, AP: 0.743

In [71]:

    

compare(y3)


    

    




NDCG: 0.786, AP: 0.743

In [73]:

    

compare(y2 + [True] * 100)


    

    




NDCG: 0.954, AP: 0.950

In [74]:

    

compare(y5 + [True] * 100)


    

    




NDCG: 0.957, AP: 0.932

In [75]:

    

compare(y1 + [False] * 100)


    

    




NDCG: 0.136, AP: 0.188

In [80]:

    

compare(y1 + [False] * 10000)


    

    




NDCG: 0.003, AP: 0.004