In [ ]:

    

import json
from pprint import pprint
import matplotlib.pyplot as plt

folder = '../Questions/'
qfile = folder + '/MultipleChoice_mscoco_val2014_questions.json';

with open(qfile) as data_file:    
    qdata = json.load(data_file)

ofile =  'coco/annotations/instances_val2014.json';
with open(ofile) as data_file:
    odata = json.load(data_file)

# dictionary mapping from category to name
cdata = odata['categories']
dict_cls_name = dict([(cdata[i]['id'],cdata[i]['name']) for i in range(len(cdata))])
dict_name_cls = dict([(cdata[i]['name'],cdata[i]['id']) for i in range(len(cdata))])  

idata = odata['images']
adata = odata['annotations']

# dictionary mapping from image id to area
dict_img_area = {}
for i in range(len(idata)):
    img = idata[i]['id']
    w = idata[i]['width']
    h = idata[i]['height']
    assert(img not in dict_img_area)
    dict_img_area[img] = w*h   
    
# compute average instance area
a = 0.0
n = 0.0
ll = 0
for k in range(len(adata)):
    img = adata[k]['image_id']
    area = adata[k]['area']/dict_img_area[img]
    if area < 0.05:
        ll += 1
    a += area
    n += 1
a = a/n
ll = ll/n
print 'Instance level: area',a,'num',n,'fraction < 5%',ll

    
# dictionary mapping from (imageid, cls) to area
dict_imgcls_area = {}
for k in range(len(adata)):
    img = adata[k]['image_id']
    cls = adata[k]['category_id']
    area = adata[k]['area']/dict_img_area[img]
    if (img,cls) in dict_imgcls_area:
        dict_imgcls_area[(img,cls)] += area
    else:
        dict_imgcls_area[(img,cls)] = area
        
# COMPUTE AVERAGE SIZE OF ALL CATEGORIES
a = 0.0
num_total = 0.0
for ic in dict_imgcls_area:
    a += dict_imgcls_area[ic]
    num_total += 1

area_avg = a/num_total
print 'Img+cat level: area',area_avg,'num',num_total


    

In [ ]:

    

# dictionary mapping from image id to list of words
dict_img_words = {}
for i in range(0,len(qdata['questions'])):
    img = qdata['questions'][i]['image_id']
    if img in dict_img_words:
        d = dict_img_words[img]
    else:
        d = set()
    q = qdata['questions'][i]['question']
    w = q.split()
    w = [ww.lower() for ww in w]
    #if w[0] == 'is' and 'person' not in set(w):
    #    w = []
    d = d.union(set(w))
    dict_img_words[img] = d

# dictionary mapping from image id to set of detected classes
dict_img_detcls = {}
for img in dict_img_words:
    words = dict_img_words[img]
    detected = []
    for w in words:
        if w in dict_name_cls:
            c = dict_name_cls[w]
            detected += [c]
    # can do something more complicated later
    dict_img_detcls[img] = set(detected)


    

In [ ]:

    

import pickle 
golden = pickle.load(open( 'coco_vectors/imgids_labels.p', "rb" ) )

for img in golden:
    cls = golden[img]
    if 897 in cls:
        print img

#dict_img_detcls = golden

for img in dict_img_detcls:
    dict_img_detcls[img] = dict_img_detcls[img].union(set(golden[img]))


    

In [ ]:

    

# COMPUTING AVERAGE SIZE OF DETECTED
ad = 0.0
nd = 0.0
au = 0.0
nu = 0.0
for ic in dict_imgcls_area:
    img = ic[0]
    cls = ic[1]
    area = dict_imgcls_area[ic]
    if cls in dict_img_detcls[img]:
        ad += area
        nd += 1
    else:
        au += area
        nu += 1

ad = ad/nd
au = au/nu
print 'Detected: area',ad,'num',nd
print 'Undetected: area',au,'num',nu

# COMPUTE PER-CATEGORY AREA
dict_cls_area = {}
for ic in dict_imgcls_area:
    cls = ic[1]
    area = dict_imgcls_area[ic]
    if cls in dict_cls_area:
        dict_cls_area[cls] += [area]
    else:
        dict_cls_area[cls] = [area]

for c in dict_cls_area:
    a = dict_cls_area[c]
    dict_cls_area[c] = sum(a)/len(a)

# COMPUTING AVERAGE PER-CATEGORY AREA
a = dict_cls_area.values()
a = sum(a)/len(a)
print 'Per-category average area',a


# COMPUTING PER-CATEGORY RECALL
dict_cls_recall = {}
for ic in dict_imgcls_area:
    img = ic[0]
    cls = ic[1]
    # this class is present in this image
    bDetected = cls in dict_img_detcls[img]
    # this class was maybe detected
    if cls in dict_cls_recall:
        dict_cls_recall[cls] += [bDetected]
    else:
        dict_cls_recall[cls] = [bDetected]

for c in dict_cls_recall:
    r = dict_cls_recall[c]
    dict_cls_recall[c] = float(sum(r))/len(r)
        
a = dict_cls_recall.values()
a = sum(a)/len(a)
print 'Per-category average recall',a

dict_cls_precision = {}
for img in dict_img_detcls:
    detcls = dict_img_detcls[img] # detected classes
    for c in detcls:
        bCorrect = (img,c) in dict_imgcls_area # present in the image
        if c in dict_cls_precision:
            dict_cls_precision[c] += [bCorrect]
        else:
            dict_cls_precision[c] = [bCorrect]

for c in dict_cls_precision:
    p = dict_cls_precision[c]
    dict_cls_precision[c] = float(sum(p))/len(p)
    
avg_p = 0.0
for c in dict_cls_recall:
    if c in dict_cls_precision:
        avg_p += dict_cls_precision[c]
    else:
        print '  (Not detected',c,dict_cls_name[c],')'
        avg_p += 1
a = avg_p/len(dict_cls_recall)
print 'Per-category average precision',a


    

In [ ]:

    

%matplotlib inline

fl = 16
ft = 13

classes = dict_cls_area.keys()
x = [dict_cls_area[c] for c in classes]
y = [dict_cls_recall[c] for c in classes]
ax1 = plt.subplot(1,2,1)
ax1.plot(x,y,'o')
ax1.set_xlim(0,0.35)
ax1.set_ylim(0,1)
ax1.set_xlabel('Object size',fontsize=fl)
ax1.set_ylabel('Recall',fontsize=fl)
plt.xticks([0, 0.1, 0.2, 0.3],fontsize=ft)
plt.yticks(fontsize=ft)

x = [dict_cls_area[c] for c in classes]
y = []
for c in classes:
    if c in dict_cls_precision:
        y += [dict_cls_precision[c]]
    else:
        y += [1]
ax2 = plt.subplot(1,2,2)
ax2.plot(x,y,'o')
ax2.set_xlim(0,0.35)
ax2.set_ylim(0,1)
ax2.set_xlabel('Object size',fontsize=fl)
ax2.set_ylabel('Precision',fontsize=fl)
plt.xticks([0, 0.1, 0.2, 0.3],fontsize=ft)
plt.yticks(fontsize=ft)

plt.tight_layout()
plt.show()


    

In [ ]:

    

detcls = set([])
for img in dict_img_detcls:
    detcls = detcls.union(dict_img_detcls[img])
    
for c in dict_cls_name:
    if c not in detcls:
        print dict_cls_name[c]