In [11]:
from __future__ import division
import numpy
import imaginet.task as task
import imaginet.defn.audiovis_rhn as audiovis
import imaginet.defn.visual2_rhn as vis2
In [119]:
import imaginet.vendrov_provider as dp
prov = dp.getDataProvider(dataset='coco', root='/home/gchrupala/reimaginet/')
sent = list(prov.iterSentences(split='val'))
In [57]:
model_s = task.load("/home/gchrupala/reimaginet/run-rhn-coco-9-resume/model.r.e9.zip")
model_w = task.load("/home/gchrupala/reimaginet/run-rhn-coco-word-4/model.r.e14.zip")
In [5]:
import json
In [6]:
score_s = json.load(open("/home/gchrupala/reimaginet/run-rhn-coco-9-resume/scores.9.json"))
score_w = json.load(open("/home/gchrupala/reimaginet/run-rhn-coco-word-4/scores.14.json"))
In [18]:
import numpy
def extreme(good, worse):
ratio = numpy.array(good['ranks']) / numpy.array(worse['ranks'])
return numpy.argsort(ratio)
In [26]:
for j in extreme(score_w, score_s)[:10]:
print score_w['ranks'][j], "/", score_s['ranks'][j], sent[j]['raw']
In [195]:
for j in extreme(score_s, score_w)[:50]:
print score_w['ranks'][j], "/", score_s['ranks'][j], j, sent[j]['raw']
In [179]:
extreme(score_w, score_s)[-10:],extreme(score_s, score_w)[:10]
Out[179]:
In [218]:
from collections import Counter
vocab = Counter((word for s in prov.iterSentences(split='train') for word in s['tokens']))
In [227]:
set_s = set(word for j in extreme(score_s, score_w)[:100] for word in sent[j]['tokens'])
set_w = set(word for j in extreme(score_w, score_s)[:100] for word in sent[j]['tokens'])
print numpy.sum([1 for word in set_s if vocab[word] < 5])
print numpy.sum([1 for word in set_w if vocab[word] < 5])
print [word for word in set_s if vocab[word] < 5]
print [word for word in set_w if vocab[word] < 5]
In [217]:
for s in prov.iterSentences(split='train'):
if 'streeet' in s['tokens']:
print s['raw']
In [30]:
def extreme_stats(good, worse, N=100):
J = extreme(good, worse)[:N]
L = [len(sent[j]['tokens']) for j in J]
R = [good['ranks'][j] / worse['ranks'][j] for j in J]
return (L, R)
In [32]:
Lw,Rw = extreme_stats(score_w, score_s)
Ls,Rs = extreme_stats(score_s, score_w)
In [36]:
numpy.max(Lw), numpy.max(Ls)
Out[36]:
In [41]:
numpy.mean(Rw), numpy.mean(Rs)
Out[41]:
In [43]:
import pandas as pd
In [180]:
data = pd.DataFrame(dict(Length=numpy.hstack([Lw,Ls]), better=numpy.hstack([numpy.repeat("text",100), numpy.repeat("speech",100)])))
In [181]:
with open("error-length.txt","w") as f:
f.write(data.to_csv(index=False))
In [120]:
images = list(prov.iterImages(split='val'))
from imaginet.evaluate import Cdist
from imaginet.simple_data import words
In [121]:
def retrieve(model, query, images, config):
task = model.task
scaler = model.scaler
batcher = model.batcher
mapper = batcher.mapper
sents = [ query ]
sents_tok = [ config['tokenize'](query) ]
predictions = config['encode_sentences'](model, sents_tok, batch_size=32)
img_fs = task.encode_images([ img['feat'] for img in images ])
correct_img = numpy.array([ [ sents[i]['imgid']==images[j]['imgid']
for j in range(len(images)) ]
for i in range(len(sents)) ] )
distances = Cdist(batch_size=2**13)(predictions, img_fs)
row = distances[0]
ranked = numpy.argsort(row)
return ranked
In [123]:
coco = json.load(open("/home/gchrupala/reimaginet/data/coco/dataset.json"))
In [124]:
COCOID = {}
for img in coco['images']:
COCOID[img['imgid']]=img['cocoid']
In [160]:
from IPython.display import Image
from IPython.core.display import HTML
def display(imgid):
return Image(url='http://mscoco.org/images/%d'%(COCOID[images2[imgid]['imgid']]))
In [165]:
def audio(query):
return query['audio']
r_w = retrieve(model_w, sent[17931], images, dict(encode_sentences=vis2.encode_sentences, tokenize=words))
r_s = retrieve(model_s, sent[17931], images, dict(encode_sentences=audiovis.encode_sentences, tokenize=audio))
In [139]:
import imaginet.data_provider as dp2
In [141]:
prov2 = dp2.getDataProvider(dataset='coco', root='/home/gchrupala/reimaginet/', audio_kind=None)
In [142]:
images2 = list(prov2.iterImages(split='val'))
In [161]:
images[0]['imgid'], images[0]['sentences'][0]['raw']
Out[161]:
In [162]:
sent[17931]['raw']
Out[162]:
In [163]:
display(r_w[0])
Out[163]:
In [164]:
display(r_s[1])
Out[164]:
In [156]:
for i in range(5):
print images2[r_s[1]]['sentences'][i]['raw']
In [169]:
def compare(i,j=0):
r_w = retrieve(model_w, sent[i], images, dict(encode_sentences=vis2.encode_sentences, tokenize=words))
r_s = retrieve(model_s, sent[i], images, dict(encode_sentences=audiovis.encode_sentences, tokenize=audio))
return(sent[i]['raw'], r_s, r_w)
In [170]:
s, i1, i2 = compare(9544)
In [171]:
s
Out[171]:
In [175]:
display(i1[1])
Out[175]:
In [176]:
display(i2[1])
Out[176]:
In [182]:
compare(11564)
Out[182]:
In [183]:
display(2312)
Out[183]:
In [185]:
display(188)
Out[185]:
In [190]:
compare(10991,j=1)
Out[190]:
In [188]:
display(2198)
Out[188]:
In [189]:
display(703)
Out[189]:
In [192]:
display(2193)
Out[192]:
In [193]:
display(1953)
Out[193]:
In [ ]: