Demonstrate Seq2Seq Wrapper with Cornell Movie Dialog Corpus


In [1]:
import tensorflow as tf
import numpy as np

# preprocessed data
from datasets.cornell_corpus import data
import data_utils

In [3]:
import importlib
importlib.reload(data)


Out[3]:
<module 'datasets.cornell_corpus.data' from '/home/suriya/_/tf/tf-seq2seq-wrapper/datasets/cornell_corpus/data.py'>

In [2]:
# load data from pickle and npy files
metadata, idx_q, idx_a = data.load_data(PATH='datasets/cornell_corpus/')
(trainX, trainY), (testX, testY), (validX, validY) = data_utils.split_dataset(idx_q, idx_a)

In [4]:
# parameters 
xseq_len = trainX.shape[-1]
yseq_len = trainY.shape[-1]
batch_size = 16
xvocab_size = len(metadata['idx2w'])  
yvocab_size = xvocab_size
emb_dim = 1024

In [5]:
import seq2seq_wrapper

In [6]:
model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,
                               yseq_len=yseq_len,
                               xvocab_size=xvocab_size,
                               yvocab_size=yvocab_size,
                               ckpt_path='ckpt/cornell_corpus/',
                               emb_dim=emb_dim,
                               num_layers=3
                               )


<log> Building Graph </log>

In [7]:
val_batch_gen = data_utils.rand_batch_gen(validX, validY, 32)
test_batch_gen = data_utils.rand_batch_gen(testX, testY, 256)
train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, batch_size)

In [8]:
sess = model.restore_last_session()

In [16]:
input_ = test_batch_gen.__next__()[0]
output = model.predict(sess, input_)
print(output.shape)


(256, 25)

In [17]:
replies = []
for ii, oi in zip(input_.T, output):
    q = data_utils.decode(sequence=ii, lookup=metadata['idx2w'], separator=' ')
    decoded = data_utils.decode(sequence=oi, lookup=metadata['idx2w'], separator=' ').split(' ')
    if decoded.count('unk') == 0:
        if decoded not in replies:
            print('q : [{0}]; a : [{1}]'.format(q, ' '.join(decoded)))
            replies.append(decoded)


q : [bill id like you to meet jack torrance]; a : [how much do you think]
q : [and who are you to talk you were nothing you couldnt even sing i must have been out of my mind]; a : [i dont know what youre talking about]
q : [by breaking up a companys assets]; a : [what are you talking about]
q : [what is it]; a : [i dont know]
q : [ill see you there]; a : [ill get out]
q : [okay ill be talking to you]; a : [youre not going to get out]
q : [i must be outta my mind buddy quit it]; a : [okay for a minute]
q : [when are you going to let the police know]; a : [you dont know what youre talking about]
q : [you can do it]; a : [yeah i think so]
q : [like hell you know if you fellows stuck together stead of letting them walk all over you they might not try it]; a : [if you werent talking about me i dont know what youre talking about]
q : [wait are you saying that i dont appreciate]; a : [i know you know what i mean]
q : [no you just looked as if you did]; a : [well i didnt believe you]
q : [youre terrible]; a : [yeah but im not]
q : [you would take a pain n the arse full stop]; a : [dont worry about it]
q : [yeah howd the kid hold up]; a : [whats that]
q : [yes well miss ratched feels that youre a disturbing influence on the other patients]; a : [dont you tell me]
q : [you leading a squad]; a : [what is it]
q : [nah i dont think so]; a : [why not]
q : [hes pissing in our faces again and were just taking it]; a : [i cant]
q : [my lady]; a : [she said you were a little girl]
q : [im sorry i dont know what else to say except im sorry]; a : [why would you like to be]
q : [mr brandon couldnt be here he might have left something for you what would it look like]; a : [thats it i think i can get it out]
q : [nobodys gonna get you now get inside]; a : [i cant go back]
q : [i just turned 25 i was 24 for a whole year]; a : [well i dont think so]
q : [make sure he doesnt leave]; a : [you sure]
q : [do we need him]; a : [i dont know what to do]
q : [youre terry unk arent you]; a : [what are you doing here]
q : [so lets go]; a : [i dont think so]
q : [ya owe me twentyfive bucks]; a : [its a good man]
q : [shall i leave]; a : [of course]
q : [unk unk probably asleep by now do you want him to see you like this]; a : [no way i dont want to get out of here]
q : [well i really think hes got a chance]; a : [i know]
q : [youd better be quiet sandy]; a : [shut up]
q : [buddy that was pretty unk of you pushing me away like that just when it was interesting]; a : [what about you]
q : [jesus christ you scared the shit out of me]; a : [whats going on]
q : [well im sorry im really sorry ellie]; a : [its okay]
q : [youre not a man because of a job duff]; a : [but i said that]
q : [he didnt lose her he threw her away]; a : [hes not dead]
q : [you unk have the gotta]; a : [what the hell are you talking about]
q : [whoa whoa what do you expect them to say youre alan unk]; a : [im sorry i dont know]
q : [doc what can i tell ya]; a : [go ahead]
q : [my lady this play will end badly i will tell]; a : [lets get out of here]
q : [what if i said goodbye]; a : [then why are you]
q : [im going to miss you]; a : [no youre not]
q : [well wed love to but were going to another party]; a : [what do you mean]
q : [dog eat dog unk you fuck other man before he fuck you and you must fuck last]; a : [what do you think we can do]
q : [not in the trunk]; a : [what about]
q : [can we light the candles now on the cake]; a : [you want to go home]
q : [you look frightened have i been saying something frightening]; a : [i dont know what you mean]
q : [you want to hear the good news first or the bad news]; a : [what the hell is that]
q : [ya gotta be a little soft to wanna be a unk its a racket where ya almost guaranteed to end up a bum]; a : [all right]
q : [you werent going with her]; a : [well shes not a little girl]
q : [i dont want land]; a : [you dont have to be a good man]
q : [what do you want from me cotton]; a : [we dont know what youre talking about]
q : [the hopes are perfect beautiful identical smooth and they are for something really amazing i feel it in my bones]; a : [youre not a fool]
q : [get married]; a : [what do you think]
q : [ha dear boy i do hope this doesnt unk a meeting in private]; a : [ill get out of here]
q : [karl you up]; a : [yeah im not going to get out of here]
q : [i just realized this is for television isnt it i cant swear up and down like i just did]; a : [its not that you know what youre talking about]
q : [yes george]; a : [and what do you want]
q : [hey eddie looks like you really stepped in it this time]; a : [thats right i dont know]
q : [you dont want to go all the way to san francisco in a unk do you i dont]; a : [im not sure]
q : [it can happen so sudden cant it being left out on your own]; a : [its not a good idea]
q : [what do you mean]; a : [i dont know i dont know what i mean]
q : [i want to]; a : [you want to go]
q : [what do you want take my wife please]; a : [well i dont know what to do]
q : [look at this the lock is totally unk]; a : [you know what it is]
q : [he wont steal im tellin you hes a pretty good ol boy keeps to himself]; a : [thats right]
q : [youre really pushing it bringing me here]; a : [what do you want me to do]
q : [im not unk with armitage and his unk breathing down my neck]; a : [you mean you dont want to go]
q : [my god these people are insane]; a : [we dont know what they are]
q : [no it wasnt like that]; a : [yes you did]
q : [dont you worry about that]; a : [dont worry about me]
q : [well i just kept unk that dish maybe it doesnt sound very sexy but it was]; a : [yeah i dont know]
q : [hey vaughan how are you karl]; a : [im fine]
q : [this isnt your room youre in unk i fucked up]; a : [im sorry i dont know what to do]
q : [r was worried about you you didnt even call youre always on my case if i dont call]; a : [i know you dont know what youre talking about]
q : [something happened you got unk in the last quarter]; a : [what do you want to do]
q : [if youre going to be unk i wish youd be a little more discreet about it rich men like unk love you and leave you]; a : [thats not what i mean]
q : [how long would they let me sleep]; a : [i dont know i dont know what to do]
q : [whats got billy so unk]; a : [i dont know what to do with it]
q : [you fucking bastard]; a : [come on]
q : [she used to call me mr right remember that buddy]; a : [how are you going to do]
q : [my unk in miami its nice down there]; a : [thats what i said]
q : [speak with my lawyer]; a : [you dont know what you mean]
q : [mr unk because you love the theatre you must have a part in my play i am writing an unk a small but vital role]; a : [thats right i dont think so]
q : [of course you dont know anything about it if you knew anything about it i wouldnt have to send you over there to cover it]; a : [thats not what you want]
q : [does mom know]; a : [she said she was in love]
q : [its okay tatum shes just doing her job right gale]; a : [shes not going to see her]
q : [wanna keep goin]; a : [im sorry]
q : [i thought so too unk a neat guy]; a : [hey i dont know what youre doing to do with it]
q : [did i wake you]; a : [no you dont]
q : [how could you do this]; a : [well i dont know what i mean]
q : [that can wait till the weekend]; a : [are you kidding]
q : [if you cant look anymore i understand]; a : [im not going to tell you that]
q : [maybe i can change him]; a : [then he doesnt know what he is]
q : [maybe if i kiss him ill feel it]; a : [then you should have to]
q : [well youre a little early]; a : [i dont know what youre doing to get out of here]
q : [i thought michael was picking me up]; a : [dont worry about it im not going to get out of here]
q : [i dont believe he did sir i couldnt find a single track just doesnt make sense]; a : [shut up what do you mean]
q : [he said that]; a : [he was here]
q : [its unk unk the name of the character she plays in the movie]; a : [i know what i mean]
q : [hes company]; a : [hes not going to see him]
q : [youre so proud youre like some retarded kid comin home from school look dad i got an f]; a : [thats right thats what you want to do]
q : [feels like theres a bullet still in my chest]; a : [right for me]
q : [not now charlie ive got a headache get used to the word roll it around your tongue for a years]; a : [its the time i dont know what to do]
q : [and the bookstore have you been working there long]; a : [i think so]
q : [you sent for me]; a : [yes yes i am]
q : [damour damour why do i know that name]; a : [well i dont know]
q : [speaking of which you run that license plate for me]; a : [i cant believe you]
q : [i dont know unk a unk]; a : [you know what i mean]
q : [how shes even forgot her own language]; a : [long night she was a very nice person]
q : [just that she got away]; a : [well i think so]
q : [anything i dont care what it is just so its something]; a : [its not that i dont know]
q : [at this hour]; a : [no problem]
q : [i swear it he wants romeo for ned and the unk men]; a : [and how much do you think]
q : [oh really i thought it was pretty good]; a : [its a good idea]
q : [look fry company says were responsible for every one of those]; a : [dont be silly]
q : [laser unk you cant get the code wrong it unk you i cant let you try it]; a : [ill take care of you]
q : [what if we were to put bruce into the park as a guest]; a : [dont worry ill be fine if you want to get out of here]
q : [could be]; a : [but what do you think]
q : [i dont know i cant say]; a : [then what did you do]

In [ ]: