In [1]:

    

from gensim import utils
import gensim.models.doc2vec
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import gensim
import sys
import numpy as np
from gensim import corpora, models
import csv
import _pickle as cPickle
from sklearn.externals import joblib
import bz2
from random import shuffle
import ast
from sklearn.linear_model import LogisticRegression


    

In [3]:

    

# Load to memory
corpus = gensim.corpora.MmCorpus('data/eos/processed_corpus_EOS.mm')
    
File = open('data/eos/doc2vec_data.txt', 'w') 
for doc in doc2vec_data:
    File.write(' '.join(token for token in doc))


    

In [ ]:

    

%%time


doc_filepath = 'data/eos/eos_text_tokenized.gz'
doc2vec_model_path = 'data/eos/doc2vec_model.d2v'

if 1 == 1:
    sentences=gensim.models.doc2vec.TaggedLineDocument(doc_filepath)
    model = gensim.models.doc2vec.Doc2Vec(sentences, size=200, window=10, min_count=5, iter=20, workers=8)
    model.save(doc2vec_model_path)
else:
    model = Doc2Vec.load(doc2vec_model_path)


    

In [5]:

    




    

In [6]:

    

#The full model was run on a server. The first output below is a sample model run on 1000 lines
#of text. The markdown cell below the output is the output on the server for 30 million lines.
sims = model.docvecs.most_similar(99)

print(sims)


    

    




[(5377, 0.5258454084396362), (5599, 0.37886685132980347), (5590, 0.36026638746261597), (7184, 0.3549143075942993), (9802, 0.34451237320899963), (4488, 0.34104377031326294), (3348, 0.34001803398132324), (814, 0.3253442645072937), (4050, 0.32484573125839233), (8534, 0.3236147165298462)]

In [7]:

    

print(model.doesnt_match("iran diplomat scarf".split()))
print(model.doesnt_match("black blue yellow shirt navy black green orange".split()))
print(model.doesnt_match("summer winter fall t-shirt spring hot cold".split()))
print(model.doesnt_match("straight slim fit custom regular winter".split()))


print(model.most_similar(positive=['saudi', 'king'], negative=['girl']))
print(model.most_similar(positive=['blue', 'shirt'], negative=['blue']))
print(model.most_similar(positive=['calvin', 'klein'], negative=['tommy']))
print(model.most_similar(positive=['cotton', 'material'], negative=['polyester']))
print(model.most_similar(positive=['nike', 'run'], negative=['express']))



print(model.most_similar_cosmul(positive=['calvin', 'klein'], negative=['tommy']) )
print(model.most_similar_cosmul(positive=['skinny', 'jean'], negative=['large']) )
print(model.most_similar_cosmul(positive=['black', 'dress'], negative=['navy']) )
print(model.most_similar_cosmul(positive=['blue', 'coat'], negative=['yellow']) )


    

    




diplomat
yellow
fall
winter
[('sheikh', 0.29759371280670166), ('abdullah', 0.2927790582180023), ('gcc', 0.2861786484718323), ('shaikh', 0.27961617708206177), ('bin', 0.2788739800453186), ('abdulaziz', 0.27637121081352234), ('nayef', 0.2659643888473511), ('renew', 0.2647008001804352), ('indian', 0.26459500193595886), ('hrh', 0.26205959916114807)]
[('couldn', 0.5273951292037964), ('iba', 0.5182523727416992), ('voorheen', 0.4993104636669159), ('aren', 0.499233216047287), ('kearney', 0.4980449676513672), ('hadn', 0.4979991316795349), ('actueel', 0.49152493476867676), ('wasn', 0.4747185707092285), ('elijah', 0.4691956639289856), ('weren', 0.4674947261810303)]






    




---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-7-741cfaac0023> in <module>()
      7 print(model.most_similar(positive=['saudi', 'king'], negative=['girl']))
      8 print(model.most_similar(positive=['blue', 'shirt'], negative=['blue']))
----> 9 print(model.most_similar(positive=['calvin', 'klein'], negative=['tommy']))
     10 print(model.most_similar(positive=['cotton', 'material'], negative=['polyester']))
     11 print(model.most_similar(positive=['nike', 'run'], negative=['express']))

/usr/local/lib/python3.5/dist-packages/gensim/models/word2vec.py in most_similar(self, positive, negative, topn, restrict_vocab, indexer)
   1202         """
   1203 
-> 1204         return self.wv.most_similar(positive, negative, topn, restrict_vocab, indexer)
   1205 
   1206     def wmdistance(self, document1, document2):

/usr/local/lib/python3.5/dist-packages/gensim/models/keyedvectors.py in most_similar(self, positive, negative, topn, restrict_vocab, indexer)
    320                 mean.append(weight * word)
    321             else:
--> 322                 mean.append(weight * self.word_vec(word, use_norm=True))
    323                 if word in self.vocab:
    324                     all_words.add(self.vocab[word].index)

/usr/local/lib/python3.5/dist-packages/gensim/models/keyedvectors.py in word_vec(self, word, use_norm)
    273                 return self.syn0[self.vocab[word].index]
    274         else:
--> 275             raise KeyError("word '%s' not in vocabulary" % word)
    276 
    277     def most_similar(self, positive=[], negative=[], topn=10, restrict_vocab=None, indexer=None):

KeyError: "word 'tommy' not in vocabulary"