In [1]:

    

import collections
import math
import os
import random
import zipfile
import numpy as np
import tensorflow as tf


    

In [2]:

    

def read_data(filename):
    with zipfile.ZipFile(filename) as f:
        data = tf.compat.as_str(f.read(f.namelist()[0])).split()
    return data

vocabulary = read_data("text8.zip")
print('Data size', len(vocabulary))


    

    




Data size 17005207

In [3]:

    

vocabulary_size = 50000

def build_dataset(words, n_words):
    """Process raw inputs into a dataset."""
    count = [['UNK', -1]]
    count.extend(collections.Counter(words).most_common(n_words - 1))
    dictionary = dict()
    for word, _ in count:
        dictionary[word] = len(dictionary)
    data = list()
    unk_count = 0
    for word in words:
        if word in dictionary:
            index = dictionary[word]
        else:
            index = 0  # dictionary['UNK']
            unk_count += 1
        data.append(index)
    count[0][1] = unk_count
    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
    return data, count, dictionary, reversed_dictionary
data, count, dictionary, reverse_dictionary = build_dataset(vocabulary,
                                                            vocabulary_size)


    

In [4]:

    

data_index = 0
def generate_batch(batch_size, num_skips, skip_window):
    global data_index
    assert batch_size % num_skips == 0
    assert num_skips <= 2 * skip_window
    batch = np.ndarray(shape=(batch_size), dtype=np.int32)
    labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
    span = 2 * skip_window + 1  # [ skip_window target skip_window ]
    buffer = collections.deque(maxlen=span)
    for _ in range(span):
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)
    for i in range(batch_size // num_skips):
        target = skip_window  # target label at the center of the buffer
        targets_to_avoid = [skip_window]
        for j in range(num_skips):
            while target in targets_to_avoid:
                target = random.randint(0, span - 1)
            targets_to_avoid.append(target)
            batch[i * num_skips + j] = buffer[skip_window]
            labels[i * num_skips + j, 0] = buffer[target]
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)
    # Backtrack a little bit to avoid skipping words in the end of a batch
    data_index = (data_index + len(data) - span) % len(data)
    return batch, labels

batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)


    

In [5]:

    

# Step 4: Build and train a skip-gram model.

batch_size = 128
embedding_size = 128  # Dimension of the embedding vector.
skip_window = 1       # How many words to consider left and right.
num_skips = 2         # How many times to reuse an input to generate a label.

# We pick a random validation set to sample nearest neighbors. Here we limit the
# validation samples to the words that have a low numeric ID, which by
# construction are also the most frequent.
valid_size = 16     # Random set of words to evaluate similarity on.
valid_window = 100  # Only pick dev samples in the head of the distribution.
valid_examples = np.random.choice(valid_window, valid_size, replace=False)
num_sampled = 64    # Number of negative examples to sample.

graph = tf.Graph()

with graph.as_default():

  # Input data.
  train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
  train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
  valid_dataset = tf.constant(valid_examples, dtype=tf.int32)

  # Ops and variables pinned to the CPU because of missing GPU implementation
  with tf.device('/cpu:0'):
    # Look up embeddings for inputs.
    embeddings = tf.Variable(
        tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
    embed = tf.nn.embedding_lookup(embeddings, train_inputs)

    # Construct the variables for the NCE loss
    nce_weights = tf.Variable(
        tf.truncated_normal([vocabulary_size, embedding_size],
                            stddev=1.0 / math.sqrt(embedding_size)))
    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))

  # Compute the average NCE loss for the batch.
  # tf.nce_loss automatically draws a new sample of the negative labels each
  # time we evaluate the loss.
  loss = tf.reduce_mean(
      tf.nn.nce_loss(weights=nce_weights,
                     biases=nce_biases,
                     labels=train_labels,
                     inputs=embed,
                     num_sampled=num_sampled,
                     num_classes=vocabulary_size))

  # Construct the SGD optimizer using a learning rate of 1.0.
  optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)

  # Compute the cosine similarity between minibatch examples and all embeddings.
  norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
  normalized_embeddings = embeddings / norm
  valid_embeddings = tf.nn.embedding_lookup(
      normalized_embeddings, valid_dataset)
  similarity = tf.matmul(
      valid_embeddings, normalized_embeddings, transpose_b=True)

  # Add variable initializer.
  init = tf.global_variables_initializer()


    

In [6]:

    

# Step 5: Begin training.
num_steps = 2000001

with tf.Session(graph=graph) as session:
  # We must initialize all variables before we use them.
  init.run()
  print('Initialized')

  average_loss = 0
  for step in range(num_steps):
    batch_inputs, batch_labels = generate_batch(
        batch_size, num_skips, skip_window)
    feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}

    # We perform one update step by evaluating the optimizer op (including it
    # in the list of returned values for session.run()
    _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)
    average_loss += loss_val

    if step % 20000 == 0:
      if step > 0:
        average_loss /= 20000
      # The average loss is an estimate of the loss over the last 2000 batches.
      print('Average loss at step ', step, ': ', average_loss)
      average_loss = 0

    # Note that this is expensive (~20% slowdown if computed every 500 steps)
    if step % 200000 == 0:
      sim = similarity.eval()
      for i in range(valid_size):
        valid_word = reverse_dictionary[valid_examples[i]]
        top_k = 8  # number of nearest neighbors
        nearest = (-sim[i, :]).argsort()[1:top_k + 1]
        log_str = 'Nearest to %s:' % valid_word
        for k in range(top_k):
          close_word = reverse_dictionary[nearest[k]]
          log_str = '%s %s,' % (log_str, close_word)
        print(log_str)
  final_embeddings = normalized_embeddings.eval()


    

    




Initialized
Average loss at step  0 :  285.415283203
Nearest to from: leh, gemological, lights, jujutsu, cages, bride, dives, confidence,
Nearest to into: musically, occlusion, pathologist, grothendieck, by, mcneil, lud, fredric,
Nearest to been: illustrators, judson, overshadowed, doubt, musab, hellenism, roaring, sato,
Nearest to and: maududi, zoot, poker, posed, benz, halfwidth, solvent, broken,
Nearest to which: devoted, mackenzie, fullwidth, originality, separable, plaintexts, churches, icsu,
Nearest to while: woolwich, disguise, dayton, metrical, pond, neutrino, especially, pottery,
Nearest to between: whistleblower, bushman, thriller, distinguishing, genotype, uruguayan, trombone, emulation,
Nearest to he: seriousness, grantley, apsu, raiding, hani, troughs, unforgettable, indoctrination,
Nearest to if: bryson, awe, dominion, heathland, demographics, negating, adhesion, cannabis,
Nearest to to: encompassed, axel, jacopo, mr, tain, poetic, compute, necromancers,
Nearest to were: horsemanship, safely, venetians, barra, warheads, sorcerers, vassar, compression,
Nearest to more: anteater, leben, uttering, emperors, advising, blackface, permitted, roebuck,
Nearest to people: niggers, crete, harassment, lenition, jurist, circling, bertrand, increasingly,
Nearest to six: sociopolitical, sangha, youngster, accorded, ionized, entropy, peres, titan,
Nearest to when: wager, accomplished, jolson, lead, examiners, glaucus, pomeranian, invoked,
Nearest to for: headlining, confessors, webern, vy, euphemism, kirghiz, conquered, backfield,
Average loss at step  20000 :  29.3341755066
Average loss at step  40000 :  6.11049232572
Average loss at step  60000 :  5.13965054524
Average loss at step  80000 :  4.82088492196
Average loss at step  100000 :  4.71097334914
Average loss at step  120000 :  4.5790661846
Average loss at step  140000 :  4.50702497419
Average loss at step  160000 :  4.47495551547
Average loss at step  180000 :  4.44545509901
Average loss at step  200000 :  4.38291290536
Nearest to from: through, during, into, in, under, after, roshan, hyi,
Nearest to into: through, from, under, affectionate, tempest, elbl, between, within,
Nearest to been: be, become, was, were, had, come, ever, recently,
Nearest to and: or, but, thibetanus, one, circ, while, szko, dasyprocta,
Nearest to which: that, this, what, it, also, but, usually, however,
Nearest to while: although, when, however, but, though, before, circ, dasyprocta,
Nearest to between: with, since, chiapas, into, around, plissken, circ, within,
Nearest to he: she, it, they, there, who, never, eventually, bpm,
Nearest to if: when, though, where, although, however, archaeopteryx, while, mitral,
Nearest to to: would, will, wct, ursus, should, through, szko, landesverband,
Nearest to were: are, was, have, had, those, be, is, been,
Nearest to more: less, most, very, dominos, greater, rather, better, veitch,
Nearest to people: men, children, those, landesverband, waw, women, zero, accounts,
Nearest to six: five, seven, four, three, eight, nine, zero, hyi,
Nearest to when: if, while, although, where, after, before, mamertines, though,
Nearest to for: after, dasyprocta, during, busan, when, against, ensuring, wct,
Average loss at step  220000 :  4.3289360687
Average loss at step  240000 :  4.39369589608
Average loss at step  260000 :  4.31544707125
Average loss at step  280000 :  4.34802190154
Average loss at step  300000 :  4.30021018691
Average loss at step  320000 :  4.31524689076
Average loss at step  340000 :  4.27723854162
Average loss at step  360000 :  4.30084432677
Average loss at step  380000 :  4.28915596345
Average loss at step  400000 :  4.25223201886
Nearest to from: through, into, during, in, rajons, under, mjw, within,
Nearest to into: through, from, within, tempest, affectionate, across, elbl, between,
Nearest to been: be, become, was, were, come, remained, recently, had,
Nearest to and: or, but, including, pngimage, hyi, rajonas, wct, however,
Nearest to which: that, this, what, usually, safl, typically, actually, but,
Nearest to while: although, when, however, though, kinderhook, but, after, before,
Nearest to between: with, lup, within, among, in, plissken, around, into,
Nearest to he: she, it, they, there, who, we, waas, bpm,
Nearest to if: when, though, where, although, since, archaeopteryx, while, however,
Nearest to to: mjt, mjw, audita, wct, dasyprocta, ursus, agave, rajonas,
Nearest to were: are, was, have, those, be, had, pontificia, rajonas,
Nearest to more: less, most, very, quite, rather, greater, larger, longer,
Nearest to people: men, women, children, jews, person, individuals, landesverband, them,
Nearest to six: four, seven, five, eight, three, nine, zero, hyi,
Nearest to when: if, while, although, where, before, after, though, during,
Nearest to for: dasyprocta, wct, including, audita, mjw, in, rajons, if,
Average loss at step  420000 :  4.2332284336
Average loss at step  440000 :  4.26715067364
Average loss at step  460000 :  4.28474414814
Average loss at step  480000 :  4.1241690493
Average loss at step  500000 :  4.25148263264
Average loss at step  520000 :  4.18772170489
Average loss at step  540000 :  4.22940573909
Average loss at step  560000 :  4.20050674417
Average loss at step  580000 :  4.20434691161
Average loss at step  600000 :  4.19083545874
Nearest to from: through, across, during, via, rajons, into, mjw, rajonas,
Nearest to into: through, within, across, from, tempest, elbl, away, affectionate,
Nearest to been: be, become, come, was, remained, already, were, recently,
Nearest to and: or, but, however, rajonas, including, although, mammuthus, thibetanus,
Nearest to which: that, this, what, actually, typically, safl, usually, rajonas,
Nearest to while: although, when, though, however, kinderhook, including, unlike, after,
Nearest to between: lup, with, among, within, across, plissken, hangs, mackaye,
Nearest to he: she, it, they, there, we, who, waas, mitral,
Nearest to if: when, though, where, although, since, before, unless, archaeopteryx,
Nearest to to: audita, dasyprocta, mjw, mjt, wct, rajonas, would, agave,
Nearest to were: are, have, was, those, these, pontificia, incline, although,
Nearest to more: less, very, most, quite, greater, rather, larger, longer,
Nearest to people: men, women, children, individuals, jews, peoples, fans, person,
Nearest to six: four, five, eight, seven, three, two, nine, hyi,
Nearest to when: if, while, although, where, before, though, after, however,
Nearest to for: busan, dasyprocta, auditum, including, mjw, of, if, without,
Average loss at step  620000 :  4.20978682024
Average loss at step  640000 :  4.20852332526
Average loss at step  660000 :  4.19281596824
Average loss at step  680000 :  4.15824690018
Average loss at step  700000 :  4.18065965886
Average loss at step  720000 :  4.18245750785
Average loss at step  740000 :  4.0688151175
Average loss at step  760000 :  4.17679225191
Average loss at step  780000 :  4.13362353218
Average loss at step  800000 :  4.16932820062
Nearest to from: through, in, into, via, rajons, during, across, mjw,
Nearest to into: through, within, across, from, onto, tempest, away, davidic,
Nearest to been: be, become, come, already, remained, recently, was, were,
Nearest to and: or, but, however, including, rajonas, hyi, mammuthus, safl,
Nearest to which: that, this, actually, what, typically, usually, islamists, safl,
Nearest to while: although, when, though, however, began, unlike, kinderhook, after,
Nearest to between: with, lup, within, among, across, hangs, blvd, inside,
Nearest to he: she, it, they, there, we, who, waas, mitral,
Nearest to if: when, though, although, where, since, unless, archaeopteryx, before,
Nearest to to: mjt, hyi, wct, ursus, dasyprocta, mjw, agave, rajonas,
Nearest to were: are, those, was, have, rajonas, these, incline, be,
Nearest to more: less, most, quite, very, greater, enough, larger, rather,
Nearest to people: men, women, individuals, jews, children, peoples, muslims, fans,
Nearest to six: seven, eight, four, five, nine, three, hyi, landesverband,
Nearest to when: if, while, although, before, where, though, after, however,
Nearest to for: dasyprocta, rajons, wct, rajonas, after, mjw, in, busan,
Average loss at step  820000 :  4.13990458595
Average loss at step  840000 :  4.14843492368
Average loss at step  860000 :  4.16720801257
Average loss at step  880000 :  4.11716017977
Average loss at step  900000 :  4.16926217705
Average loss at step  920000 :  4.14734866341
Average loss at step  940000 :  4.12868653712
Average loss at step  960000 :  4.14620680249
Average loss at step  980000 :  4.13878755189
Average loss at step  1000000 :  4.04580348073
Nearest to from: through, via, rajons, during, across, besides, into, enhances,
Nearest to into: through, within, across, onto, tempest, from, davidic, affectionate,
Nearest to been: be, become, come, already, were, remained, was, recently,
Nearest to and: but, rajonas, or, agave, thibetanus, dasyprocta, hyi, landesverband,
Nearest to which: that, this, what, typically, actually, who, safl, but,
Nearest to while: although, when, though, however, whereas, kinderhook, unlike, whilst,
Nearest to between: lup, among, within, with, across, inside, blvd, marle,
Nearest to he: she, it, they, there, we, who, mitral, whitlam,
Nearest to if: when, though, unless, where, since, before, although, because,
Nearest to to: hyi, mjt, mjw, dasyprocta, ursus, audita, wct, rajonas,
Nearest to were: are, was, those, these, have, be, incline, rajonas,
Nearest to more: less, quite, most, very, greater, worse, larger, enough,
Nearest to people: men, women, individuals, jews, children, peoples, muslims, residents,
Nearest to six: seven, eight, four, five, nine, three, zero, two,
Nearest to when: if, while, although, where, before, after, unless, though,
Nearest to for: dasyprocta, rajons, busan, rajonas, wct, audita, mjw, abandons,
Average loss at step  1020000 :  4.10359834481
Average loss at step  1040000 :  4.10447594263
Average loss at step  1060000 :  4.10583353707
Average loss at step  1080000 :  4.1073725705
Average loss at step  1100000 :  4.12490047302
Average loss at step  1120000 :  4.1265602081
Average loss at step  1140000 :  4.09083834941
Average loss at step  1160000 :  4.15155592268
Average loss at step  1180000 :  4.10331000839
Average loss at step  1200000 :  4.09355439812
Nearest to from: through, via, besides, rajons, during, in, across, rajonas,
Nearest to into: through, within, onto, across, tempest, from, davidic, satisfies,
Nearest to been: be, become, already, remained, come, historically, were, previously,
Nearest to and: but, or, however, nor, rajonas, thibetanus, including, although,
Nearest to which: that, this, actually, typically, what, safl, who, islamists,
Nearest to while: although, when, though, whilst, whereas, however, unlike, kinderhook,
Nearest to between: with, lup, within, among, across, from, inside, over,
Nearest to he: she, it, they, there, we, waas, who, mitral,
Nearest to if: when, though, unless, where, since, although, hence, because,
Nearest to to: mjt, rajonas, hyi, dasyprocta, mjw, wct, landesverband, audita,
Nearest to were: are, was, have, those, these, rajonas, incline, pontificia,
Nearest to more: less, quite, most, very, worse, larger, enough, greater,
Nearest to people: men, women, individuals, jews, americans, peoples, muslims, citizens,
Nearest to six: seven, five, four, eight, three, nine, two, hyi,
Nearest to when: if, while, although, where, though, after, unless, before,
Nearest to for: dasyprocta, rajons, rajonas, busan, hbox, mjw, audita, hyi,
Average loss at step  1220000 :  4.084660001
Average loss at step  1240000 :  4.10924338448
Average loss at step  1260000 :  4.10428494163
Average loss at step  1280000 :  3.97969400813
Average loss at step  1300000 :  4.09817235332
Average loss at step  1320000 :  4.03226239958
Average loss at step  1340000 :  4.09316765916
Average loss at step  1360000 :  4.07587454007
Average loss at step  1380000 :  4.08070993
Average loss at step  1400000 :  4.06950101672
Nearest to from: via, through, rajons, besides, across, during, safl, mjw,
Nearest to into: through, onto, within, across, davidic, tempest, from, beyond,
Nearest to been: be, become, already, come, was, were, historically, previously,
Nearest to and: or, but, nor, whereas, rajonas, kinderhook, including, while,
Nearest to which: that, this, actually, typically, what, islamists, safl, but,
Nearest to while: although, whereas, when, though, whilst, however, including, unlike,
Nearest to between: lup, among, with, across, within, inside, shaku, over,
Nearest to he: she, it, they, there, we, mitral, who, michelob,
Nearest to if: when, though, unless, although, where, hence, because, whenever,
Nearest to to: dasyprocta, rajonas, hyi, mjt, audita, mjw, wct, wideawake,
Nearest to were: are, was, have, those, these, rajonas, incline, pontificia,
Nearest to more: less, quite, most, very, greater, larger, enough, worse,
Nearest to people: women, individuals, men, jews, fans, persons, americans, children,
Nearest to six: eight, four, seven, five, three, nine, hyi, rajonas,
Nearest to when: if, while, before, where, although, though, unless, after,
Nearest to for: busan, dasyprocta, in, rajons, rajonas, audita, of, after,
Average loss at step  1420000 :  4.09283684272
Average loss at step  1440000 :  4.09022417452
Average loss at step  1460000 :  4.0639601844
Average loss at step  1480000 :  4.03760708206
Average loss at step  1500000 :  4.09348774075
Average loss at step  1520000 :  4.07851810901
Average loss at step  1540000 :  3.9597240613
Average loss at step  1560000 :  4.07499863696
Average loss at step  1580000 :  4.01649859191
Average loss at step  1600000 :  4.07773584781
Nearest to from: via, through, besides, across, into, during, in, rajons,
Nearest to into: onto, through, within, across, from, tempest, beyond, davidic,
Nearest to been: be, become, already, come, fallen, grown, recently, previously,
Nearest to and: but, rajonas, or, hyi, whereas, kinderhook, safl, although,
Nearest to which: that, this, typically, actually, what, islamists, usually, rajonas,
Nearest to while: although, whilst, when, whereas, though, however, after, began,
Nearest to between: lup, among, with, across, within, inside, shaku, rajonas,
Nearest to he: she, it, they, there, we, who, rohrabacher, mitral,
Nearest to if: when, unless, though, where, although, hence, whenever, since,
Nearest to to: dasyprocta, hyi, mjt, wct, rajonas, mjw, audita, landesverband,
Nearest to were: are, was, those, have, these, incline, rajonas, remain,
Nearest to more: less, quite, most, very, worse, fewer, greater, heavier,
Nearest to people: individuals, women, men, jews, persons, peoples, citizens, children,
Nearest to six: five, four, eight, seven, three, nine, zero, two,
Nearest to when: if, although, while, where, though, before, after, unless,
Nearest to for: dasyprocta, rajonas, if, rajons, hbox, audita, busan, despite,
Average loss at step  1620000 :  4.04256130984
Average loss at step  1640000 :  4.06324257388
Average loss at step  1660000 :  4.04869626679
Average loss at step  1680000 :  4.05765993543
Average loss at step  1700000 :  4.08312831153
Average loss at step  1720000 :  4.05981061043
Average loss at step  1740000 :  4.02303984957
Average loss at step  1760000 :  4.04922945347
Average loss at step  1780000 :  4.0568610615
Average loss at step  1800000 :  3.94806807138
Nearest to from: via, through, besides, rajons, during, across, into, in,
Nearest to into: through, onto, within, across, tempest, from, davidic, beyond,
Nearest to been: be, become, already, fallen, come, were, grown, undergone,
Nearest to and: but, or, rajonas, thibetanus, hyi, whereas, landesverband, one,
Nearest to which: this, that, what, actually, safl, typically, instead, islamists,
Nearest to while: although, whilst, whereas, when, though, despite, however, after,
Nearest to between: lup, across, among, within, with, inside, shaku, tulare,
Nearest to he: she, they, it, there, we, who, whitlam, michelob,
Nearest to if: when, unless, though, although, where, whenever, because, hence,
Nearest to to: hyi, rajonas, dasyprocta, mjt, mjw, wct, landesverband, microcebus,
Nearest to were: are, was, those, these, incline, have, be, rajonas,
Nearest to more: less, most, quite, very, worse, fewer, larger, heavier,
Nearest to people: individuals, women, men, jews, persons, citizens, residents, tourists,
Nearest to six: seven, four, eight, five, nine, three, zero, hyi,
Nearest to when: if, while, although, unless, where, before, after, though,
Nearest to for: after, busan, alongside, dasyprocta, rajons, during, despite, rajonas,
Average loss at step  1820000 :  4.0366222593
Average loss at step  1840000 :  4.01050049885
Average loss at step  1860000 :  4.04104860071
Average loss at step  1880000 :  4.02129215995
Average loss at step  1900000 :  4.03252021322
Average loss at step  1920000 :  4.07884682775
Average loss at step  1940000 :  4.00356332458
Average loss at step  1960000 :  4.07981533021
Average loss at step  1980000 :  4.03165701723
Average loss at step  2000000 :  4.02683587987
Nearest to from: via, besides, through, rajons, in, safl, during, handicap,
Nearest to into: onto, through, within, across, davidic, tempest, from, beyond,
Nearest to been: be, become, already, grown, fallen, come, historically, undergone,
Nearest to and: but, nor, or, rajonas, however, dasyprocta, UNK, hyi,
Nearest to which: that, this, typically, what, actually, but, safl, islamists,
Nearest to while: although, whilst, whereas, when, though, however, despite, including,
Nearest to between: lup, among, with, within, across, albertine, shaku, over,
Nearest to he: she, they, it, there, we, rohrabacher, who, whitlam,
Nearest to if: when, unless, though, where, hence, whenever, although, since,
Nearest to to: rajonas, dasyprocta, audita, mjt, hyi, landesverband, mjw, for,
Nearest to were: are, those, these, incline, was, have, rajonas, remain,
Nearest to more: less, quite, fewer, most, worse, very, stronger, larger,
Nearest to people: individuals, men, women, citizens, americans, jews, tourists, residents,
Nearest to six: seven, five, eight, four, three, hyi, nine, rajonas,
Nearest to when: if, while, where, although, unless, though, after, before,
Nearest to for: rajonas, dasyprocta, rajons, if, busan, consider, to, after,

In [7]:

    

# Step 6: Visualize the embeddings.


def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):
  assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'
  plt.figure(figsize=(18, 18))  # in inches
  for i, label in enumerate(labels):
    x, y = low_dim_embs[i, :]
    plt.scatter(x, y)
    plt.annotate(label,
                 xy=(x, y),
                 xytext=(5, 2),
                 textcoords='offset points',
                 ha='right',
                 va='bottom')

  plt.savefig(filename)

try:
  # pylint: disable=g-import-not-at-top
  from sklearn.manifold import TSNE
  import matplotlib.pyplot as plt

  tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
  plot_only = 500
  low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
  labels = [reverse_dictionary[i] for i in range(plot_only)]
  plot_with_labels(low_dim_embs, labels)

except ImportError:
  print('Please install sklearn, matplotlib, and scipy to show embeddings.')