In [1]:
Compare word frequencies to log-uniform (sampling model for nce Candidate Sampling)

Patrick Coady (

from wordvector import WordVector
from windowmodel import WindowModel
import docload
import numpy as np
from sklearn.utils import shuffle
import matplotlib.pyplot as plt

In [2]:
files = ['../data/adventures_of_sherlock_holmes.txt',

word_counter, word_list, num_lines, num_words = docload.load_books(
    files, gutenberg=True)

print('Document loaded and processed: {} lines, {} words.'
      .format(num_lines, num_words))

dictionary = docload.build_dict(word_counter)
reverse_dictionary = {v: k for k, v in dictionary.items()}

Document loaded and processed: 24080 lines, 244986 words.

In [3]:
top_words = word_counter.most_common(50000)
top_words.sort(key=lambda t: -t[1])
counts = np.array([t[1] for t in top_words])
counts = counts/sum(counts)  # actual work probability
x = np.array(range(len(counts)))
log_uniform = (np.log(x+2) - np.log(x+1))/np.log(len(counts)+1) # ideal log-uniform

In [6]:
plt.plot(x, counts)
plt.plot(x, log_uniform)

# reasonable match, but definitely under-samples first 200 words

In [ ]: