In [1]:

    

import tensorflow as tf


    

In [2]:

    

file = open('../../text/text8', 'r')


    

In [3]:

    

text = file.read()


    

In [4]:

    

words = text.split(' ')


    

In [5]:

    

len(words)


    

    
Out[5]:





17005208

In [7]:

    

u_words = get_unique_words(words[0:500])


    

In [6]:

    

def get_unique_words(words):
    unique_words = []
    for word in words:
        if word not in unique_words:
            unique_words.append(word)
            
    return unique_words


    

In [9]:

    

u_words


    

    
Out[9]:





['',
 'anarchism',
 'originated',
 'as',
 'a',
 'term',
 'of',
 'abuse',
 'first',
 'used',
 'against',
 'early',
 'working',
 'class',
 'radicals',
 'including',
 'the',
 'diggers',
 'english',
 'revolution',
 'and',
 'sans',
 'culottes',
 'french',
 'whilst',
 'is',
 'still',
 'in',
 'pejorative',
 'way',
 'to',
 'describe',
 'any',
 'act',
 'that',
 'violent',
 'means',
 'destroy',
 'organization',
 'society',
 'it',
 'has',
 'also',
 'been',
 'taken',
 'up',
 'positive',
 'label',
 'by',
 'self',
 'defined',
 'anarchists',
 'word',
 'derived',
 'from',
 'greek',
 'without',
 'archons',
 'ruler',
 'chief',
 'king',
 'political',
 'philosophy',
 'belief',
 'rulers',
 'are',
 'unnecessary',
 'should',
 'be',
 'abolished',
 'although',
 'there',
 'differing',
 'interpretations',
 'what',
 'this',
 'refers',
 'related',
 'social',
 'movements',
 'advocate',
 'elimination',
 'authoritarian',
 'institutions',
 'particularly',
 'state',
 'anarchy',
 'most',
 'use',
 'does',
 'not',
 'imply',
 'chaos',
 'nihilism',
 'or',
 'anomie',
 'but',
 'rather',
 'harmonious',
 'anti',
 'place',
 'regarded',
 'structures',
 'coercive',
 'economic',
 'relations',
 'based',
 'upon',
 'voluntary',
 'association',
 'autonomous',
 'individuals',
 'mutual',
 'aid',
 'governance',
 'while',
 'easily',
 'offer',
 'visions',
 'they',
 'believe',
 'truly',
 'free',
 'however',
 'ideas',
 'about',
 'how',
 'an',
 'anarchist',
 'might',
 'work',
 'vary',
 'considerably',
 'especially',
 'with',
 'respect',
 'economics',
 'disagreement',
 'brought',
 'origins',
 'predecessors',
 'kropotkin',
 'others',
 'argue',
 'before',
 'recorded',
 'history',
 'human',
 'was',
 'organized',
 'on',
 'principles',
 'anthropologists',
 'follow',
 'engels',
 'believing',
 'hunter',
 'gatherer',
 'bands',
 'were',
 'egalitarian',
 'lacked',
 'division',
 'labour',
 'accumulated',
 'wealth',
 'decreed',
 'law',
 'had',
 'equal',
 'access',
 'resources',
 'william',
 'godwin',
 'organisation',
 'rothbard',
 'find',
 'attitudes',
 'taoism',
 'ancient',
 'china',
 'found',
 'similar',
 'stoic',
 'zeno',
 'citium',
 'according',
 'repudiated',
 'omnipotence',
 'its',
 'intervention',
 'regimentation',
 'proclaimed',
 'sovereignty',
 'moral',
 'individual',
 'anabaptists',
 'one',
 'six',
 'th',
 'century',
 'europe',
 'sometimes',
 'considered',
 'religious',
 'forerunners',
 'modern',
 'bertrand',
 'russell',
 'his',
 'western',
 'writes',
 'all',
 'since',
 'held',
 'good',
 'man',
 'will',
 'guided',
 'at',
 'every',
 'moment',
 'holy',
 'spirit',
 'premise',
 'arrive',
 'communism',
 'true',
 'levellers',
 'communistic',
 'movement',
 'during',
 'time',
 'civil',
 'war',
 'some',
 'era',
 'mean',
 'something',
 'other',
 'than',
 'louis',
 'armand',
 'baron',
 'de',
 'lahontan',
 'nouveaux',
 'voyages',
 'dans',
 'l',
 'am',
 'rique',
 'septentrionale',
 'seven',
 'zero',
 'three',
 'where',
 'he',
 'described',
 'indigenous',
 'american',
 'which',
 'no',
 'laws',
 'prisons',
 'priests',
 'private',
 'property',
 'being',
 'libertarian',
 'leader',
 'indian',
 'repeatedly',
 'stated',
 'so',
 'ancestors',
 'nine']

In [ ]: