In [8]:

    

""" Imports """
import re
from nltk.tokenize import word_tokenize, sent_tokenize
import collections
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

"""Global definitons"""
_start = 'S_START'
_end = 'S_END'


    

In [9]:

    

""" util definitions"""

def hyperbolic(net):
    return np.tanh(net)

def relu(net):
    return np.maximum(0,net)

def softmax(net):
    _exp = np.exp(net)
    return _exp/np.sum(_exp)

def predict(scores):
    return np.argmax(scores)


    

In [10]:

    

class WordItem:
    def __init__(self,word,count=0):
        self.word = word
        self.count = count


    

In [25]:

    

class RNNlayer:
    
    """ 
    RNN nodes for decoder
    
    hidden state at time step t of decoder is conditioned on hidden state at time step t-1,
    output at time step t-1 and input at time step t
    """
    
    def __init__(self, inputSize, outputSize, W_Embedding_french, idx, bptt_truncate = 5, hiddenDim = 10):
        """
        inputSize = dimensions of the input embedding 
        outputSize = vocabulary size
        hiddenDim = size of the hidden unit in RNN
        bptt_truncate = truncate the number of time steps we calculate the gradient during backpropagation
        """
        self.inputSize = inputSize
        self.outputSize = outputSize
        self.W_Embedding_french = W_Embedding_french
        self.hiddenDim = hiddenDim
        self. bptt_truncate = bptt_truncate
        self.idx = idx;
        
        self.w_in = np.random.uniform(-np.sqrt(1./inputSize), np.sqrt(1./inputSize),(hiddenDim, inputSize))
        self.w_hh = np.random.uniform(-np.sqrt(1./hiddenDim), np.sqrt(1./hiddenDim),(hiddenDim, hiddenDim))
        self.w_outH = np.random.uniform(-np.sqrt(1./hiddenDim), np.sqrt(1./hiddenDim),(inputSize, hiddenDim))
        self.w_out = np.random.uniform(-np.sqrt(1./hiddenDim), np.sqrt(1./hiddenDim),(outputSize, hiddenDim))
        
    def forwardProp(self, inSentence, expSent):
        """
        inSentence: word indices in input language vocabulary
        expSent: word indices in target language vocabulary
        """
        
        #Total number of time steps equal to number of words in the sentence
        T = len(inSentence)
        
        #Saving all hidden states and outputs during forward propagation
        _h = np.zeros((T,self.hiddenDim))
        _o = np.zeros((T,self.outputSize))
        
        #Initializing initial output as the start token
        _o[-1] = np.zeros(self.outputSize)
        _o[-1][idx] = 1;
        
        #For each time step calculating hidden state and output
        for t in np.arange(T):
            outIdx = predict(_o[t-1])
            _h[t] = hyperbolic(self.w_in.dot(inSentence[t]) + self.w_hh.dot(_h[t-1]) + self.w_outH.dot(self.W_Embedding_french[outIdx]))
            _o[t] = softmax(self.w_out.dot(_h[t]))
            
        return _o, _h
    
    def calculateLoss(self, inSentence, expSentence):
        
        #For each sentence
        o, h = self.forwardProp(inSentence, expSentence)
        #TODO recheck this part
        correctPred = o[np.arange(len(expSentence)), expSentence]
        #Loss for each sentence
        l = -1 * np.sum(np.log(correctPred))
        return l
    
    def calculateTotalLoss(self, inSentence, expSentences):
        
        L = 0.0
        for i in range(len(inSentence)):
            if len(inSentence[i]) == len(expSentences[i]) :
                L += self.calculateLoss(inSentence[i], expSentences[i])
            
        return L
    
    def backPropTT(self, inSentence, expSentence):
        
        # Total number of time steps equal to number of words in the sentence
        T = len(expSentence)
        
        # Performing forward propagation
        o, h = self.forwardProp(inSentence, expSentence)
        
        # Defining gradient variables
        dLdin = np.zeros(self.w_in.shape)
        dLdhh = np.zeros(self.w_hh.shape)
        dLdoutH = np.zeros(self.w_outH.shape)
        dLdout = np.zeros(self.w_out.shape)
        
        # Calculating the difference between output and actual output
        delta_o = o
        delta_o[np.arange(T), expSentence] -= 1
        #print 'delta_o', delta_o
        
        # Calculating gradients backwards through time
        for t in np.arange(T)[::-1]:
            #Output gradient is only dependent on time step t
            dLdout += np.outer(delta_o[t], h[t])
            
            # Initial delta calculation propagating gradients from output
            delta_t = self.w_out.T.dot(delta_o[t]) * (1 - (h[t] ** 2))
            
            # Backpropagation through time (for at most self.bptt_truncate steps)
            for bptt_step in np.arange(max(0, t-self.bptt_truncate), t+1)[::-1]:
                # print "Backpropagation step t=%d bptt step=%d " % (t, bptt_step)
                # Add to gradients at each previous step
                dLdhh += np.outer(delta_t, h[bptt_step-1])              
                dLdin += np.outer(delta_t, inSentence[bptt_step-1])
                dLdoutH += np.outer(delta_t, self.W_Embedding_french[predict(o[t-1])])
                # Update delta for next step dL/dz at t-1
                delta_t = self.w_hh.T.dot(delta_t) * (1 - h[bptt_step-1] ** 2)
            """TODO review backprop implementation"""
            
        return dLdin, dLdhh, dLdoutH, dLdout
        #return dLdin, dLdhh, dLdout
    
    def sgd_step(self, inSentence, expSentence, learningRate):
        
        """ Performs a single stochastic gradient step"""
        
        # Calculating gradients
        dLdin, dLdhh, dLdoutH, dLdout = self.backPropTT(inSentence, expSentence)
        #dLdin, dLdhh, dLdout = self.backPropTT(inSentence, expSentence)
        
        # Updating parameters
        self.w_in -= learningRate * dLdin
        self.w_hh -= learningRate * dLdhh
        self.w_outH -= learningRate * dLdoutH
        self.w_out -= learningRate * dLdout
        
    def train_Decoder_With_SGD(self, X_train, Y_train, learningRate = 0.05, nepochs = 200):
        """TODO evaluate losses and update learning rate if required"""
        loss = 100000000000
        for epoch in range(nepochs):
            for i in range(len(Y_train)):
                if len(X_train[i]) == len(Y_train[i]) :
                    self.sgd_step(X_train[i], Y_train[i], learningRate)
            newLoss = self.calculateTotalLoss(X_train, Y_train)
            print epoch, " ", newLoss
            #if newLoss > loss :
            #    break
            #loss = newLoss


    

In [16]:

    

""" Word preprocessing """
def dataset(_fi='/home/jazzycrazzy/PythonScripts/dataset.csv', _fo = 'testfile.txt'):
    file_in = open(_fi)
    #file_out = open(_fo,'wb')

    words = [] #stores unique words encountered in the document as WordItem objects
    _dict = {} #temporary dictionary to maintain count of each word
    
    _dict['UNK'] = 0

    for l in file_in:
        #file_out.write(l+'\n')
        l = _start+' '+l+' '+_end
        split = word_tokenize(l.decode('utf-8'))
        for w in split:
            if len(w)==0:
                continue
            elif len(w) > 15: #if word's length is greater than 15 counting it as unknown
                _dict['UNK'] += 1
                continue
            if w not in _dict:
                _dict[w] = 1
            _dict[w] += 1
            
    _vocab = {} #dictionary with words as keys and values as indices of them in 'word' list
    _vocab['UNK'] = len(words)
    words.append(WordItem('UNK',_dict['UNK']))
    for k,v in _dict.iteritems():
        if v > 9 and k != 'UNK':
        #if k != 'UNK':
            _vocab[k] = len(words)
            words.append(WordItem(k,v))
        else:
            words[0].count += 1
    
    #cleaning up unnecessary memory
    del _dict
    file_in.close()
    #file_out.close()
    
    return _vocab, words

def UnigramTable(_vocab, words):
    """ Calculates probabilities based on count of each word present"""
    pow = 0.75
    totalFreqPow = 0.0
    unigramTable = {}
    
    l = [words[i].count**pow for i in range(len(_vocab))]
    totalFreqPow = np.sum(l)
    
    for i in range(len(_vocab)):
        unigramTable[i] = (words[i].count**pow)/totalFreqPow
    
    del l
    return unigramTable

def hotVector(wordIndex,vocabSize):
    """ Returns hot vector representation of a word """
    hVector = np.zeros(vocabSize)
    hVector[wordIndex-1] = 1
    return hVector

def softmax(net):
    """ calculates softmax score - target score normalized with noise scores and calculated as probability"""
    _exp = np.exp(net)
    return _exp/np.sum(_exp)

def sigmoid(net):
    """ Applies sigmoid logistic function on net """
    return 1.0/(1+np.exp(-net))

def randomIdx(k, vocabSize, current):
    """ Returns k indices from with unigram table randomly with respect to each word's probablity """
    global _unigramTable1
    idxs = list(np.random.choice(vocabSize, k+1, False, p = _unigramTable1.values()))
    if current in idxs:
        idxs.remove(current)
    else:
        del idxs[-1]
    return idxs
    
def softmaxCostGradient(net, target):
    prob = softmax(net)
    print(prob)
    
    
def negSamplingCostGradient(out, context, emb, vocabSize, learningRate, W_Output, k = 10):
    
    errorHidden = np.zeros(shape=(emb.size,1))
    
    actOut = sigmoid(out[context])
    negSamples = randomIdx(k, vocabSize, context)
    _negSamples = [-out[sample] for sample in negSamples]
    
    # error for context word
    e = -np.log(actOut) - np.sum(np.log(sigmoid(np.array(_negSamples))))
    
    """ calculating gradients for output vectors for both target and negative samples
    calculating hidden layer error for each context word """
    # Updating output weight vector for context word
    delta = actOut - 1
    errorHidden += delta * W_Output[:,context:context+1]
    W_Output[:,context:context+1] -= learningRate * np.reshape(delta * emb,(emb.size,1))
    
    # Updating output weight vectors for negative sampling
    for sample in negSamples:
        delta = sigmoid(out[sample])
        errorHidden += delta * W_Output[:,sample:sample+1]
        W_Output[:,sample:sample+1] -= learningRate * np.reshape(delta * emb,(emb.size,1))
    
    return errorHidden,e    
    
def skipgram(target,contextWords, vocabSize, learningRate, W_Embedding, W_Output):
    
    """
    will be called on each window with
    target: Target word index
    contextWords: Arrray of integers representing context words
    """
    loss = 0
    k = 10 #Number of negative samples
    emb = W_Embedding[target]
    out = np.matmul(emb,W_Output) # [1 x EmbSize].[EmbSize x VocabSize]
    #print out.shape
    _predicted = []
    EH = np.zeros(shape=(emb.size,1))
    for context in contextWords:
        #predicted = hotVector(context, vocabSize)
        #softmaxCostGradient(out,context)
        _EH,_e = negSamplingCostGradient(out, context, emb, vocabSize, learningRate, W_Output, k)
        EH += _EH
        loss += _e
        #EH += sof
        
    #updating hidden layer input vector embedding
    W_Embedding[target] -= learningRate * EH.T[0]
    return loss


    

In [14]:

    

""" Creates word embeddings in vector space representation """

""" Feedforward Neural Net Language model """
#Input layer

#Projection layer

#Hidden layer

#Output layer

#Initialization
fin='/Users/preethikapachaiyappa/Documents/MachineLearning/Data/English-equalLength1.txt'#/home/jazzycrazzy/PythonScripts/dataset.csv'
fin1='/Users/preethikapachaiyappa/Documents/MachineLearning/Data/French-equalLength1.txt'
fout = 'testfile.txt'
fout1 = 'testfile1.txt'
_vocab, words = dataset(fin, fout)
_vocab_f, words_f = dataset(fin1, fout1)
_unigramTable = UnigramTable(_vocab, words)
_unigramTable1 = UnigramTable(_vocab_f, words_f)

learningRate = 0.1
vocabSize = len(words)
vocabSize_f = len(words_f)
emb_size = 10
win_size = 2
target = None
contextWords = []
epoch = 20

print _vocab
print _vocab_f


# No need of hidden layer since when the embedding matrix is multiplied with hot vector 
#it essentially gives that embedding row
W_Embedding = np.random.randn(vocabSize,emb_size) #Embedding matrix
W_Output = np.random.randn(emb_size,vocabSize) #Outputlayer weight matrix Emb_size x Vocab

W_Embedding_f = np.random.randn(vocabSize_f,emb_size) #Embedding matrix
W_Output_f = np.random.randn(emb_size,vocabSize_f) #Outputlayer weight matrix Emb_size x Vocab

for _ in np.arange(epoch):
    
    totalLoss = 0
    loss = 0
    
    fileIn = open(fin)
    for l in fileIn:
        l = _start+' '+l+' '+_end
        tokens = word_tokenize(l.decode('utf-8'))
        #print 'tokens',tokens
        for token in tokens:
            
            loss = 0
            contextWords = []
            
            if token in _vocab:
                target = _vocab[token]
                trgtIdx = tokens.index(token)
                cntxtIdxs = range(trgtIdx-win_size, trgtIdx+win_size+1)
                cntxtIdxs.remove(trgtIdx)
                for idx in cntxtIdxs:
                    #check for first word and last word and use UNK for context words for window where words not available
                    if idx >-1 and idx < len(tokens) and tokens[idx] in _vocab:
                        contextWords = np.append(contextWords, _vocab[tokens[idx]])
                    else:
                        contextWords = np.append(contextWords, _vocab['UNK'])
                loss += skipgram(target, contextWords, vocabSize, learningRate, W_Embedding, W_Output)
        totalLoss += loss
    print 'Total Loss:',totalLoss


    

    




{u'limited': 1154, u'four': 1, u'whose': 767, u'crises': 1152, u'poorest': 1155, u'Western': 3, u'under': 1156, u'risk': 1157, u'Iranian': 394, u'regional': 4, u'every': 1159, u'reforms': 768, u'affect': 769, u'bringing': 28, u'1930\u2019s': 5, u'school': 1160, u'companies': 770, u'solution': 771, u'Paul': 6, u'Europeans': 1161, u'force': 772, u'leaders': 1162, u'direct': 1163, u'budget': 264, u'second': 7, u'even': 774, u'established': 813, u'decisions': 1358, u'contributed': 8, u'above': 396, u'Yeltsin': 819, u'new': 775, u'increasing': 53, u'ever': 776, u'told': 397, u'incentives': 134, u'men': 56, u'here': 9, u'hundreds': 828, u'protection': 398, u'active': 779, u'path': 833, u'100': 780, u'study': 399, u'changed': 1164, u'Hamas': 10, u'credit': 782, u'military': 11, u'changes': 1165, u'secure': 400, u'campaign': 784, u'Rather': 785, u'highly': 401, u'brought': 13, u'total': 402, u'would': 1166, u'army': 786, u'China\u2019s': 403, u'voters': 645, u'negative': 404, u'call': 787, u'asset': 1167, u'until': 15, u'successful': 16, u'hold': 17, u'must': 1168, u'me': 1169, u'join': 1170, u'rights': 788, u'pursue': 18, u'work': 406, u'era': 407, u'my': 876, u'example': 19, u'give': 789, u'currency': 20, u'want': 21, u'keep': 708, u'end': 1172, u'recovery': 408, u'thing': 901, u'provide': 409, u'damage': 653, u'far': 1363, u'Muslims': 1173, u'answer': 790, u'widespread': 1174, u'A': 23, u'rise': 1158, u'after': 410, u'wrong': 24, u'president': 791, u'law': 411, u'All': 792, u'effective': 25, u'Japanese': 1175, u'maintain': 794, u'suggest': 1219, u'democratic': 26, u'order': 413, u'operations': 795, u'Brazil': 137, u'office': 414, u'over': 1176, u'before': 796, u'His': 27, u'personal': 797, u'expectations': 1177, u'better': 799, u'production': 416, u'weeks': 800, u'easier': 29, u'Spain': 1178, u'then': 417, u'them': 418, u'affected': 419, u'diseases': 1179, u'break': 420, u'effects': 30, u'they': 421, u'schools': 172, u'bank': 422, u'structural': 31, u'represents': 32, u'Indeed': 802, u'India': 803, u'--': 573, u'victory': 1181, u'each': 1182, u'side': 804, u'mean': 180, u'financial': 33, u'principles': 805, u'currencies': 1043, u'crucial': 806, u'encourage': 34, u'newly': 1183, u'independence': 1184, u'University': 35, u'given': 1285, u'free': 1185, u'North': 1485, u'wanted': 1187, u'enormous': 36, u'created': 423, u'September': 424, u'days': 1188, u'choices': 1507, u'oppose': 425, u'industrial': 1091, u'Palestinian': 426, u'primary': 1189, u'adopted': 366, u'another': 427, u'ECB': 807, u'Congress': 1191, u'top': 1192, u'needed': 37, u'rates': 1009, u'too': 1193, u'recent': 269, u'collapse': 39, u'serve': 1194, u'took': 1195, u'target': 428, u'likely': 773, u'nations': 41, u'matter': 808, u'historical': 429, u'powers': 430, u'modern': 809, u'Instead': 431, u'seem': 811, u'seek': 812, u'relatively': 1196, u'forced': 432, u'strength': 433, u'climate': 447, u'IMF': 1287, u'-': 1048, u'Minister': 42, u'They': 43, u'Bank': 44, u'though': 1197, u'Latin': 574, u'treaty': 1198, u'points': 814, u'principle': 815, u'Copenhagen': 972, u'consumer': 816, u'came': 45, u'saying': 46, u'Union': 48, u'solutions': 1199, u'European': 436, u'lessons': 49, u'just': 853, u'rich': 50, u'do': 437, u'exports': 817, u'government\u2019s': 818, u'Nevertheless': 51, u'de': 438, u'stop': 820, u'despite': 1200, u'report': 1201, u'Organization': 821, u'Palestinians': 822, u'Soviet': 440, u'societies': 52, u'countries': 1202, u'Washington': 823, u'bad': 824, u'depends': 441, u'decided': 825, u'result': 54, u'best': 55, u'said': 826, u'capacity': 1115, u'away': 442, u'unable': 443, u'cooperation': 444, u'approach': 1203, u'we': 445, u'never': 777, u'Central': 827, u'nature': 57, u'weak': 1204, u'however': 1205, u'extent': 58, u'countries\u2019': 364, u'debt': 59, u'improve': 1207, u'faced': 1208, u'protect': 1209, u'country': 60, u'uncertainty': 829, u'against': 830, u'players': 1210, u'argue': 61, u'asked': 62, u'and': 63, u'Given': 831, u'initiative': 832, u'Ireland': 448, u'trust': 1211, u'speak': 449, u'conference': 1212, u'basis': 834, u'union': 64, u'three': 835, u'been': 1213, u'.': 65, u'much': 66, u'interest': 836, u'basic': 837, u'expected': 1215, u'life': 67, u'families': 450, u'Poland': 68, u'child': 69, u'worked': 70, u'160': 838, u'Secretary': 1216, u'has': 76, u'presidency': 71, u'economies': 839, u'air': 452, u'aim': 453, u'aid': 454, u'balance': 840, u'launched': 455, u'mistake': 456, u'economics': 781, u'Both': 1410, u'is': 841, u'it': 842, u'Bush': 1217, u'experts': 1218, u'in': 843, u'if': 844, u'things': 72, u'make': 845, u'Kremlin': 1220, u'democracies': 73, u'complex': 1221, u'potentially': 846, u'Of': 276, u'President': 847, u'several': 1222, u'independent': 457, u'Similarly': 370, u'hand': 458, u'opportunity': 848, u'cycle': 1223, u'programs': 849, u'S_START': 459, u'savings': 1224, u'greatest': 1225, u'claims': 850, u'the': 460, u'investments': 851, u'left': 852, u'proposed': 1226, u'unemployment': 1227, u'human': 854, u'yet': 855, u'previous': 74, u'candidate': 856, u'had': 75, u'board': 461, u'easy': 451, u'save': 857, u'possible': 77, u'cultural': 1229, u'advanced': 1230, u'desire': 78, u'50': 1231, u'steps': 79, u'security': 462, u'right': 80, u'old': 81, u'deal': 858, u'people': 82, u'successfully': 1232, u'born': 1242, u'election': 1233, u'extended': 1000, u'short-term': 859, u'for': 83, u'everything': 1234, u'individuals': 278, u'He': 1235, u'core': 1236, u'confrontation': 860, u'manufacturing': 85, u'First': 86, u'attacks': 1237, u'dollars': 87, u'months': 464, u'costs': 169, u'ensure': 1238, u'efforts': 1272, u'raised': 498, u'civil': 1239, u'down': 861, u'Korean': 90, u'reducing': 91, u'rely': 1240, u'support': 92, u'initial': 862, u'sovereign': 465, u'way': 466, u'was': 467, u'war': 468, u'therefore': 14, u'form': 863, u'offer': 93, u'becoming': 469, u'differences': 1241, u'failure': 864, u'true': 470, u'Commission': 463, u'later': 94, u'stronger': 351, u'evidence': 472, u"''": 95, u'exist': 96, u'Gaza': 865, u'negotiations': 97, u'regime': 1316, u'no': 1243, u'whereas': 1244, u'when': 1245, u'reality': 473, u'role': 98, u"'s": 99, u'models': 100, u'felt': 866, u'fell': 783, u'authorities': 867, u'billion': 868, u'Obama': 953, u'longer': 1349, u'together': 474, u'Second': 869, u'time': 101, u'push': 102, u'serious': 1246, u'neighbors': 1247, u'diplomacy': 475, u'global': 476, u'focus': 1248, u'Today': 1249, u'certainly': 477, u'depend': 870, u'environment': 1250, u'promoting': 1251, u'sovereignty': 871, u'advantage': 1252, u'choice': 103, u'governments': 872, u'perceived': 478, u'level': 1253, u'did': 873, u'standards': 1254, u'leave': 104, u'1999': 405, u'team': 105, u'George': 479, u'prevent': 106, u'revolution': 480, u'trend': 1255, u'dealing': 874, u'cost': 1399, u'appear': 481, u'assistance': 482, u'current': 107, u'shared': 483, u'falling': 108, u'boost': 1443, u'Libya': 484, u'water': 1257, u'twentieth': 1258, u'groups': 1259, u'address': 109, u'alone': 110, u'along': 111, u'appears': 485, u'change': 486, u'Portugal': 917, u'shift': 644, u'India\u2019s': 281, u'usually': 487, u'lending': 488, u'When': 878, u'crisis': 1260, u'market': 489, u'troops': 663, u'working': 112, u'positive': 664, u'by': 282, u'live': 490, u'today': 1261, u'``': 114, u'afford': 115, u'cases': 1264, u'effort': 880, u'organizations': 491, u'German': 1265, u'Lebanon': 1267, u'values': 117, u'hope': 358, u'growing': 881, u'making': 882, u'claim': 883, u'citizens': 88, u'Federal': 119, u'subsidies': 493, u'allowed': 120, u'Putin': 884, u'requirements': 1268, u'divided': 121, u'bonds': 359, u'1': 1269, u'vital': 1270, u'Why': 122, u'economy': 494, u'information': 1271, u'may': 885, u'use': 1467, u'fueled': 123, u'membership': 886, u'produce': 495, u'such': 124, u'data': 711, u'grow': 713, u'man': 887, u'natural': 125, u'so': 126, u'increase': 2, u'talk': 888, u'Islam': 127, u'indeed': 496, u'Hezbollah': 497, u'mainly': 1273, u'Fed': 89, u'years': 128, u'stability': 889, u'course': 129, u'cuts': 636, u'argued': 890, u'still': 499, u'group': 735, u'how': 22, u'forms': 500, u'offers': 1274, u'policy': 892, u'World': 893, u'main': 894, u'decades': 130, u'happened': 1510, u'finance': 895, u'half': 501, u'not': 502, u'now': 503, u'killed': 896, u'developments': 897, u'term': 1275, u'name': 1276, u'establishing': 131, u'significantly': 505, u'Barack': 506, u'owing': 132, u'year': 507, u'happen': 508, u'avoided': 509, u'plans': 983, u'living': 898, u'shown': 510, u'factors': 1278, u'looking': 511, u'shows': 512, u'earlier': 1339, u'theory': 1279, u'Rights': 899, u'million': 133, u'mechanisms': 900, u'possibility': 778, u'quite': 135, u'Security': 1280, u'care': 513, u'transition': 514, u'turn': 1281, u'place': 1282, u'massive': 136, u'think': 902, u'first': 903, u'Even': 515, u'saving': 138, u'There': 516, u'Asia': 517, u'one': 139, u'world\u2019s': 1283, u'long': 710, u'directly': 518, u'vote': 140, u'impossible': 519, u'open': 141, u'millions': 1284, u'city': 142, u'little': 905, u'trillion': 1286, u'households': 520, u'Such': 40, u'mostly': 521, u'that': 522, u'today\u2019s': 143, u'than': 523, u'population': 1288, u'10': 906, u'third': 793, u'require': 1289, u'diplomatic': 1290, u'future': 144, u'were': 907, u'outcome': 1292, u'prospect': 145, u'investors': 524, u'turned': 146, u'say': 147, u'\u201d': 1293, u'any': 1294, u'prosperity': 1295, u'efficient': 909, u'ideas': 1296, u'potential': 910, u'take': 148, u'performance': 911, u'concern': 531, u'begin': 525, u'sure': 1297, u'price': 526, u'especially': 912, u'average': 149, u'farmers': 1303, u'implications': 527, u'drive': 150, u'managing': 151, u'Europe\u2019s': 913, u'surplus': 152, u'show': 914, u'Obama\u2019s': 1298, u'ground': 12, u'We': 915, u'slow': 153, u'behind': 916, u'only': 529, u'going': 154, u'get': 918, u'nearly': 919, u'hedge': 1299, u'prime': 155, u'where': 156, u'vision': 157, u'relative': 920, u'elected': 530, u'encouraged': 1302, u'That': 158, u'ways': 1304, u'weapons': 1305, u'enough': 921, u'between': 532, u'across': 922, u'infrastructure': 923, u'jobs': 159, u'blame': 533, u'20': 992, u'cities': 1307, u'come': 1308, u'many': 160, u'region': 1309, u'according': 924, u'comes': 534, u'among': 925, u'cancer': 926, u'period': 1310, u'Monetary': 415, u'But': 1311, u'defined': 295, u'don\u2019t': 161, u'borders': 927, u'wants': 162, u'direction': 1312, u'minister': 1313, u'former': 163, u'those': 929, u'case': 1314, u'developing': 537, u'these': 538, u'cash': 1315, u"n't": 931, u'ongoing': 932, u'overcome': 801, u'policies': 164, u'situation': 165, u'eventually': 933, u'middle': 979, u'technology': 166, u'worry': 539, u'movements': 934, u'different': 935, u'develop': 541, u'pay': 936, u'same': 937, u'speech': 938, u'events': 1317, u'status': 1318, u'oil': 939, u'I': 168, u'II': 234, u'running': 940, u'changing': 1320, u'largely': 941, u'It': 1321, u'severe': 543, u'without': 1322, u'relief': 1323, u'In': 1324, u'model': 1325, u'If': 1326, u'United': 1327, u'being': 170, u'money': 943, u'actions': 1328, u'death': 544, u'thinking': 545, u'seems': 1329, u'treatment': 546, u'Although': 944, u'competitiveness': 173, u'real': 547, u'around': 174, u'read': 548, u'Many': 1330, u'early': 549, u'inflation': 1331, u'world': 1051, u'accepted': 1332, u'collective': 550, u'facing': 945, u'benefit': 551, u'London': 946, u'either': 947, u'fully': 600, u'Moreover': 948, u'reduced': 1333, u'West': 1334, u'Since': 1335, u'competition': 290, u'International': 1336, u'legal': 1337, u'conservative': 553, u'critical': 950, u'deficit': 554, u'provides': 1338, u'scientific': 175, u'business': 555, u'leadership': 177, u'on': 1340, u'central': 556, u'of': 1341, u'industry': 178, u'violence': 951, u'nothing': 1406, u'stand': 1342, u'act': 179, u'or': 1343, u'No': 181, u'accounts': 336, u'parties': 182, u'your': 557, u'intervention': 558, u'legacy': 1291, u'her': 183, u'Though': 952, u'assumed': 560, u'there': 1345, u'start': 561, u'low': 562, u'combined': 575, u'complete': 184, u'regard': 1347, u'Treaty': 954, u'technologies': 185, u'promote': 1348, u'with': 186, u',': 798, u'default': 563, u'emissions': 955, u'\u201cthe': 564, u'politically': 187, u'agree': 188, u'regimes': 189, u'gone': 190, u'certain': 191, u'moved': 565, u'deep': 956, u'an': 193, u'How': 194, u'as': 195, u'at': 196, u'politics': 1350, u'again': 957, u'As': 674, u'summit': 301, u'you': 566, u'More': 1386, u'At': 675, u'poor': 375, u'Great': 567, u'important': 958, u'fiscal': 568, u'bilateral': 1351, u'calls': 1352, u'invest': 1353, u'mass': 199, u'external': 536, u'all': 1354, u'consider': 200, u'caused': 201, u'lack': 1355, u'dollar': 960, u'month': 569, u'welfare': 202, u'talks': 961, u'follow': 1356, u'settlement': 1357, u'religious': 570, u'children': 962, u'opportunities': 1359, u'to': 203, u'program': 1360, u'health': 1387, u'activities': 1361, u'worse': 1362, u'very': 571, u'worst': 1364, u'decide': 572, u'fall': 963, u'stimulus': 1365, u'list': 1366, u'joined': 204, u'large': 205, u'small': 206, u'ten': 1367, u'past': 207, u'Only': 964, u'perspective': 965, u'further': 966, u'East': 1369, u'investment': 208, u'what': 1370, u'Korea': 435, u'consumers': 576, u'learned': 577, u'public': 968, u'contrast': 209, u'movement': 969, u'turmoil': 578, u'full': 210, u'strong': 579, u'losses': 580, u'experience': 211, u'amount': 581, u'social': 212, u'action': 213, u'followed': 1228, u'ask': 582, u'armed': 970, u'Europe': 971, u'conventional': 583, u'takes': 584, u'two': 973, u'governance': 974, u'Russian': 214, u'markets': 1372, u'achieving': 975, u'more': 215, u'American': 976, u'So': 586, u'particular': 977, u'none': 978, u'science': 216, u'coalition': 587, u'remain': 980, u'Japan\u2019s': 1374, u'Nor': 679, u'learn': 218, u'history': 588, u'share': 981, u'accept': 219, u'states': 220, u'sense': 221, u'huge': 222, u'needs': 982, u'Parliament': 1375, u'goal': 1376, u'rather': 1377, u'tried': 589, u'Israel': 540, u'coming': 984, u'Syria': 590, u'response': 985, u'a': 591, u'short': 1378, u'Mexico': 592, u'perhaps': 1511, u'responsibility': 986, u'fundamental': 223, u'media': 542, u'banks': 593, u'What': 987, u'help': 594, u'developed': 1379, u'soon': 595, u'trade': 1300, u'held': 596, u'already': 988, u'through': 989, u'suffer': 990, u'its': 224, u'25': 991, u'rapidly': 225, u'actually': 597, u'late': 993, u'absence': 535, u'systems': 598, u'might': 1380, u'good': 994, u'return': 1381, u'food': 599, u'framework': 1382, u'communities': 1383, u'easily': 995, u'always': 227, u'inequality': 1384, u'Council': 601, u'found': 228, u'week': 229, u'everyone': 997, u'generation': 1385, u'energy': 1346, u'hard': 998, u'reduce': 230, u'idea': 999, u'him': 1411, u'country\u2019s': 167, u'beyond': 602, u'insurance': 1001, u'really': 231, u'since': 603, u'research': 232, u'sustain': 604, u'issue': 605, u'Ukraine': 1002, u'belief': 233, u'According': 606, u'reason': 607, u'base': 608, u'members': 1003, u'imagine': 1371, u'put': 609, u'beginning': 1005, u'generate': 1388, u'benefits': 1006, u'People': 1389, u'macroeconomic': 1319, u'threat': 1390, u'fallen': 1391, u'major': 235, u'feel': 1392, u'radical': 1393, u'number': 236, u'Indian': 1394, u'done': 1007, u'story': 1396, u'leading': 1397, u'least': 1395, u'regulation': 1008, u'warming': 1515, u'relationship': 237, u'immediate': 238, u'part': 1010, u'believe': 1011, u'financing': 611, u'risks': 1400, u'toward': 612, u'strengthen': 1401, u'roughly': 942, u'substantial': 613, u'States': 1518, u'Arab': 1012, u'also': 239, u'internal': 240, u'Germany': 1402, u'build': 1014, u'Kim': 1015, u'With': 615, u'play': 241, u'towards': 616, u'measures': 1404, u'reach': 1405, u'most': 1016, u'plan': 242, u'significant': 1017, u'services': 1018, u'The': 618, u'extremely': 1019, u'constitution': 1256, u'Egypt': 1020, u'clear': 619, u'sometimes': 243, u'EU': 1004, u'traditional': 1407, u'institutions': 1408, u'sector': 1021, u'particularly': 1022, u'Islamist': 621, u'businesses': 1023, u'find': 1024, u'Kingdom': 244, u'justice': 622, u'Muslim': 245, u'French': 624, u'failed': 246, u'equity': 1472, u'Prime': 247, u'his': 1409, u'hit': 699, u'gains': 1403, u'kind': 1398, u'famous': 625, u'While': 626, u'rest': 171, u'closely': 248, u'international': 875, u'during': 627, u';': 680, u'enemy': 249, u'banking': 250, u'common': 1025, u'sustainable': 1412, u'set': 251, u'For': 252, u'achieved': 1413, u'France': 628, u'culture': 629, u'see': 253, u'individual': 1026, u'are': 1415, u'close': 630, u'#': 254, u'threats': 1416, u'Blair': 631, u'various': 1417, u'probably': 632, u'conditions': 633, u'corruption': 1418, u'available': 256, u'recently': 1419, u'creating': 1420, u'targeted': 257, u'attention': 732, u'opposition': 1421, u'African': 1422, u'both': 634, u'prospects': 258, u'last': 1504, u'annual': 1027, u'foreign': 1028, u'became': 1423, u'long-term': 259, u'let': 313, u'whole': 1516, u'Yet': 1029, u'point': 1030, u'reasons': 1424, u'community': 261, u'simply': 1031, u'throughout': 1032, u'War': 637, u'decline': 1425, u'raise': 1033, u'incomes': 1426, u'create': 1034, u'political': 1427, u'due': 1428, u'strategy': 1429, u'Most': 638, u'meeting': 1035, u'dialogue': 639, u'gas': 1036, u'else': 640, u'fund': 262, u'lives': 641, u'YORK': 263, u'demand': 1430, u'prices': 1038, u'Israel\u2019s': 1431, u'look': 642, u'elections': 1039, u'while': 643, u'behavior': 1432, u'Likewise': 686, u'Cold': 265, u'larger': 1465, u'Italy': 1433, u'century': 1040, u'rulers': 646, u'itself': 1041, u'seen': 810, u'Unfortunately': 371, u'Russia\u2019s': 1434, u'rules': 266, u'conflict': 648, u'higher': 1435, u'development': 1042, u'used': 649, u'comprehensive': 267, u'flows': 1436, u'moment': 1044, u'levels': 268, u'moving': 1437, u'euro': 38, u'lower': 1438, u'task': 1045, u'person': 270, u'Economic': 1046, u'organization': 1047, u'competitive': 1439, u'shape': 1049, u'openly': 1050, u'questions': 1440, u'Internet': 650, u'alternative': 1052, u'continent': 1441, u'Asia\u2019s': 272, u'cut': 1053, u'$': 651, u'majority': 1013, u'workers': 1442, u'source': 1054, u'merely': 877, u'On': 273, u'victims': 274, u'forces': 434, u'demands': 275, u'big': 1055, u'matters': 1056, u'game': 652, u'projects': 277, u'Pakistan': 1445, u'lost': 750, u'confront': 1446, u'continue': 84, u'popular': 654, u'essential': 1447, u'often': 1057, u'creation': 655, u'some': 656, u'back': 1058, u'urgent': 657, u'trends': 658, u'economic': 1452, u'scale': 1059, u'decision': 1060, u'integration': 1061, u'per': 1062, u'Fund': 1448, u'innovation': 1449, u'be': 279, u'run': 659, u'continuing': 1063, u'agreement': 280, u'step': 660, u'output': 552, u'abroad': 662, u'wealth': 879, u'goods': 116, u'anything': 283, u'range': 1451, u'into': 284, u'within': 665, u'Their': 666, u'appropriate': 285, u'NEW': 286, u'bankruptcy': 1064, u'spending': 287, u'question': 1453, u'fast': 904, u'forward': 1065, u':': 288, u'himself': 667, u'elsewhere': 289, u'hopes': 1066, u'Germany\u2019s': 1089, u'raising': 1454, u'GDP': 291, u'Africa': 292, u'up': 293, u'us': 294, u'Those': 113, u'similar': 669, u'called': 1455, u'associated': 1456, u'Perhaps': 1067, u'influence': 296, u'To': 930, u'single': 1068, u'New': 949, u'%': 1069, u'May': 1070, u'peace': 1457, u'These': 1262, u'politicians': 670, u'income': 1458, u'problems': 1459, u'sides': 297, u'structure': 671, u'ago': 298, u'land': 299, u'Tunisia': 1071, u'age': 300, u'required': 672, u'An': 673, u'2000': 1072, u'2001': 197, u'2006': 1073, u'Asian': 1074, u'2004': 1075, u'2005': 1076, u'2008': 1077, u'2009': 1078, u'requires': 676, u'having': 302, u'once': 1460, u'results': 303, u'existing': 304, u'go': 677, u'issues': 967, u'young': 305, u'send': 306, u'Not': 678, u'include': 1461, u'dramatically': 1462, u'resources': 217, u'outside': 1306, u'continues': 307, u'continued': 308, u'entire': 226, u'positions': 996, u'try': 309, u'Defense': 1079, u'Turkey': 1080, u'power': 176, u'rate': 1368, u'giving': 1081, u'access': 1082, u'capital': 1463, u'firms': 310, u'America': 681, u'America\u2019s': 682, u'can': 492, u'impact': 1464, u'proliferation': 311, u'led': 312, u'exchange': 1083, u'commercial': 683, u'Greek': 684, u'poverty': 260, u'others': 1084, u'extreme': 1085, u'great': 314, u'Chinese': 1086, u'involved': 315, u'30': 1087, u'suggests': 1466, u'products': 687, u'opinion': 316, u'makes': 317, u'danger': 688, u'manage': 689, u'private': 1088, u'confidence': 318, u'from': 1468, u'consumption': 1469, u'&': 1470, u'remains': 690, u'ministers': 319, u'next': 320, u'few': 1471, u'doubt': 321, u'NATO': 322, u'themselves': 908, u'reflects': 692, u'started': 693, u'women': 1473, u'account': 1090, u'sanctions': 694, u'this': 323, u'challenge': 324, u'recession': 695, u'meet': 696, u'something': 1475, u'control': 697, u'Israeli': 698, u'process': 325, u'tax': 1474, u'high': 326, u'effectively': 1092, u'reserves': 1093, u'US': 412, u'Party': 1476, u'Russia': 1450, u'united': 1477, u'democracy': 1094, u'six': 1478, u'regions': 1095, u'instead': 1479, u'British': 700, u'light': 1480, u'One': 327, u'Middle': 701, u'allow': 328, u'Greece': 329, u'institutional': 1096, u'move': 330, u'own': 1532, u'including': 703, u'policymakers': 439, u'industries': 1097, u'labor': 1098, u'willing': 1099, u'2011': 331, u'2010': 332, u'2013': 333, u'2012': 334, u'greater': 1100, u'mandate': 1101, u'assets': 959, u'day': 1102, u'presidential': 335, u'rising': 1344, u'From': 1103, u'doing': 337, u'society': 338, u'Treasury': 339, u'our': 1481, u'special': 706, u'out': 1482, u"'": 340, u'China': 1104, u'cause': 707, u'announced': 395, u'This': 1483, u'conflicts': 1484, u'allies': 1105, u'could': 341, u'area': 559, u'times': 1171, u'barriers': 1106, u'powerful': 1180, u'strategic': 1107, u'reached': 1108, u'quality': 709, u'management': 1186, u'Iran': 342, u'Eastern': 343, u'system': 344, u'relations': 712, u'priority': 1190, u'their': 1486, u'attack': 714, u'Iraq': 345, u'interests': 346, u'environmental': 1109, 'UNK': 0, u'elite': 347, u'defense': 1414, u'providing': 715, u'unlikely': 1110, u'have': 1111, u'need': 716, u'border': 717, u'clearly': 348, u'Development': 718, u'After': 1487, u'able': 719, u'concerns': 1112, u'which': 1488, u'worldwide': 349, u'unless': 1113, u'agenda': 702, u'1980\u2019s': 1114, u'who': 1489, u'why': 1490, u'so-called': 350, u'consequences': 720, u'Some': 471, u'disease': 1116, u'face': 352, u'fact': 353, u'goals': 1491, u'emerging': 721, u'currently': 255, u'agreed': 1492, u'supported': 1117, u'bring': 354, u'fear': 1493, u'debate': 1494, u'decade': 355, u'partners': 722, u'based': 723, u'knowledge': 1118, u'Meanwhile': 356, u'(': 724, u'controls': 1119, u'should': 357, u'York': 1120, u'fuel': 1263, u'local': 1266, u'achieve': 725, u'means': 691, u'overall': 726, u'legitimate': 727, u'words': 1495, u'areas': 1121, u'following': 118, u'taxes': 360, u'she': 728, u'view': 1496, u'Afghanistan': 528, u'national': 1122, u'\u2013': 47, u'officials': 729, u'terms': 446, u'liberalization': 1123, u'reform': 1124, u'nuclear': 361, u'below': 1538, u'tend': 730, u'state': 731, u'progress': 1125, u'neither': 1301, u'ability': 1498, u'importance': 733, u'deliver': 1126, u'job': 1499, u'key': 734, u'police': 891, u'Iran\u2019s': 1500, u'taking': 1127, u'deficits': 362, u'drug': 363, u'Finally': 1501, u'news': 1206, u'respect': 1502, u'proposal': 1128, u'eurozone': 365, u'am': 192, u'essentially': 1129, u'Nations': 1130, u'general': 1503, u'present': 1505, u'And': 736, u'unlike': 1506, u'value': 1131, u'General': 1132, u'will': 1508, u'EU\u2019s': 1509, u'2007': 198, u'supply': 1522, u'almost': 1133, u'thus': 1373, u'helped': 1134, u'Turkey\u2019s': 1135, u'terrorist': 367, u'productivity': 1136, u')': 1137, u'began': 737, u'administration': 1138, u'member': 1512, u'parts': 738, u'largest': 1513, u'party': 739, u'terrorism': 368, u'1970\u2019s': 740, u'difficult': 1514, u'effect': 741, u'identity': 610, u'destruction': 742, u'off': 369, u'center': 1139, u'well': 743, u'fighting': 1517, u'thought': 614, u'position': 617, u'latest': 620, u'restore': 744, u'excessive': 1519, u'less': 623, u'increasingly': 745, u'hands': 704, u'domestic': 1140, u'sources': 746, u'Argentina': 1520, u'taken': 585, u'rapid': 1521, u'Turkish': 635, u'add': 1141, u'book': 1541, u'nor': 504, u'Despite': 747, u'By': 748, u'increased': 647, u'government': 372, u'increases': 373, u'five': 374, u'know': 1524, u'loss': 749, u'necessary': 1444, u'like': 1142, u'success': 1143, u'lose': 751, u'become': 661, u'because': 1525, u'authority': 668, u'growth': 1526, u'export': 1527, u'proper': 1144, u'home': 752, u'employment': 1528, u'lead': 1529, u'avoid': 376, u'does': 377, u'leader': 1530, u'?': 753, u'S_END': 378, u'monetary': 754, u'expansion': 755, u'pressure': 685, u'although': 1145, u'loans': 379, u'about': 1146, u'freedom': 756, u'recognized': 757, u'UN': 758, u'Americans': 1531, u'Japan': 1147, u'equally': 759, u'alliance': 380, u'commitment': 381, u'Iraqi': 705, u'quickly': 1214, u'mere': 382, u'additional': 760, u'biggest': 1148, u'building': 383, u'buy': 1149, u'funds': 1533, u'but': 1150, u'gain': 761, u'highest': 762, u'he': 763, u'challenges': 1534, u'made': 1535, u'whether': 1536, u'dangerous': 928, u'official': 384, u'imbalances': 385, u'record': 1537, u'Britain': 1497, u'limit': 764, u'problem': 386, u'recognize': 387, u'1990\u2019s': 1539, u'education': 765, u'compared': 388, u'40': 1151, u'Islamic': 1540, u'other': 1523, u'boom': 1542, u'ultimately': 1277, u'chance': 389, u'rural': 393, u'friends': 766, u'South': 390, u'using': 271, u'Thus': 1153, u'rule': 391, u'compete': 392, u'understand': 1037, u'Saudi': 1543}
{u'd\xe9j\xe0': 678, u'Apr\xe8s': 1, u'consiste': 679, u'veut': 1068, u"l'Europe": 335, u'aider': 680, u'Turquie': 1070, u's\xe9curit\xe9': 2, u'devenir': 3, u'crises': 1071, u'\xab': 1072, u'lors': 4, u'processus': 1074, u'l\u2019a': 5, u'aux': 681, u'libert\xe9': 1075, u'si\xe8cle': 682, u'environnement': 238, u'voie': 683, u'tandis': 684, u'voir': 685, u'consid\xe9rable': 6, u'mauvais': 336, u'Premier': 1078, u'solution': 686, u'fonctionnement': 1079, u'importantes': 8, u'Etats': 722, u'actuellement': 687, u'force': 688, u'limites': 1069, u'Avec': 689, u'directement': 9, u'chez': 337, u'tirer': 10, u'communaut\xe9': 338, u'risques': 1081, u'familles': 11, u'propre': 691, u'\xc0': 339, u'protection': 340, u'100': 692, u'pr\xe8s': 341, u'm\xeame': 1082, u'servir': 971, u'Hamas': 12, u'permis': 693, u'populaire': 1083, u'derni\xe8res': 343, u'changer': 1084, u'reste': 694, u'outre': 1085, u'commerciaux': 1073, u'avantages': 695, u'total': 344, u'tels': 696, u'leurs': 13, u'seraient': 345, u'assez': 1086, u'type': 697, u'mois': 1087, u'Europ\xe9ens': 1088, u'utilis\xe9': 14, u'Ceci': 1089, u'sortir': 15, u'1990': 346, u'Cour': 347, u'semble': 698, u'provoque': 791, u'transparence': 348, u'g\xe9rer': 1090, u'mots': 241, u'derri\xe8re': 16, u"l'esprit": 1091, u'loi': 490, u'mondial': 1092, u'avaient': 17, u'Le': 18, u'Quand': 19, u'La': 20, u'soutien': 1093, u'\xe9trangers': 891, u'produits': 349, u'rythme': 21, u"l'OTAN": 699, u'quot': 700, u'l\u2019\xe9conomie': 701, u'nombre': 307, u'quoi': 702, u'particulier': 560, u'A': 22, u'moteur': 704, u'strat\xe9gique': 350, u'progr\xe8s': 1098, u'laquelle': 1099, u'dur\xe9e': 1100, u'erreurs': 351, u'environ': 1101, u'financement': 140, u'r\xe9duire': 353, u'peur': 23, u'suffisamment': 839, u'peut': 25, u'l\u2019\xe9tat': 706, u'Malheureusement': 707, u'afin': 708, u'occidentaux': 354, u'moiti\xe9': 709, u'enfants': 355, u'famille': 711, u'vite': 1103, u',': 712, u'l\xe9gitime': 26, u'd\xe9sormais': 155, u'production': 356, u'fin': 27, u'g\xe9n\xe9ralement': 890, u'mettre': 1104, u'recours': 28, u'strat\xe9gie': 357, u'grand': 561, u'prosp\xe9rit\xe9': 714, u'terroristes': 29, u'vouloir': 30, u'd\u2019aujourd\u2019hui': 716, u'musulmans': 1105, u'politiques': 717, u'choix': 1106, u"qu'elle": 31, u'r\xe9forme': 718, u'qu\u2019en': 32, u'cr\xe9er': 719, u'financi\xe8res': 359, u'impossible': 444, u'aurait': 33, u'dettes': 1109, u'toute': 34, u'forte': 360, u'spectaculaire': 35, u'd\xe9cennie': 36, u'actuelle': 720, u'Cette': 1110, u'normes': 721, u'sauver': 723, u'repr\xe9sentent': 724, u'am\xe9ricaine': 725, u'veulent': 37, u'ralentissement': 726, u'villes': 38, u'longue': 198, u'maladies': 39, u'Congr\xe8s': 361, u'Cela': 362, u'leaders': 1080, u'\xe9nerg\xe9tique': 363, u'v\xe9ritablement': 727, u'engagement': 40, u'Br\xe9sil': 41, u'Cet': 213, u'Ces': 42, u'liquidit\xe9s': 729, u"l'inflation": 1111, u'prendre': 365, u'r\xe9sultats': 1112, u'nations': 44, u'notamment': 730, u'Japon': 1113, u'poids': 1114, u'Etats-Unis': 1115, u'Alors': 732, u'telle': 733, u'employ\xe9s': 366, u'alors': 367, u'-': 1116, u'industrielle': 1117, u'faveur': 734, u'forces': 368, u'technologique': 735, u'fiscales': 736, u'totalement': 1118, u'doit': 45, u'\xe9tat': 369, u'secteurs': 1119, u'points': 737, u's\xe9rieux': 1120, u'certainement': 738, u'pourraient': 370, u'Union': 46, u'prix': 47, u'solutions': 1121, u'semblent': 739, u'pris': 48, u'cr\xe9ation': 371, u'capacit\xe9': 1122, u'cependant': 1123, u'carbone': 740, u'r\xe8gles': 741, u'de': 372, u'd\u2019autres': 49, u'du': 373, u'malgr\xe9': 374, u'bas': 742, u'furent': 375, u'guerres': 743, u'climatique': 1125, u'avanc\xe9es': 50, u'individus': 1127, u'fait': 51, u'\xe9change': 52, u'guerre': 53, u'n\xe9cessaire': 1128, u'r\xe9cemment': 1129, u'discussions': 377, u'actifs': 1130, u'majeur': 378, u'sorte': 744, u'investisseurs': 302, u'bien': 379, u'sait': 745, u'accord': 380, u'nature': 306, u'derni\xe8re': 1147, u"n'en": 56, u'souvenir': 57, u'terme': 381, u'd\xfb': 382, u'pr\xe9vu': 746, u'davantage': 58, u'candidat': 1131, u'\xc9tats-Unis': 59, u'ils': 383, u'demander': 60, u'contribution': 748, u'l\xe9gitimit\xe9': 1132, u'compromis': 61, u'imp\xf4ts': 750, u'BCE': 62, u'scientifique': 751, u'nucl\xe9aires': 297, u'dangereux': 324, u'sc\xe9nario': 1133, u'parler': 64, u'm\xe9dicaments': 1134, u'.': 65, u'puissance': 753, u'revenu': 384, u'd\xe9cision': 385, u'penser': 66, u'd\xe9fis': 1135, u'meilleurs': 67, u'meilleure': 334, u'susceptibles': 1067, u'fortement': 754, u'efficace': 755, u'160': 756, u'commerce': 68, u't\xeate': 69, u'ait': 386, u'confiance': 70, u'Ce': 757, u'mon\xe9taires': 758, u'sauvetage': 1136, u'reconnaissance': 759, u'tant': 342, u"Aujourd'hui": 71, u'biais': 760, u'Bush': 1138, u'il': 761, u'r\xe9serves': 1094, u'chiffres': 763, u'quand': 764, u'\xe9videmment': 765, u'aient': 1139, u'quant': 766, u'p\xe9trole': 387, u'chinoises': 388, u'elle': 767, u'cr\xe9anciers': 72, u'S_START': 389, u'1970': 390, u'bonne': 73, u'the': 391, u'Bien': 376, u'Moyen-Orient': 74, u'l\u2019histoire': 768, u'transformer': 392, u'v\xe9rit\xe9': 1140, u'milliers': 1141, u'contribuer': 75, u'candidats': 769, u'visant': 76, u'conflits': 770, u'deux': 452, u'capitaux': 394, u'possible': 77, u'unique': 78, u'aujourd\u2019hui': 395, u'l\u2019Iran': 396, u'l\u2019Irak': 470, u'mesure': 771, u'fournir': 7, u'50': 1142, u'explique': 1143, u'complexe': 80, u'froide': 81, u'n\u2019en': 1144, u'domaines': 772, u'augment\xe9': 773, u'\xe9chelle': 79, u'Commission': 397, u'\xe9conomie': 411, u'Banque': 398, u'atteint': 774, u'ici': 1145, u'donne': 565, u'Les': 399, u'gouvernement': 1146, u'\xe9vident': 775, u'participation': 400, u"l'on": 1312, u'dollars': 86, u'cadre': 777, u'plus': 1149, u'efforts': 1150, u'dirigeant': 778, u'mondialisation': 87, u'fera': 1151, u'Fed': 88, u'pays': 402, u'lien': 779, u'son': 89, u'loin': 403, u'changements': 1152, u'lieu': 780, u'essentiellement': 90, u'jeunes': 1153, u'mondiaux': 91, u'avait': 92, u"c'est": 781, u'accords': 404, u'd\xe8s': 405, u'pressions': 406, u'ligne': 93, u'tr\xe8s': 454, u'th\xe9orie': 82, u'fort': 934, u'pratiquement': 782, u'prochaines': 407, u'voire': 94, u'qu\u2019un': 408, u'd\xe9montr\xe9': 409, u'\xe9conomistes': 1154, u'arabes': 1155, u'd\xe9velopp\xe9s': 1365, u'moderne': 95, u'ch\xf4mage': 783, u'limit\xe9e': 784, u"''": 96, u'peuples': 1213, u'cons\xe9quent': 97, u'Gaza': 785, u'face': 300, u'ni': 1215, u'Isra\xebl': 98, u'r\xe9ellement': 786, u'ne': 1157, u'biens': 99, u'russe': 787, u'objectifs': 100, u'doute': 1158, u'religieux': 788, u'\xe9lections': 1159, u'm\xe9dias': 1160, u'secteur': 1234, u'sujet': 412, u'relance': 413, u'concept': 414, u'corps': 101, u'global': 415, u'efficaces': 789, u'baisse': 416, u'retour': 1162, u'signifie': 1163, u'd\u2019Isra\xebl': 504, u'zone': 417, u'donner': 418, u'chances': 1367, u'\xe9trang\xe8res': 257, u'nouvelle': 1102, u'Plus': 790, u'hausse': 1164, u'pr\xe9sidence': 419, u'dix': 792, u'dit': 793, u'capitalisme': 650, u'pr\xeats': 102, u'M.': 103, u'droit': 533, u'toutefois': 795, u"d'une": 420, u'quelques': 421, u'd\xe9clin': 796, u'autorit\xe9s': 104, u'constitution': 1165, u'seule': 1166, u'd\xe9veloppement': 1167, u'Qui': 1168, u'international': 797, u'seuls': 1169, u'groupe': 1170, u'Libye': 422, u'principales': 548, u'demeure': 423, u'change': 424, u'moyens': 798, u'financiers': 105, u'cette': 106, u'domaine': 425, u'Pourquoi': 800, u'poursuivre': 107, u'entier': 1171, u'savoir': 1314, u'parmi': 1172, u'd\u2019un': 108, u'Russie': 803, u'd\xe9fendre': 109, u'propres': 1173, u'vous': 804, u'``': 110, u'pratique': 1174, u'volont\xe9': 111, u'auront': 426, u'reprise': 581, u'Inde': 1175, u'Lors': 1095, u'garantir': 1176, u'pratiques': 1177, u'Kosovo': 1179, u'travailleurs': 1178, u'auraient': 1096, u'cas': 427, u'car': 428, u'sous': 806, u'investissements': 429, u'mouvement': 1180, u'pourrait': 112, u'd\xe9fi': 808, u'r\xe9duction': 430, u'parce': 809, u"jusqu'\xe0": 401, u'1': 1181, u'Avant': 1182, u'd\xe9cembre': 113, u'date': 114, u'mal': 811, u'permet': 812, u'facile': 622, u'avant': 813, u'si': 115, u'nous': 814, u'sa': 116, u'priv\xe9': 117, u'se': 118, u'mont\xe9e': 119, u'd\xe9faut': 1184, u'nbsp': 815, u'\xe9tant': 1257, u'hauts': 220, u'probl\xe8me': 1185, u'police': 816, u'Ainsi': 431, u'forme': 432, u'souverainet\xe9': 1186, u'main': 818, u'non': 433, u'faillite': 120, u'nation': 121, u'stimuler': 762, u'possibilit\xe9': 122, u'nos': 434, u'avons': 1188, u'stabilit\xe9': 821, u'en': 435, u'savent': 822, u'Barack': 436, u'r\xe9ponse': 1189, u'eu': 437, u'et': 438, u'qu\u2019elle': 824, u'internes': 123, u'Certains': 1190, u'valeur': 1191, u'viable': 439, u'd\xe9penses': 690, u'Guerre': 825, u'principe': 826, u'place': 1192, u'massive': 124, u'prot\xe9ger': 125, u'exportations': 827, u'ministre': 828, u'nouvelles': 440, u'Certes': 1193, u'centrale': 829, u'tenir': 441, u'Asie': 442, u'revenus': 1194, u'violences': 443, u'd\xe9mocratique': 1195, u'ont': 126, u'millions': 1196, u'droite': 1197, u'peut-\xeatre': 728, u'droits': 43, u'structures': 128, u'connu': 731, u'pouvait': 1032, u'besoins': 129, u'\xe9viter': 130, u'montrent': 830, u'Nous': 445, u'population': 1198, u'11': 831, u'10': 832, u'12': 833, u'tendance': 132, u'sera': 1199, u'l\u2019Occident': 834, u'aller': 133, u'Mais': 447, u'plut\xf4t': 978, u'premier': 448, u'qu\u2019il': 134, u'financier': 135, u'ans': 1200, u'entra\xeen\xe9': 449, u'presque': 450, u'priv\xe9s': 136, u's\u2019est': 451, u"l'Union": 137, u'De': 835, u'masse': 138, u'perdre': 139, u'terrorisme': 1201, u'Du': 836, u'paix': 837, u'Am\xe9rique': 838, u'doivent': 453, u'contr\xf4le': 141, u'conflit': 455, u'mains': 456, u'Ch\xe1vez': 523, u'diff\xe9rents': 1264, u'trop': 1203, u'serait': 142, u'Saddam': 1204, u'diff\xe9rentes': 143, u'presse': 144, u'concurrence': 24, u'l\u2019Asie': 458, u'monde': 145, u'd\xe9terminer': 459, u'nouveau': 840, u'r\xe9gional': 841, u'leur': 460, u'responsables': 410, u'l\u2019id\xe9e': 461, u'vision': 146, u'nouveaux': 147, u'l\u2019un': 842, u'co\xfbts': 1206, u'universit\xe9s': 843, u'partisans': 710, u'mod\xe8le': 844, u'3': 462, u'chaque': 1207, u'suppl\xe9mentaires': 845, u'l\u2019emploi': 1062, u'facteur': 846, u'\xe9conomique': 148, u'l\u2019opposition': 851, u'Palestiniens': 847, u'je': 848, u'vrai': 463, u'vue': 464, u'implique': 465, u'l\u2019avenir': 466, u'gouvernance': 149, u'tous': 849, u'tout': 850, u'aura': 467, u'autour': 150, u'c\xf4t\xe9': 852, u'r\xe9cente': 853, u'\xe9t\xe9': 151, u'devrait': 1208, u'moyenne': 152, u'60': 1209, u'besoin': 153, u'traitement': 468, u'seulement': 154, u'capable': 854, u'lorsque': 855, u"qu'il": 469, u'sunnites': 156, u'pr\xe9sent': 577, u'venir': 157, u'concernant': 1211, u'direction': 1212, u'ceux': 887, u'quatre': 856, u"n'a": 893, u"l'id\xe9e": 857, u'\xe9lev\xe9': 471, u'parfois': 1214, u'pouvons': 858, u'situation': 158, u'nationales': 859, u'soit': 472, u'victimes': 713, u'd\xe9clar\xe9': 860, u'par': 861, u'pas': 862, u'devoir': 546, u"n'est": 473, u'nombreuses': 1216, u'n\xe9gociations': 863, u'diff\xe9rences': 1366, u'faible': 864, u'lequel': 865, u'bancaires': 1217, u'Il': 1218, u'ces': 519, u'Etat': 1037, u'taille': 474, u'consommateurs': 475, u'actions': 1219, u'europ\xe9en': 476, u'N\xe9anmoins': 951, u'4': 866, u'perte': 867, u'combien': 1220, u'certaine': 477, u'succ\xe8s': 160, u'accepter': 1221, u'certains': 478, u'Tous': 479, u'Tout': 480, u'mon': 868, u'payer': 521, u'o\xf9': 1222, u'aucun': 715, u'degr\xe9': 483, u'r\xf4le': 484, u'Deux': 869, u'avanc\xe9s': 870, u'laisser': 161, u'mesures': 1210, u'constitue': 162, u'\xe9poque': 163, u'Enfin': 1375, u'sommes': 871, u'r\xe9soudre': 485, u'peuple': 1224, u"qu'un": 203, u'm\xe9thodes': 872, u'chefs': 486, u'leadership': 164, u'seconde': 997, u'on': 1225, u'central': 487, u'of': 1226, u'violence': 874, u'tentatives': 488, u'ou': 1227, u'l\u2019Inde': 1228, u'parties': 165, u'd\xe9s\xe9quilibres': 166, u'm\xeames': 875, u'celui': 1230, u'plusieurs': 167, u'Poutine': 168, u'Obama': 876, u'asiatiques': 491, u'mati\xe8re': 1231, u'r\xe9pondre': 169, u'bancaire': 1232, u'syst\xe8mes': 1343, u'technologies': 170, u'autre': 1108, u'internationale': 1233, u"qu'ils": 492, u'USA': 877, u'l\u2019Am\xe9rique': 171, u'certain': 172, u'\xe9lecteurs': 493, u'an': 173, u'rapide': 878, u'puissances': 879, u'pourquoi': 880, u'publiques': 1040, u'clairement': 881, u'arabe': 494, u'actuel': 1161, u'agricoles': 882, u'5': 1235, u'personnes': 883, u'Au': 583, u'prise': 884, u'discours': 1236, u'britannique': 495, u'qu\u2019ils': 496, u'important': 885, u'cr\xe9dit': 176, u'citoyens': 886, u'menace': 267, u'd\xe9ficit': 888, u'recherches': 177, u'd\xe9mocratiques': 1238, u'responsabilit\xe9': 919, u'dollar': 1076, u'd\xe9pit': 1077, u"l'Iran": 207, u'pauvret\xe9': 497, u'pouvoirs': 178, u'qu\u2019\xe0': 498, u'Elle': 499, u'victoire': 889, u'vers': 500, u'devront': 892, u'march\xe9s': 501, u'projet': 179, u'sont': 352, u'pression': 180, u'int\xe9r\xeats': 358, u'sans': 181, u'europ\xe9ens': 182, u'parvient': 1241, u'large': 183, u"l'UE": 1242, u'programme': 1243, u'tel': 1244, u'l\u2019\xe9lection': 894, u'conscience': 503, u'sud': 1245, u'devenu': 184, u'sur': 1246, u'peuvent': 185, u'gagner': 186, u'public': 1126, u'Malgr\xe9': 187, u'\xe0': 895, u'\xe9chou\xe9': 269, u'contraire': 505, u'gr\xe2ce': 1247, u'social': 189, u'consommation': 896, u'vie': 190, u'ressources': 506, u'armes': 897, u"d'autres": 898, u'l\u2019ensemble': 899, u'R\xe9publique': 1248, u'Europe': 900, u'mort': 191, u'\xc9tat': 901, u'prochain': 1249, u'n\u2019\xe9tait': 1250, u'budg\xe9taire': 1251, u'r\xe9alit\xe9': 508, u'Si': 509, u'suite': 586, u'Cor\xe9e': 1252, u'gauche': 510, u'am\xe9ricains': 192, u'science': 193, u'des': 903, u'coalition': 511, u'sociale': 194, u'taux': 904, u'Nord': 1253, u'\xe9conomies': 1254, u'cours': 1255, u'court': 1256, u'rapport': 905, u'r\xe9cession': 1258, u'ensemble': 906, u'pertes': 1259, u'remettre': 923, u'notre': 195, u'croissance': 907, u'petits': 908, u'diplomatique': 1260, u'vient': 909, u'Syrie': 512, u'a': 513, u'chute': 910, u'grands': 1261, u'trois': 911, u'Comme': 514, u'grande': 1262, u"l'\xe9conomie": 1263, u'l\u2019Union': 196, u'signe': 197, u'attentes': 515, u'soldats': 457, u'mission': 1265, u'20': 913, u'surtout': 666, u'changement': 516, u'mani\xe8re': 1266, u'termes': 199, u'Lorsque': 1273, u'n\u2019ont': 316, u'n\u2019y': 517, u'au-del\xe0': 1267, u'personne': 201, u'cet': 518, u'soient': 202, u'n\u2019a': 520, u'Afrique': 914, u'faire': 482, u'porte': 915, u'eux-m\xeames': 1223, u'pr\xe9sident': 916, u'compris': 1268, u'cinq': 1381, u'troisi\xe8me': 917, u"d'\xeatre": 1269, u'commerciales': 918, u'quelque': 204, u'c\u0153ur': 1237, u"l'Irak": 205, u'statut': 206, u'juillet': 1270, u'valeurs': 1239, u'participants': 920, u'effets': 1240, u'Gr\xe8ce': 1271, u'base': 524, u'Une': 921, u'dire': 525, u'selon': 507, u'nationaux': 526, u'En': 922, u'projets': 1272, u'avec': 212, u'Et': 924, u'dont': 925, u'ville': 794, u'mise': 527, u'n\xe9cessairement': 209, u'donc': 926, u'principes': 210, u'passer': 1274, u'majorit\xe9': 927, u'jeu': 211, u'\xe9conomiques': 928, u"qu'une": 529, u'contre': 214, u'financer': 1275, u'Depuis': 705, u'repr\xe9sente': 1276, u'travail': 929, u'ainsi': 215, u'Am\xe9ricains': 930, u'part': 931, u'r\xe9gions': 216, u'longtemps': 530, u'atteindre': 932, u"l'Am\xe9rique": 933, u'produit': 217, u'v\xe9ritable': 531, u'double': 1277, u'dans': 1278, u'ann\xe9e': 935, u'organisations': 218, u'concentrer': 159, u's\u2019agit': 219, u"l'homme": 221, u'jour': 535, u'faisant': 936, u'suivi': 536, u'dernier': 223, u't\xe2che': 1280, u'capacit\xe9s': 537, u'plan': 224, u'services': 937, u'infrastructures': 574, u'maladie': 938, u'd\xe9mocratie': 576, u'institutions': 1283, u'obligations': 939, u'europ\xe9ennes': 1284, u'mon\xe9taire': 225, u'produire': 226, u'toujours': 227, u'relation': 940, u'gouvernements': 228, u'font': 1285, u'devra': 1137, u'combattre': 538, u'quel': 539, u'Ensuite': 1286, u'Je': 1287, u'parlement': 941, u'titres': 229, u'montre': 540, u'mieux': 1288, u'2': 127, u'banques': 1289, u'circonstances': 1290, u'europ\xe9enne': 1291, u'premi\xe8re': 230, u'd\u2019\xc9tat': 231, u'ses': 232, u'potentiel': 942, u'int\xe9rieure': 943, u'Cependant': 233, u'France': 541, u'donn\xe9es': 623, u'#': 235, u'probable': 542, u'vont': 631, u'femmes': 1292, u'conditions': 544, u'corruption': 1293, u'gu\xe8re': 944, u'Sans': 545, u'd\xe9cid\xe9': 645, u'effet': 547, u'Dans': 945, u'cons\xe9quences': 549, u'opposition': 1294, u'genre': 550, u'montr\xe9': 551, u'recherche': 236, u'approche': 946, u'erreur': 237, u'l\u2019euro': 552, u'principalement': 553, u'globale': 947, u'point': 948, u'simple': 949, u'sc\xe8ne': 1295, u'moins': 1296, u'pu': 1297, u'si\xe8cles': 950, u'Plan': 668, u'dialogue': 554, u'gaz': 952, u'aussi': 534, u'plan\xe8te': 555, u'fiscale': 953, u'devraient': 954, u'envers': 556, u'fut': 955, u'r\xe9elle': 1298, u'puissant': 956, u'nucl\xe9aire': 239, u'mener': 957, u'permettrait': 799, u'\xe9taient': 240, u'grandes': 958, u'l\u2019Allemagne': 557, u'facteurs': 558, u'comprendre': 208, u'puisse': 559, u'soci\xe9t\xe9s': 959, u'principaux': 703, u'n\xe9cessit\xe9': 242, u'historique': 243, u'scientifiques': 961, u'r\xe9ussi': 244, u'propri\xe9taires': 1279, u'deviennent': 84, u'bon': 801, u'emplois': 562, u'moment': 962, u'000': 563, u'Ils': 1299, u'euro': 245, u'travers': 963, u'entre': 964, u'permettent': 1300, u'contraintes': 1301, u'march\xe9': 1302, u'Son': 246, u'y': 965, u'Grande': 481, u'occidentale': 247, u'coup': 564, u'trait\xe9': 1304, u'j\u2019ai': 248, u'questions': 1305, u'exp\xe9riences': 54, u'\xe9missions': 222, u'PIB': 749, u'source': 966, u'difficile': 1306, u'financi\xe8re': 1307, u'On': 249, u'cons\xe9quence': 250, u'demande': 251, u'uniquement': 1309, u'compte': 967, u'Or': 252, u'sant\xe9': 253, u'sovi\xe9tique': 1310, u'Pakistan': 1311, u'consensus': 83, u'flux': 968, u'continue': 254, u'campagne': 776, u'technologie': 85, u'soci\xe9t\xe9': 255, u'publics': 969, u'autres': 566, u'jouer': 567, u'C\u2019est': 970, u'jamais': 568, u'rien': 1313, u'proche': 256, u'peu': 972, u'depuis': 973, u'\u0153uvre': 569, u'qu\u2019une': 570, u"s'agit": 974, u'l\u2019homme': 802, u'exemple': 1315, u'Comment': 571, u'Liban': 258, u'\xeatre': 572, u'juste': 975, u'pauvres': 807, u'&': 1333, u'n\xe9cessaires': 976, u'donn\xe9': 573, u'devenue': 1316, u'question': 1317, u'manque': 977, u'long': 1318, u'avenir': 200, u'suis': 259, u'Elles': 260, u':': 131, u'chinoise': 1281, u'ailleurs': 261, u'meilleur': 262, u'renforcer': 575, u'pire': 1319, u'affaires': 979, u'un': 263, u'naissance': 1282, u'influence': 264, u'%': 980, u'groupes': 1320, u'tentative': 1321, u'elles': 981, u'lesquelles': 265, u'tard': 266, u'Selon': 1322, u'rester': 578, u'chacun': 579, u'l\u2019ann\xe9e': 1323, u'critiques': 580, u'naturellement': 982, u'jusqu\u2019\xe0': 1324, u'vraiment': 663, u'politiciens': 1325, u'relativement': 1326, u"l'euro": 983, u'2000': 984, u'2001': 985, u'2006': 986, u'2007': 987, u'2004': 175, u'utiliser': 584, u'2008': 988, u'2009': 989, u'proposition': 585, u'largement': 1327, u'profond\xe9ment': 268, u'pouvoir': 188, u'militaire': 805, u'sens': 270, u'\xbb': 1328, u'op\xe9rations': 990, u'yeux': 991, u'trouvent': 279, u"s'est": 992, u'militaires': 993, u'l\u2019\xe9nergie': 1329, u'l\u2019on': 1387, u'\xe9changes': 1330, u'capital': 1331, u';': 587, u'cela': 588, u'd\u2019int\xe9r\xeat': 994, u'chose': 1332, u'commercial': 589, u'les': 271, u'\xe9tait': 590, u's\xfbr': 591, u'30': 995, u'beaucoup': 272, u'gens': 592, u"n'ont": 960, u'activit\xe9s': 593, u'autant': 273, u'co\xfbt': 274, u'l\u2019arm\xe9e': 275, u'd\xe9ficits': 996, u'fran\xe7ais': 594, u'commun': 276, u'premi\xe8res': 595, u'faudra': 277, u'celles': 873, u'l\u2019effondrement': 1334, u'M\xeame': 596, u'milliards': 597, u'coop\xe9ration': 998, u'mauvaise': 598, u'train': 999, u'immobiliers': 1338, u'entreprises': 1335, u'devant': 278, u'sanctions': 599, u'puissent': 234, u'pour': 280, u'\xe9lev\xe9s': 1336, u'devons': 600, u'objectif': 1000, u's\u2019en': 1001, u'Parti': 1337, u'risque': 601, u'r\xe9formes': 1002, u'six': 1339, u'Grande-Bretagne': 1340, u'titre': 602, u'souvent': 1003, u'probablement': 1004, u'pensent': 1005, u'classe': 281, u'claire': 604, u'r\xe9solution': 605, u'septembre': 282, u'publique': 606, u'ordre': 1341, u'comme': 283, u'd\xe9mocraties': 284, u'vivent': 285, u'deuxi\xe8me': 607, u'le': 1006, u'budg\xe9taires': 608, u'aucune': 609, u'vivant': 1008, u'choses': 312, u'hommes': 610, u'2010': 286, u'pleine': 611, u'l\u2019an': 1342, u'trouve': 287, u'\xe9galement': 612, u'importante': 288, u'nationale': 613, u'Toutefois': 614, u'importants': 289, u'fronti\xe8res': 1344, u'n\u2019est': 1011, u'80': 1345, u'programmes': 1012, u'occidentales': 290, u'd\xe9cennies': 1013, u"d'int\xe9r\xeat": 1346, u'l\xe0': 1014, u'd\xe9but': 615, u'Chine': 1015, u'd\u2019\xeatre': 1016, u'cause': 616, u'temps': 291, u'que': 1017, u'pendant': 1347, u"d'un": 292, u'qui': 1018, u'parvenir': 1019, u'islamique': 1107, u'fa\xe7on': 293, u'faibles': 1097, u"l'Allemagne": 1020, u'r\xe9sultat': 1021, u'permettra': 1348, u'Rien': 294, u'pourra': 617, u'permettre': 1349, u'suffit': 1022, u'mai': 810, u'am\xe9ricaines': 1350, u'mondiale': 1351, u'Irak': 295, u'anciens': 1023, u'durable': 618, u'relations': 619, u'banque': 1148, u'plupart': 620, u'd\xe9cisions': 621, u'compr\xe9hension': 1024, u'n\xe9anmoins': 296, u'constituent': 1352, 'UNK': 0, u'Pourtant': 1124, u'riches': 1353, u'derniers': 1354, u'nombreux': 1183, u'patients': 1355, u'gestion': 624, u'modernes': 1356, u'encore': 1357, u'Par': 1025, u'r\xe9gime': 603, u'maintenir': 625, u'qualit\xe9': 626, u'faut': 1358, u'priorit\xe9': 1359, u'concerne': 1026, u'physique': 627, u'latine': 393, u'seront': 1360, u'mis': 1027, u'am\xe9liorer': 1361, u'centre': 1362, u'dette': 298, u'commenc\xe9': 1363, u'maintenant': 628, u'r\xe9fl\xe9chir': 629, u'lutte': 1364, u'd\u2019Europe': 299, u'd\xe9fense': 1156, u's\u2019ils': 630, u'Blair': 543, u'membres': 632, u'passe': 633, u'Parlement': 634, u'locaux': 635, u'principal': 301, u'durant': 1368, u'(': 636, u'rendre': 1369, u'men\xe9e': 1028, u'libre': 637, u"qu'elles": 1029, u'trouver': 638, u'pr\xeat': 303, u'centrales': 639, u'dirigeants': 640, u'partie': 1187, u'ouverte': 641, u'partir': 1370, u'partis': 1371, u'Aujourd\u2019hui': 1030, u'Afghanistan': 304, u'prises': 305, u'national': 1031, u"C'est": 642, u'terrain': 643, u'existe': 1033, u'\u2013': 446, u'pass\xe9': 644, u'preuve': 1202, u'd\xe9bats': 1034, u'l\u2019UE': 55, u'lui': 646, u'tenu': 1205, u'l\u2019autre': 1372, u'raisons': 1035, u'sociaux': 647, u'importance': 648, u'attaques': 308, u'certaines': 1373, u'tiers': 1374, u'mais': 817, u'la': 1007, u'comment': 1036, u'ce': 309, u'rapidement': 310, u'responsable': 364, u'apr\xe8s': 489, u'restent': 311, u'tient': 1229, u'productivit\xe9': 1038, u'eux': 1009, u'Nations': 1039, u'potentiels': 649, u'ann\xe9es': 1376, u"d'entre": 502, u'riche': 1377, u'au': 174, u'Fonds': 651, u'contexte': 652, u'tendances': 1378, u'va': 313, u'partout': 1379, u'vu': 314, u'raison': 315, u'sociales': 1380, u'fonds': 653, u')': 1041, u'parti': 654, u'r\xe9chauffement': 819, u'Pendant': 1042, u'une': 747, u'b\xe9n\xe9fices': 522, u'g\xe9opolitique': 1382, u'Pour': 655, u'r\xe9volution': 656, u'moyen': 528, u'premiers': 1383, u'chinois': 532, u'l\u2019Ukraine': 657, u'obstacles': 658, u'position': 1043, u'diff\xe9rence': 1384, u"qu'\xe0": 317, u'femme': 659, u'politique': 318, u'monnaie': 1044, u'telles': 820, u'ensuite': 1045, u'l\u2019Europe': 1303, u'ayant': 319, u'd\xe9clarations': 63, u'devises': 660, u'p\xe9riode': 1308, u'r\xe9gion': 661, u'crise': 320, u'd\u2019\xe9nergie': 662, u'c\u2019est': 1385, u'am\xe9ricain': 1010, u'Sud': 1386, u'puissants': 1046, u'syst\xe8me': 1047, u'est': 582, u'avoir': 321, u'?': 664, u'S_END': 322, u'FMI': 902, u'pourcentage': 1388, u'Un': 665, u'\u0155': 323, u'Europ\xe9enne': 1048, u'internationales': 1049, u'chance': 332, u'pourtant': 1389, u'\xe9tats': 1390, u'civile': 912, u'milieu': 752, u'Conseil': 667, u'Mexique': 1050, u'1980': 1051, u'toutes': 1052, u'1989': 1053, u'internationaux': 325, u'probl\xe8mes': 1054, u'simplement': 669, u'Des': 1055, u'\xe9trang\xe8re': 670, u'but': 1056, u'd\u2019une': 1391, u'construction': 326, u'humain': 1392, u'g\xe9n\xe9ral': 671, u'r\xe9gionale': 327, u'celle': 673, u'sein': 1057, u'\xe9mergents': 1393, u'nouvel': 328, u'priv\xe9es': 1058, u'agir': 823, u"aujourd'hui": 1059, u'pr\xe9c\xe9dent': 1060, u'pr\xe9server': 329, u'importations': 674, u'seul': 330, u'40': 1061, u'futur': 675, u'r\xe9ussite': 676, u'd\xe9bat': 331, u'fois': 677, u'membre': 1063, u'augmentation': 1064, u'difficult\xe9s': 1065, u'lui-m\xeame': 1394, u'\xc9tats': 672, u'niveau': 333, u'augmenter': 1066}






    




/Users/preethikapachaiyappa/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:92: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
/Users/preethikapachaiyappa/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:103: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
/Users/preethikapachaiyappa/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:104: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future






    




Total Loss: 37472.3951729
Total Loss: 33952.3828929
Total Loss: 34079.606341
Total Loss: 34181.0310067
Total Loss: 34241.8576626
Total Loss: 34040.1619295
Total Loss: 34256.56869
Total Loss: 34312.5718729
Total Loss: 34115.0493199
Total Loss: 34136.7114393
Total Loss: 34189.9331039
Total Loss: 34240.7381353
Total Loss: 34201.6724879
Total Loss: 34242.018254
Total Loss: 34091.8234161
Total Loss: 34080.5613835
Total Loss: 34244.2514971
Total Loss: 34308.7619727
Total Loss: 34206.0111802
Total Loss: 34258.6154221

In [17]:

    

contextWords = []
    
for _ in np.arange(epoch):
    
    totalLoss = 0
    loss = 0
    
    fileIn = open(fin1)
    for l in fileIn:
        l = _start+' '+l+' '+_end
        tokens = word_tokenize(l.decode('utf-8'))
        #print 'tokens',tokens
        for token in tokens:
            
            loss = 0
            contextWords = []
            
            if token in _vocab_f:
                target = _vocab_f[token]
                trgtIdx = tokens.index(token)
                cntxtIdxs = range(trgtIdx-win_size, trgtIdx+win_size+1)
                cntxtIdxs.remove(trgtIdx)
                for idx in cntxtIdxs:
                    #check for first word and last word and use UNK for context words for window where words not available
                    if idx >-1 and idx < len(tokens) and tokens[idx] in _vocab_f:
                        contextWords = np.append(contextWords, _vocab_f[tokens[idx]])
                    else:
                        contextWords = np.append(contextWords, _vocab_f['UNK'])
                loss += skipgram(target, contextWords, vocabSize_f, learningRate, W_Embedding_f, W_Output_f)
        totalLoss += loss
    print 'Total Loss:',totalLoss
                

print(W_Embedding_f)

idx = _vocab_f[_start]


    

    




/Users/preethikapachaiyappa/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:92: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
/Users/preethikapachaiyappa/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:103: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
/Users/preethikapachaiyappa/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:104: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future






    




Total Loss: 35995.6176593
Total Loss: 33123.2675223
Total Loss: 33207.6133279
Total Loss: 33075.6164044
Total Loss: 33446.590819
Total Loss: 33200.581148
Total Loss: 33237.2641109
Total Loss: 33067.18647
Total Loss: 33262.5368821
Total Loss: 33205.10318
Total Loss: 33428.5308258
Total Loss: 33320.2787369
Total Loss: 33252.7893135
Total Loss: 33175.4004516
Total Loss: 33211.1464751
Total Loss: 33311.5987104
Total Loss: 33381.1454684
Total Loss: 33284.8474257
Total Loss: 33268.6784086
Total Loss: 33157.5456106
[[ 0.75156866  0.2517222  -0.68399158 ...,  0.92442616  0.60871534
  -0.09872267]
 [-0.67502403 -1.50688523  0.64144321 ...,  0.03777165  0.5064405
   0.60732242]
 [ 0.12958977  0.20589492  1.04009858 ..., -0.8653133  -0.35369848
  -0.52664951]
 ..., 
 [-0.0231531   1.04862207  0.30375387 ..., -1.05858505 -0.15990741
  -0.05112539]
 [-0.0831807   1.36102096  0.89386512 ...,  0.30678219  1.15645574
  -0.30025071]
 [-0.17060401 -0.02922701  0.12065742 ..., -0.74238374 -0.576227
  -0.69378836]]

In [ ]:

    

#mean_list = W_Embedding.mean(0)
#print mean_list

#W_Embedding_new = W_Embedding - mean_list
#print W_Embedding_new


    

In [26]:

    

inSentence = []
expSentence = []

fileIn0 = open(fin)
for l in fileIn0 :
    #l = _start+' '+l+' '+_end
    tokens = word_tokenize(l.decode('utf-8'))
    inSent = []
    for token in tokens :
        target = ""
        if token not in _vocab : 
            target = _vocab['UNK']
        else : 
            target = _vocab[token]
        vec = W_Embedding[target]
        vec_list = vec.tolist()
        inSent.append(vec_list)
    inSentence.append(inSent)

fileIn1 = open(fin1)
for l in fileIn1 :
    #l = _start+' '+l+' '+_end
    tokens = word_tokenize(l.decode('utf-8'))
    expSent = []
    for token in tokens :
        target = ""
        if token not in _vocab_f : 
            target = _vocab_f['UNK']
        else : 
            target = _vocab_f[token]
        expSent.append(target)
    expSentence.append(expSent)

#print inSentence
#print expSentence
        
a = RNNlayer(10,vocabSize_f,W_Embedding_f,idx)
a.train_Decoder_With_SGD(inSentence, expSentence, 0.1, 25)


    

    




0   123745.986541
1   125267.999661
2   129562.649855
3   128628.700571
4   126392.519699
5   128751.585344
6   126271.729103
7   127132.250987
8   127833.522475
9   126623.776688
10   131706.67726
11   126457.866613
12   124988.975815
13   123397.555218
14   132079.42921
15   128461.236529
16   125680.370529
17   125065.295053
18   122556.686152
19   128573.841962
20   125839.119875
21   127053.354018
22   126303.789215
23   129499.81539
24   131367.347243

In [34]:

    

inSentence = []
input = "American Parliament"
#target = _vocab[input]
#vec = W_Embedding_new[target]
#inSentence.append(vec)
tokens = word_tokenize(input.decode('utf-8'))
inSent = []
for token in tokens :
    target = _vocab[token]
    vec = W_Embedding[target]
    vec_list = vec.tolist()
    inSent.append(vec_list)
inSentence.append(inSent)
print inSentence

o,h = a.forwardProp(inSentence[0],None)
print o
words1 = o.argmax(axis=1)
print words1
for i in range(len(words1)) :
    print words_f[words1[i]].word


    

    




[[[-0.023195924361487638, -0.14407989065954266, 0.49097463076905157, -0.43153845176902067, 0.9646324114327803, -0.22700313499949196, 0.3492607853411439, 0.494048916612572, 0.32866152141974336, -0.5031084153348868], [0.737518627046467, -0.1031075717148144, -0.2667072526006662, -0.6326892444716452, 0.8513651367537206, -0.19226917849216824, 0.6697243635830863, 0.9355203332930365, 0.6054522914283671, -0.8305105622508467]]]
[[  1.98514423e-01   4.71606115e-04   3.05549984e-04 ...,   3.62127774e-05
    1.03240760e-04   3.62231086e-05]
 [  1.36222423e-01   6.10040712e-06   1.42902140e-04 ...,   4.02458211e-05
    4.04570752e-05   1.61371244e-04]]
[  0 372]
UNK
de