In [1]:

    

import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import wordcloud as wc


    

In [29]:

    

path = os.path.join(os.getcwd(), "..", "param", "ap")
#alpha_path = os.path.join(path, "alpha.dat")
beta_path = os.path.join(path, "beta.dat")

#alpha = pd.read_table(alpha_path, sep=" ", header = None)
beta = pd.read_table(beta_path, sep=" ", header = None)

vocab_path = os.path.join(os.getcwd(), "..", "datasets", "ap", "vocab.txt" )
vocab = pd.read_table(vocab_path, header=None)


    

In [30]:

    

alpha


    

    
Out[30]:






  

    

      

      
0
      
1
      
2
      
3
      
4
      
5
      
6
      
7
      
8
      
9
      
...
      
90
      
91
      
92
      
93
      
94
      
95
      
96
      
97
      
98
      
99
    
  
  

    

      
0
      
1
      
1
      
1
      
1
      
1
      
1
      
1
      
1
      
1
      
1
      
...
      
1
      
1
      
1
      
1
      
1
      
1
      
1
      
1
      
1
      
1
    
  

1 rows × 100 columns

In [31]:

    

sorted_topics = alpha.transpose().sort_values(0, ascending=False).index
for topic in sorted_topics:
    top_words = beta.loc[topic].sort_values(ascending=False).index
    
    if topic < 10:
        print topic, " :" ,
    else:
        print topic , ":" , 
        
    for i in xrange(30):
        print vocab[0].loc[top_words[i]], 
    print
    print


    

    




0  : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

63 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

73 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

72 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

71 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

70 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

69 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

68 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

67 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

66 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

65 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

64 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

62 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

1  : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

61 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

60 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

59 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

58 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

57 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

56 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

55 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

54 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

53 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

52 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

74 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

75 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

76 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

77 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

98 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

97 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

96 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

95 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

94 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

93 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

92 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

91 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

90 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

89 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

88 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

87 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

86 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

85 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

84 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

83 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

82 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

81 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

80 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

79 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

78 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

51 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

50 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

49 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

24 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

22 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

21 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

20 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

19 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

18 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

17 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

16 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

15 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

14 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

13 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

12 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

11 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

10 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

9  : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

8  : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

7  : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

6  : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

5  : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

4  : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

3  : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

2  : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

23 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

25 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

48 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

26 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

47 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

46 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

45 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

44 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

43 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

42 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

41 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

40 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

39 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

38 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

37 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

36 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

35 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

34 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

33 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

32 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

31 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

30 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

29 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

28 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

27 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

99 : i new percent people year two million president last government years first police state states officials soviet united bush time three billion today national told american thursday house federal week

In [40]:

    

topic = 51
top_words = beta.loc[topic].sort_values(ascending=False).index[0:400]
word_list = []
for word_index in top_words:
    if word_index != 0:
        word_freq = (vocab[0].loc[word_index], beta.loc[topic][word_index].round(5))
        word_list.append(word_freq)
        
plt.figure(figsize=(20,10))
wordcloud = wc.WordCloud(width=800, height=400).generate_from_frequencies(word_list)
plt.imshow(wordcloud)
plt.axis("off")
# plt.savefig("topic_images/topic86.png")
plt.show()


    

In [6]:

    

sorted_topics = alpha.transpose().sort_values(0, ascending=False).index
for topic in sorted_topics:
    top_words = beta.loc[topic].sort_values(ascending=False).index[0:400]
    word_list = []
    for word_index in top_words:
        if word_index != 0:
            word_freq = (vocab[0].loc[word_index], beta.loc[topic][word_index].round(5))
            word_list.append(word_freq)
    
    if beta.loc[topic].loc[top_words[0]] - beta.loc[topic].loc[top_words[1]] > 0.01:
        image_name = vocab[0].loc[top_words[0]]
    else:
        image_name = vocab[0].loc[top_words[0]]+"_"+vocab[0].loc[top_words[1]]
        
    plt.figure(figsize=(20,10))
    wordcloud = wc.WordCloud(width=800, height=400).generate_from_frequencies(word_list)
    plt.imshow(wordcloud)
    plt.axis("off")
    plt.savefig("topic_images2/"+image_name+".png")
    plt.close()