In [2]:

    

import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
import nltk
import re

punctuation = re.compile(r'[0-9]')
from nltk.sentiment.vader import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer()

def read_data(path):
    old_data = pd.DataFrame.from_csv(path)  #take first column as index
    train = old_data.head(n=200)
    
    # combine all the strings of each tuple
    train2 = train[['Label']].copy()
    
    #all the new column
#     new_column = []
    pos = []
    neg = []
    compound = []
    neutral = []
#     example = ""
    example_list = []
    for row in train.itertuples():
        for i in range(2,27):
#             example = example + row[i]
            example_list.append(row[i])
        temp1 = " "
#         print(example_list)
        example = temp1.join(example_list)
        
        #process example
#         print (example)
#         example2 = example.lower()
        example3 = CountVectorizer().build_tokenizer()(example)
        example4 = [punctuation.sub("", word) for word in example3]
        temp = " "
        example5 = temp.join(example4)
#         print(example5)
        result = sid.polarity_scores(example5)
        pos.append(result['pos'])
        neg.append(result['neg'])
        compound.append(result['compound'])
        neutral.append(result['neu'])
#         new_column.append(example)
#         example = ""
        example_list = []
    
#     train2['news']=new_column
    train2['pos']=pos
    train2['neg']=neg
    train2['compound']=compound
    train2['neutral']=neutral
    return train2

data = read_data("./Documents/Cornell/Courses/MPS Project/Combined_News_DJIA.csv")
data.to_csv("./Documents/Cornell/Courses/MPS Project/data_after_polarity.csv")
print("Done!")


    

    




Done!

In [19]:

    

train = pd.DataFrame.from_csv("./Documents/Cornell/Courses/MPS Project/Combined_News_DJIA.csv")
train.info()


    

    




<class 'pandas.core.frame.DataFrame'>
DatetimeIndex: 1989 entries, 2008-08-08 to 2016-07-01
Data columns (total 26 columns):
Label    1989 non-null int64
Top1     1989 non-null object
Top2     1989 non-null object
Top3     1989 non-null object
Top4     1989 non-null object
Top5     1989 non-null object
Top6     1989 non-null object
Top7     1989 non-null object
Top8     1989 non-null object
Top9     1989 non-null object
Top10    1989 non-null object
Top11    1989 non-null object
Top12    1989 non-null object
Top13    1989 non-null object
Top14    1989 non-null object
Top15    1989 non-null object
Top16    1989 non-null object
Top17    1989 non-null object
Top18    1989 non-null object
Top19    1989 non-null object
Top20    1989 non-null object
Top21    1989 non-null object
Top22    1989 non-null object
Top23    1989 non-null object
Top24    1987 non-null object
Top25    1987 non-null object
dtypes: int64(1), object(25)
memory usage: 419.6+ KB