In [2]:

    

%load_ext watermark

%watermark -a 'Vahid Mirjalili' -d -p scikit-learn,numpy,numexpr,pandas,matplotlib,plotly -v


    

    




Vahid Mirjalili 21/12/2014 

CPython 2.7.3
IPython 2.3.1

scikit-learn 0.15.2
numpy 1.9.1
numexpr 2.2.2
pandas 0.15.1
matplotlib 1.4.2
plotly 1.4.7

In [3]:

    

from matplotlib import pyplot as plt

import pandas as pd
import numpy as np
import scipy
import sklearn

%matplotlib inline


    

In [4]:

    

df = pd.read_table('../data/labeledTrainData.tsv')

df.head()


    

    
Out[4]:






  

    

      

      
id
      
sentiment
      
review
    
  
  

    

      
0
      
 5814_8
      
 1
      
 With all this stuff going down at the moment w...
    
    

      
1
      
 2381_9
      
 1
      
 \The Classic War of the Worlds\" by Timothy Hi...
    
    

      
2
      
 7759_3
      
 0
      
 The film starts with a manager (Nicholas Bell)...
    
    

      
3
      
 3630_4
      
 0
      
 It must be assumed that those who praised this...
    
    

      
4
      
 9495_8
      
 1
      
 Superbly trashy and wondrously unpretentious 8...
    
  

In [5]:

    

X_train = df.loc[:, 'review']

y_train = df.loc[:, 'sentiment']

X_train.head()


    

    
Out[5]:





0    With all this stuff going down at the moment w...
1    \The Classic War of the Worlds\" by Timothy Hi...
2    The film starts with a manager (Nicholas Bell)...
3    It must be assumed that those who praised this...
4    Superbly trashy and wondrously unpretentious 8...
Name: review, dtype: object

In [6]:

    

import nltk
import string
import re
from collections import Counter

from nltk.corpus import stopwords


    

In [7]:

    

## For more info, see http://www.cs.duke.edu/courses/spring14/compsci290/assignments/lab02.html

stemmer = nltk.stem.porter.PorterStemmer()

def get_tokens(inp_txt):
    
    ## Lower case: ABC -> abc
    txt_lower = inp_txt.lower()
  
    ## Remove punctuations (!, ', ", ., :, ;, )
    #txt_lower_nopunct = txt_lower.translate(string.maketrans("",""), string.punctuation)
    #print(txt_lower_nopunct)
    
    
    ## Tokenize:
    tokens = nltk.word_tokenize(txt_lower) #_nopunct)
    #tokens = nltk.wordpunct_tokenize(txt_lower)
    
    ## remove stop-words:
    tokens_filtered = [w for w in tokens if not w in stopwords.words('english')]
    
    ## stemming:
    stems = [stemmer.stem(t) for t in tokens_filtered]
    stems_nopunct = [s for s in stems if re.match('^[a-zA-Z]+$', s) is not None]
    return (stems_nopunct)


    

In [8]:

    

get_tokens("What's in a name? That which we call a rose by any other name would smell as sweet.")

## Note: you need to download punkt package in nltk:
# import nltk
# nltk.download(punkt)


    

    
Out[8]:





[u'name', u'call', u'rose', u'name', u'would', u'smell', u'sweet']

In [9]:

    

tfidf = sklearn.feature_extraction.text.TfidfVectorizer(
    encoding = 'utf-8',
    decode_error = 'replace',
    strip_accents = 'ascii',
    analyzer = 'word',
    smooth_idf = True,
    tokenizer = get_tokens
)

tfidf


    

    
Out[9]:





TfidfVectorizer(analyzer='word', binary=False, charset=None,
        charset_error=None, decode_error='replace',
        dtype=<type 'numpy.int64'>, encoding='utf-8', input=u'content',
        lowercase=True, max_df=1.0, max_features=None, min_df=1,
        ngram_range=(1, 1), norm=u'l2', preprocessor=None, smooth_idf=True,
        stop_words=None, strip_accents='ascii', sublinear_tf=False,
        token_pattern=u'(?u)\\b\\w\\w+\\b',
        tokenizer=<function get_tokens at 0x8c7f848>, use_idf=True,
        vocabulary=None)

In [10]:

    

## Shakespear quote
example_txt_1 = "What's in a name? That which we call a rose by any other name would smell as sweet."
example_txt_2 = "To be, or not to be: that is the question."

tfidf = tfidf.fit([example_txt_1 + example_txt_2])

example1 = tfidf.transform([example_txt_1])
example2 = tfidf.transform([example_txt_2])

print('Features: %s' %tfidf.get_feature_names())
print('Example1: %s' %example1.toarray())
print('Example2: %s' %example2.toarray())


    

    




Features: [u'call', u'name', u'question', u'rose', u'smell', u'would']
Example1: [[ 0.35355339  0.70710678  0.          0.35355339  0.35355339  0.35355339]]
Example2: [[ 0.  0.  1.  0.  0.  0.]]

In [12]:

    

tfidf_train = tfidf.fit_transform(X_train.ravel())

print('Feature-set size: %s' %len(tfidf_train.get_feature_names()))


    

    




---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-12-b7100aec5d86> in <module>()
      1 tfidf_train = tfidf.fit_transform(X_train.ravel())
      2 
----> 3 print('Feature-set size: %s' %len(tfidf_train.get_feature_names()))

/usr/local/lib/python2.7/dist-packages/scipy/sparse/base.pyc in __getattr__(self, attr)
    497             return self.getnnz()
    498         else:
--> 499             raise AttributeError(attr + " not found")
    500 
    501     def transpose(self):

AttributeError: get_feature_names not found

In [ ]:

    

import pickle

pkl_out = open('../data/pickle/tfidf_object.pkl', 'w')
pickle.dump(tfidf, pkl_out)
pkl_out.close()