In [1]:

    

import numpy as np
import pandas as pd
import nltk
from gen_features import FeatureGenerator
from model_loop import ModelLoop
from sklearn.model_selection import train_test_split


    

    




/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)
/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/grid_search.py:43: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.
  DeprecationWarning)

In [2]:

    

# test-train first; change to take a dataframe
# save those to to file
# call FeatureGenerator on training data; call FeatureGenerator.train() --> saves sparse matrices in attribute
# call FeatureGenerator on test data; call Feature Generator.transform(new_datafile)
# f.x --> training sparse matrix
# f.new_x --> testing sparse matrix


    

In [3]:

    

data_fp = '../data_cleaning/articles1.csv'
data = pd.read_csv(data_fp)


    

In [4]:

    

data.drop('Unnamed: 0', axis=1, inplace=True)


    

In [5]:

    

data = data.sample(n=1000)


    

In [6]:

    

X, y = train_test_split(data)


    

In [7]:

    

X.to_csv('data/X_sample.csv')


    

In [8]:

    

y.to_csv('data/y_sample.csv')


    

In [9]:

    

config = {
    'datafile': 'data/X_sample.csv',
    'test_datafile': 'data/y_sample.csv',
    'text_label': 'content',
    'y_label': 'label',
    'fts_to_try': ['CountVectorize', 'TfidfVectorize', 'CountPOS'],
}


    

In [10]:

    

args = {k: v for k, v in config.items() if k != 'test_datafile'}


    

In [11]:

    

gen = FeatureGenerator(**args)


    

    




Creating feature:  CountVectorize
Parameters:  {'ngram_range': (1, 1)}
xft size (750, 3000)
Creating feature:  TfidfVectorize
Parameters:  {'ngram_range': (1, 2)}
Creating feature:  CountPOS
Parameters:  {'language': 'english'}
6044 features generated for 750 examples

In [12]:

    

gen.transform(config['test_datafile'])


    

    




6044 features generated for 250 examples

In [13]:

    

run = 'test_3-2'
models = ['NB', 'RF', 'ET', 'LR', 'SVM', 'SGD', 'KNN']
iterations = 1
output_dir = 'output_{}/'.format(run)
ks = [0.05, 0.10]


    

In [14]:

    

loop = ModelLoop(gen.X_train, gen.X_test, gen.y_train, gen.y_test, models, iterations, output_dir,
                 ks = ks, method='csc', roc=True)


    

In [15]:

    

loop.run()


    

    




---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-15-5602f7091302> in <module>()
----> 1 loop.run()

/Users/aldengolab/winter16/amlpp/fake-news-detection/pipeline/model_loop.py in run(self)
    215         parser = spacy.load('en')
    216         f = get_feature_transformer(parser)
--> 217         self.X_train = f.fit_transform(self.raw_X_train)
    218         self.X_test = f.transform(self.raw_X_test)
    219 

/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/pipeline.py in fit_transform(self, X, y, **fit_params)
    732             delayed(_fit_transform_one)(trans, name, weight, X, y,
    733                                         **fit_params)
--> 734             for name, trans, weight in self._iter())
    735 
    736         if not result:

/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in __call__(self, iterable)
    756             # was dispatched. In particular this covers the edge
    757             # case of Parallel used with an exhausted iterator.
--> 758             while self.dispatch_one_batch(iterator):
    759                 self._iterating = True
    760             else:

/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in dispatch_one_batch(self, iterator)
    606                 return False
    607             else:
--> 608                 self._dispatch(tasks)
    609                 return True
    610 

/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in _dispatch(self, batch)
    569         dispatch_timestamp = time.time()
    570         cb = BatchCompletionCallBack(dispatch_timestamp, len(batch), self)
--> 571         job = self._backend.apply_async(batch, callback=cb)
    572         self._jobs.append(job)
    573 

/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py in apply_async(self, func, callback)
    107     def apply_async(self, func, callback=None):
    108         """Schedule a func to be run"""
--> 109         result = ImmediateResult(func)
    110         if callback:
    111             callback(result)

/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py in __init__(self, batch)
    324         # Don't delay the application, to avoid keeping the input
    325         # arguments in memory
--> 326         self.results = batch()
    327 
    328     def get(self):

/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in __call__(self)
    129 
    130     def __call__(self):
--> 131         return [func(*args, **kwargs) for func, args, kwargs in self.items]
    132 
    133     def __len__(self):

/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in <listcomp>(.0)
    129 
    130     def __call__(self):
--> 131         return [func(*args, **kwargs) for func, args, kwargs in self.items]
    132 
    133     def __len__(self):

/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/pipeline.py in _fit_transform_one(transformer, name, weight, X, y, **fit_params)
    575                        **fit_params):
    576     if hasattr(transformer, 'fit_transform'):
--> 577         res = transformer.fit_transform(X, y, **fit_params)
    578     else:
    579         res = transformer.fit(X, y, **fit_params).transform(X)

/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/pipeline.py in fit_transform(self, X, y, **fit_params)
    299         """
    300         last_step = self._final_estimator
--> 301         Xt, fit_params = self._fit(X, y, **fit_params)
    302         if hasattr(last_step, 'fit_transform'):
    303             return last_step.fit_transform(Xt, y, **fit_params)

/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/sklearn/pipeline.py in _fit(self, X, y, **fit_params)
    235             else:
    236                 Xt = transform.fit(Xt, y, **fit_params_steps[name]) \
--> 237                               .transform(Xt)
    238         if self._final_estimator is None:
    239             return Xt, {}

/Users/aldengolab/winter16/amlpp/fake-news-detection/pipeline/transform_features.py in transform(self, X, **transform_params)
    137         X can be an any iterable - array, pandas dataframe, list, etc.
    138         '''
--> 139         return [self.cleanText(text) for text in X]
    140 
    141     def fit(self, X, y=None, **fit_params):

/Users/aldengolab/winter16/amlpp/fake-news-detection/pipeline/transform_features.py in <listcomp>(.0)
    137         X can be an any iterable - array, pandas dataframe, list, etc.
    138         '''
--> 139         return [self.cleanText(text) for text in X]
    140 
    141     def fit(self, X, y=None, **fit_params):

/Users/aldengolab/miniconda3/envs/amlpp/lib/python3.6/site-packages/scipy/sparse/base.py in __iter__(self)
    146     def __iter__(self):
    147         for r in xrange(self.shape[0]):
--> 148             yield self[r, :]
    149 
    150     def getmaxprint(self):

TypeError: 'coo_matrix' object is not subscriptable

In [ ]:

    

pd.read_csv(output_dir + 'simple_report.csv', quotechar='"', skipinitialspace = True)


    

In [ ]:

    




    

In [ ]:

    




    

In [ ]: