In [2]:

    

import pandas as pd
import fastparquet
from gensim.models.wrappers import FastText
#import gensim

import matplotlib.pyplot as plt
from nltk.tokenize import sent_tokenize
from nltk.tokenize import word_tokenize
from nltk.stem.porter import PorterStemmer
import nltk
nltk.download('punkt')
nltk.download("stopwords") 
from nltk.corpus import stopwords

import re
import string


    

    




C:\Users\goldaw\AppData\Local\Continuum\Anaconda3\lib\site-packages\gensim\utils.py:855: UserWarning: detected Windows; aliasing chunkize to chunkize_serial
  warnings.warn("detected Windows; aliasing chunkize to chunkize_serial")






    




[nltk_data] Downloading package punkt to
[nltk_data]     C:\Users\goldaw\AppData\Roaming\nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package stopwords to
[nltk_data]     C:\Users\goldaw\AppData\Roaming\nltk_data...
[nltk_data]   Package stopwords is already up-to-date!

In [3]:

    

pd.options.mode.chained_assignment = None


    

In [4]:

    

# load data from precomputed dataframe
pfile = fastparquet.ParquetFile('5col_DFrame.parq') 
df = pfile.to_pandas() # all columns 
#df2 = pfile.to_pandas(columns=['floats', 'times']) # pick some columns


    

In [5]:

    

df.tail()


    

    
Out[5]:







  

    

      

      
sentiment
      
text
      
text_clean1
      
hashtags
      
wordlist
      
no_names_hashtags
    
  
  

    

      
10494
      
Negative
      
The question about God and the Veterans. What ...
      
question god veterans softball gopdebates
      
[gopdebates]
      
[question, god, veterans, softball]
      
question god veterans softball
    
    

      
10495
      
Negative
      
I thought #LastComicStanding airs on Wednesday...
      
thought lastcomicstanding airs wednesday night...
      
[lastcomicstanding, gopdebates]
      
[thought, airs, wednesday, nights]
      
thought airs wednesday nights
    
    

      
10496
      
Negative
      
Bingo! Put that in your article!!! #GOPDebates...
      
bingo put article gopdebates httpstcoxaaqwagf
      
[gopdebates]
      
[bingo, put, article]
      
bingo put article
    
    

      
10497
      
Negative
      
RT @RWSurferGirl: Fox is cherry picking the ca...
      
fox cherry picking candidates jeb softball que...
      
[gopdebates, gopdebates]
      
[fox, cherry, picking, candidates, jeb, softba...
      
fox cherry picking candidates jeb softball que...
    
    

      
10498
      
Neutral
      
Waiting on Trumps answer about God #GOPDebates...
      
waiting trumps answer god gopdebates aintenoug...
      
[gopdebates, aintenoughpopcornforthis]
      
[waiting, trumps, answer, god]
      
waiting trumps answer god
    
  

In [7]:

    

# write data to txt
dftrain = pd.DataFrame()
#dftrain.loc[:, 'data'] = "__" + df.loc[:,'sentiment'] + "__" + " " + df.loc[:,'text_clean1']

dftrain.loc[:, 'data'] = "__label__" + df.loc[:,'sentiment'] + " " + df.loc[:,'text_clean1']



with open('train.txt', 'w') as f:
    dftrain.iloc[0:10000].to_csv(f, header=None, index=None, sep='\n', quoting=csv.QUOTE_NONE, quotechar='')


with open('test.txt', 'w') as f:
    dftrain.iloc[10000:].to_csv(f, header=None, index=None, sep='\n', quoting=csv.QUOTE_NONE, quotechar='')


    

In [ ]: