In [1]:

    

import tweepy
import time
import sys
import pickle
import datetime
import pandas


    

In [3]:

    

path="./data/"


    

In [4]:

    

filename="auth_data"
fileobject=open(path+filename,"rb")
auth_data=pickle.load(fileobject)
fileobject.close()


    

In [5]:

    

#account=auth_data1['account']
CONSUMER_KEY=auth_data['CONSUMER_KEY']
CONSUMER_SECRET=auth_data['CONSUMER_SECRET']
ACCESS_KEY=auth_data['ACCESS_KEY']
ACCESS_SECRET=auth_data['ACCESS_SECRET']


    

In [6]:

    

auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)

api = tweepy.API(auth)


    

In [7]:

    

#Error handling
if (not api):
    print ("Problem tconnecting to API")


    

In [34]:

    

columns=['id','created_at','description','screen_name','name','protected','verified','lang',
         'followers_count','friends_count','statuses_count','favourites_count'] 
#fa_users_data=pandas.DataFrame(columns=columns)


    

In [94]:

    

#load from fa_users_data pickle
filename="fa_users_data"
fa_users_data=pandas.read_pickle(path+filename)


    

In [98]:

    

len(fa_users_data.id)


    

    
Out[98]:





1187

In [97]:

    

for t in tweepy.Cursor(api.search,q="*",lang="fa").items(1000):
    user=t.user
    if not (user.id in fa_users_data['id'].tolist()):
        user_row=[getattr(user,label) for label in columns]
        fa_users_data.loc[len(fa_users_data.index)]=user_row


    

In [100]:

    

# above omitted for brevity
c = tweepy.Cursor(api.search,q="*",lang="fa").items()
while True:
    try:
        t = c.next()
        user=t.user
        if not (user.id in fa_users_data['id'].tolist()):
            user_row=[getattr(user,label) for label in columns]
            fa_users_data.loc[len(fa_users_data.index)]=user_row
    except tweepy.TweepError:
        #save fa_users_data
        filename="fa_users_data"
        fa_users_data.to_pickle(path+filename)
        print(len(fa_users_data.id))
        time.sleep(60 * 15)
        continue
    except StopIteration:
        break


    

    




1861
2350
2804






    




---------------------------------------------------------------------------
TweepError                                Traceback (most recent call last)
<ipython-input-100-8c1a46ffa03d> in <module>()
      4     try:
----> 5         t = c.next()
      6         user=t.user

/home/alireza/anaconda3/lib/python3.6/site-packages/tweepy/cursor.py in next(self)
    196             # Reached end of current page, get the next page...
--> 197             self.current_page = self.page_iterator.next()
    198             self.page_index = -1

/home/alireza/anaconda3/lib/python3.6/site-packages/tweepy/cursor.py in next(self)
    107         if self.index >= len(self.results) - 1:
--> 108             data = self.method(max_id=self.max_id, parser=RawParser(), *self.args, **self.kargs)
    109 

/home/alireza/anaconda3/lib/python3.6/site-packages/tweepy/binder.py in _call(*args, **kwargs)
    244         else:
--> 245             return method.execute()
    246 

/home/alireza/anaconda3/lib/python3.6/site-packages/tweepy/binder.py in execute(self)
    228                 else:
--> 229                     raise TweepError(error_msg, resp, api_code=api_error_code)
    230 

TweepError: Twitter error response: status code = 429

During handling of the above exception, another exception occurred:

KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-100-8c1a46ffa03d> in <module>()
     13         fa_users_data.to_pickle(path+filename)
     14         print(len(fa_users_data.id))
---> 15         time.sleep(60 * 15)
     16         continue
     17     except StopIteration:

KeyboardInterrupt: 

In [99]:

    

#save fa_users_data
filename="fa_users_data"
fa_users_data.to_pickle(path+filename)