In [11]:

    

import tweepy
import re
import json

import random

from time import sleep

import sqlite3 as lite

import datetime, time, os, sys
import argparse, ConfigParser
Config = ConfigParser.ConfigParser()
Config.read('config.cnf')

consumer_key = Config.get('surveysbot', 'consumer_key')
consumer_secret = Config.get('surveysbot', 'consumer_secret')
access_token = Config.get('surveysbot', 'access_token')
access_token_secret = Config.get('surveysbot', 'access_token_secret')

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
# set up access to the Twitter API
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)


    

In [6]:

    

user = api.get_user(screen_name='academicornot')


    

In [18]:

    

for i in range(10):
    print i, i%4


    

    




0 0
1 1
2 2
3 3
4 0
5 1
6 2
7 3
8 0
9 1

In [13]:

    

tweet = api.user_timeline(count = 1)[0]


    

In [23]:

    




    

    
Out[23]:





'api'

In [8]:

    

api.user_timeline(id = self.client_id, count = 1)[0]


    

    
Out[8]:





Status(contributors=None, truncated=False, text=u'test reply tweet', is_quote_status=False, in_reply_to_status_id=821076426157801473, id=821078420066967552, favorite_count=0, _api=<tweepy.api.API object at 0x1130d7c90>, author=User(follow_request_sent=False, has_extended_profile=False, profile_use_background_image=False, _json={u'follow_request_sent': False, u'has_extended_profile': False, u'profile_use_background_image': False, u'default_profile_image': False, u'id': 388503174, u'profile_background_image_url_https': u'https://abs.twimg.com/images/themes/theme15/bg.png', u'verified': False, u'translator_type': u'none', u'profile_text_color': u'000000', u'profile_image_url_https': u'https://pbs.twimg.com/profile_images/779936957707145216/UNvHY6nz_normal.jpg', u'profile_sidebar_fill_color': u'000000', u'entities': {u'url': {u'urls': [{u'url': u'https://t.co/2pqTi1XObo', u'indices': [0, 23], u'expanded_url': u'http://alperin.ca', u'display_url': u'alperin.ca'}]}, u'description': {u'urls': []}}, u'followers_count': 1379, u'profile_sidebar_border_color': u'000000', u'id_str': u'388503174', u'profile_background_color': u'000000', u'listed_count': 93, u'is_translation_enabled': False, u'utc_offset': -28800, u'statuses_count': 4618, u'description': u'Latin American extraordinaire; scholar; family man; publishing revolutionary; self-declared king of bocce. Tweets #openaccess #altmetrics. Part of @pkp @MpubSFU', u'friends_count': 839, u'location': u'Vancouver, British Columbia', u'profile_link_color': u'222222', u'profile_image_url': u'http://pbs.twimg.com/profile_images/779936957707145216/UNvHY6nz_normal.jpg', u'following': False, u'geo_enabled': True, u'profile_banner_url': u'https://pbs.twimg.com/profile_banners/388503174/1461175575', u'profile_background_image_url': u'http://abs.twimg.com/images/themes/theme15/bg.png', u'screen_name': u'juancommander', u'lang': u'en', u'profile_background_tile': False, u'favourites_count': 2004, u'name': u'Juan Pablo Alperin', u'notifications': False, u'url': u'https://t.co/2pqTi1XObo', u'created_at': u'Mon Oct 10 21:12:33 +0000 2011', u'contributors_enabled': False, u'time_zone': u'Pacific Time (US & Canada)', u'protected': False, u'default_profile': False, u'is_translator': False}, time_zone=u'Pacific Time (US & Canada)', id=388503174, description=u'Latin American extraordinaire; scholar; family man; publishing revolutionary; self-declared king of bocce. Tweets #openaccess #altmetrics. Part of @pkp @MpubSFU', _api=<tweepy.api.API object at 0x1130d7c90>, verified=False, profile_text_color=u'000000', profile_image_url_https=u'https://pbs.twimg.com/profile_images/779936957707145216/UNvHY6nz_normal.jpg', profile_sidebar_fill_color=u'000000', is_translator=False, geo_enabled=True, entities={u'url': {u'urls': [{u'url': u'https://t.co/2pqTi1XObo', u'indices': [0, 23], u'expanded_url': u'http://alperin.ca', u'display_url': u'alperin.ca'}]}, u'description': {u'urls': []}}, followers_count=1379, protected=False, id_str=u'388503174', default_profile_image=False, listed_count=93, lang=u'en', utc_offset=-28800, statuses_count=4618, profile_background_color=u'000000', friends_count=839, profile_link_color=u'222222', profile_image_url=u'http://pbs.twimg.com/profile_images/779936957707145216/UNvHY6nz_normal.jpg', notifications=False, default_profile=False, profile_background_image_url_https=u'https://abs.twimg.com/images/themes/theme15/bg.png', profile_banner_url=u'https://pbs.twimg.com/profile_banners/388503174/1461175575', profile_background_image_url=u'http://abs.twimg.com/images/themes/theme15/bg.png', name=u'Juan Pablo Alperin', is_translation_enabled=False, profile_background_tile=False, favourites_count=2004, screen_name=u'juancommander', url=u'https://t.co/2pqTi1XObo', created_at=datetime.datetime(2011, 10, 10, 21, 12, 33), contributors_enabled=False, location=u'Vancouver, British Columbia', profile_sidebar_border_color=u'000000', translator_type=u'none', following=False), _json={u'contributors': None, u'truncated': False, u'text': u'test reply tweet', u'is_quote_status': False, u'in_reply_to_status_id': 821076426157801473, u'id': 821078420066967552, u'favorite_count': 0, u'source': u'<a href="http://alperin.ca" rel="nofollow">juancommander</a>', u'retweeted': False, u'coordinates': None, u'entities': {u'symbols': [], u'user_mentions': [], u'hashtags': [], u'urls': []}, u'in_reply_to_screen_name': u'juancommander', u'in_reply_to_user_id': 388503174, u'retweet_count': 0, u'id_str': u'821078420066967552', u'favorited': False, u'user': {u'follow_request_sent': False, u'has_extended_profile': False, u'profile_use_background_image': False, u'default_profile_image': False, u'id': 388503174, u'profile_background_image_url_https': u'https://abs.twimg.com/images/themes/theme15/bg.png', u'verified': False, u'translator_type': u'none', u'profile_text_color': u'000000', u'profile_image_url_https': u'https://pbs.twimg.com/profile_images/779936957707145216/UNvHY6nz_normal.jpg', u'profile_sidebar_fill_color': u'000000', u'entities': {u'url': {u'urls': [{u'url': u'https://t.co/2pqTi1XObo', u'indices': [0, 23], u'expanded_url': u'http://alperin.ca', u'display_url': u'alperin.ca'}]}, u'description': {u'urls': []}}, u'followers_count': 1379, u'profile_sidebar_border_color': u'000000', u'id_str': u'388503174', u'profile_background_color': u'000000', u'listed_count': 93, u'is_translation_enabled': False, u'utc_offset': -28800, u'statuses_count': 4618, u'description': u'Latin American extraordinaire; scholar; family man; publishing revolutionary; self-declared king of bocce. Tweets #openaccess #altmetrics. Part of @pkp @MpubSFU', u'friends_count': 839, u'location': u'Vancouver, British Columbia', u'profile_link_color': u'222222', u'profile_image_url': u'http://pbs.twimg.com/profile_images/779936957707145216/UNvHY6nz_normal.jpg', u'following': False, u'geo_enabled': True, u'profile_banner_url': u'https://pbs.twimg.com/profile_banners/388503174/1461175575', u'profile_background_image_url': u'http://abs.twimg.com/images/themes/theme15/bg.png', u'screen_name': u'juancommander', u'lang': u'en', u'profile_background_tile': False, u'favourites_count': 2004, u'name': u'Juan Pablo Alperin', u'notifications': False, u'url': u'https://t.co/2pqTi1XObo', u'created_at': u'Mon Oct 10 21:12:33 +0000 2011', u'contributors_enabled': False, u'time_zone': u'Pacific Time (US & Canada)', u'protected': False, u'default_profile': False, u'is_translator': False}, u'geo': None, u'in_reply_to_user_id_str': u'388503174', u'lang': u'en', u'created_at': u'Mon Jan 16 19:35:24 +0000 2017', u'in_reply_to_status_id_str': u'821076426157801473', u'place': None}, coordinates=None, entities={u'symbols': [], u'user_mentions': [], u'hashtags': [], u'urls': []}, in_reply_to_screen_name=u'juancommander', id_str=u'821078420066967552', retweet_count=0, in_reply_to_user_id=388503174, favorited=False, source_url=u'http://alperin.ca', user=User(follow_request_sent=False, has_extended_profile=False, profile_use_background_image=False, _json={u'follow_request_sent': False, u'has_extended_profile': False, u'profile_use_background_image': False, u'default_profile_image': False, u'id': 388503174, u'profile_background_image_url_https': u'https://abs.twimg.com/images/themes/theme15/bg.png', u'verified': False, u'translator_type': u'none', u'profile_text_color': u'000000', u'profile_image_url_https': u'https://pbs.twimg.com/profile_images/779936957707145216/UNvHY6nz_normal.jpg', u'profile_sidebar_fill_color': u'000000', u'entities': {u'url': {u'urls': [{u'url': u'https://t.co/2pqTi1XObo', u'indices': [0, 23], u'expanded_url': u'http://alperin.ca', u'display_url': u'alperin.ca'}]}, u'description': {u'urls': []}}, u'followers_count': 1379, u'profile_sidebar_border_color': u'000000', u'id_str': u'388503174', u'profile_background_color': u'000000', u'listed_count': 93, u'is_translation_enabled': False, u'utc_offset': -28800, u'statuses_count': 4618, u'description': u'Latin American extraordinaire; scholar; family man; publishing revolutionary; self-declared king of bocce. Tweets #openaccess #altmetrics. Part of @pkp @MpubSFU', u'friends_count': 839, u'location': u'Vancouver, British Columbia', u'profile_link_color': u'222222', u'profile_image_url': u'http://pbs.twimg.com/profile_images/779936957707145216/UNvHY6nz_normal.jpg', u'following': False, u'geo_enabled': True, u'profile_banner_url': u'https://pbs.twimg.com/profile_banners/388503174/1461175575', u'profile_background_image_url': u'http://abs.twimg.com/images/themes/theme15/bg.png', u'screen_name': u'juancommander', u'lang': u'en', u'profile_background_tile': False, u'favourites_count': 2004, u'name': u'Juan Pablo Alperin', u'notifications': False, u'url': u'https://t.co/2pqTi1XObo', u'created_at': u'Mon Oct 10 21:12:33 +0000 2011', u'contributors_enabled': False, u'time_zone': u'Pacific Time (US & Canada)', u'protected': False, u'default_profile': False, u'is_translator': False}, time_zone=u'Pacific Time (US & Canada)', id=388503174, description=u'Latin American extraordinaire; scholar; family man; publishing revolutionary; self-declared king of bocce. Tweets #openaccess #altmetrics. Part of @pkp @MpubSFU', _api=<tweepy.api.API object at 0x1130d7c90>, verified=False, profile_text_color=u'000000', profile_image_url_https=u'https://pbs.twimg.com/profile_images/779936957707145216/UNvHY6nz_normal.jpg', profile_sidebar_fill_color=u'000000', is_translator=False, geo_enabled=True, entities={u'url': {u'urls': [{u'url': u'https://t.co/2pqTi1XObo', u'indices': [0, 23], u'expanded_url': u'http://alperin.ca', u'display_url': u'alperin.ca'}]}, u'description': {u'urls': []}}, followers_count=1379, protected=False, id_str=u'388503174', default_profile_image=False, listed_count=93, lang=u'en', utc_offset=-28800, statuses_count=4618, profile_background_color=u'000000', friends_count=839, profile_link_color=u'222222', profile_image_url=u'http://pbs.twimg.com/profile_images/779936957707145216/UNvHY6nz_normal.jpg', notifications=False, default_profile=False, profile_background_image_url_https=u'https://abs.twimg.com/images/themes/theme15/bg.png', profile_banner_url=u'https://pbs.twimg.com/profile_banners/388503174/1461175575', profile_background_image_url=u'http://abs.twimg.com/images/themes/theme15/bg.png', name=u'Juan Pablo Alperin', is_translation_enabled=False, profile_background_tile=False, favourites_count=2004, screen_name=u'juancommander', url=u'https://t.co/2pqTi1XObo', created_at=datetime.datetime(2011, 10, 10, 21, 12, 33), contributors_enabled=False, location=u'Vancouver, British Columbia', profile_sidebar_border_color=u'000000', translator_type=u'none', following=False), geo=None, in_reply_to_user_id_str=u'388503174', lang=u'en', created_at=datetime.datetime(2017, 1, 16, 19, 35, 24), in_reply_to_status_id_str=u'821076426157801473', place=None, source=u'juancommander', retweeted=False)

In [2]:

    

variants = range(1,7) + range(13,19)
random.shuffle(variants) # so that we don't test variant 1 more than the others on restarts

print variants


    

    




[4, 6, 1, 15, 2, 14, 17, 13, 5, 16, 3, 18]

In [36]:

    

#juancommander
consumer_key = 'DTHU5ejF9M83tS8JWeoeFGvix'
consumer_secret = 'bQH60O4NEcigLaE7wEbf9InfoqW5x27fUe69SMLpTmaWY1N6mk'
access_token = '388503174-ILHzQL5HmYyWOTAkZ5nm9aVLWwjD0dprkQDxc8CD'
access_token_secret = 'O0ZQJToztY4V4JgWKiSl5wh1ajvweuDSsrYOGmStwifu9'

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api_juancommander = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)


    

In [3]:

    

#sfupubresearch
consumer_key = 'x4x6rrFokpD3awsx4UYeSYiha'
consumer_secret = 'XjPQsXHxYzUhpQW3pUGtoqeqpj9w32udQyOrpJtZM1wUwFQE0x'
access_token = '707775458608164864-58j5LZ4Sryh0kKyru2444AHsZSNGASS'
access_token_secret = 'AXAXsrV2B2nZbDmvKhCCD5dILeFfUzK5fSJgbjieNWmqE'

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api2 = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)


    

In [7]:

    

mentions = api2.mentions_timeline(count = 10)


    

In [8]:

    

api.update_statusapi.update_status('Goodnight @juancommander. %s' %)


    

    
Out[8]:





769431422360752128

In [39]:

    

def test_mentions():
    t = 'ah ah ah ah staying alive @sfupubresearch, staying alive %s %s' % ( datetime.datetime.now().isoformat(), ''.join([random.choice('abcdefghijklmnopqrstuvwxyz') for x in range(8)]))
    testtweet = api.update_status(t)

    for i in [1,2]:
        time.sleep(30)
        mentions = api2.mentions_timeline(count = 5)
        for mention in mentions: 
            if mention.id == testtweet.id:
                api.destroy_status(testtweet.id)
                print '%s was mentioned' % testtweet.id
                return True
    
    print 'mentions not working as of %s' & datetime.datetime.now().isoformat()
    return False

while not test_mentions()
    time.sleep(60*60)


    

    




---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-39-d3c1a31e7a36> in <module>()
     13     return false
     14 
---> 15 print test_mentions()

<ipython-input-39-d3c1a31e7a36> in test_mentions()
     10                 api_juancommander.destroy_status(testtweet.id)
     11                 print '%s was mentioned' % testtweet.id
---> 12                 return true
     13     return false
     14 

NameError: global name 'true' is not defined





    




769437768237715456 was mentioned

In [58]:

    

consumer_key = Config.get('twittersfupubresearch', 'consumer_key')
consumer_secret = Config.get('twittersfupubresearch', 'consumer_secret')
access_token = Config.get('twittersfupubresearch', 'access_token')
access_token_secret = Config.get('twittersfupubresearch', 'access_token_secret')

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
# set up access to the Twitter API
api2 = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)


    

In [61]:

    

for i in [1,2,3]:
        print '.'*i + ' ',


    

    




.  ..  ... 

In [31]:

    

screen_name = 'stefhaustein'
tweet = api.update_status("It is me again. Doing another test. Did you get a notification this time? Can you reply?\n\nThanks @%s!" % screen_name);


    

In [3]:

    

import pandas as pd
con = lite.connect('data/slate.db')


    

In [5]:

    

def grouped_first(grouped):
    if len(grouped) > 0:
        return grouped.iloc[0]
    return np.nan


    

In [4]:

    

df = pd.read_sql("SELECT id_str, tweet FROM tweet_data WHERE tweet IS NOT NULL ", con, index_col='id_str')
    
df['tweet'] = df.tweet.apply(lambda x: json.loads(x) if x is not None else None)
df['user_id_str'] = df.tweet.apply(lambda x: x['user']['id_str'])


    

In [6]:

    

df = df.reset_index().groupby('user_id_str').agg(grouped_first).reset_index()
df['created_at'] = df.tweet.apply(lambda x: time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(x['created_at'],'%a %b %d %H:%M:%S +0000 %Y')))
df['created_at_dayofweek'] = df.tweet.apply(lambda x: x['created_at'][0:3])
df['screen_name'] = df.tweet.apply(lambda x: x['user']['screen_name'])

df['user_utc_offset'] = df.tweet.apply(lambda x: x['user']['utc_offset'])
df['user_followers_count'] = df.tweet.apply(lambda x: x['user']['followers_count'])
df['user_friends_count'] = df.tweet.apply(lambda x: x['user']['friends_count'])
df['user_description'] = df.tweet.apply(lambda x: re.sub( '\s+', ' ', x['user']['description']).strip())
df['is_retweet'] = df.tweet.apply(lambda x: 'retweeted_status' in x)
df['retweet_of_status_id_str'] = df.tweet.apply(lambda x: x['retweeted_status']['id_str'] if 'retweeted_status' in x else None)
df['retweet_of_screen_name'] = df.tweet.apply(lambda x: x['retweeted_status']['user']['screen_name'] if 'retweeted_status' in x else None)
df['is_reply'] = df.tweet.apply(lambda x: x['in_reply_to_status_id'] != None)
df['in_reply_to_status_id_str'] = df.tweet.apply(lambda x: x['in_reply_to_status_id_str'])
df['in_reply_to_screen_name'] = df.tweet.apply(lambda x: x['in_reply_to_screen_name'])
df['event_number'] = df.index.map(lambda x: df.index.get_loc(x))
df['text'] = df.tweet.apply(lambda x: re.sub( '\s+', ' ', x['text']).strip()) # remove commas for CSV simplicity


    

In [7]:

    

print len(df)


    

    




35618

In [8]:

    

variants = range(1,7) + range(13,19)
def get_variant(i):
    return variants[i % len(variants)]

def get_tweet_question(variant, screen_name):
    tw = lead = question = prepend = append = ''
    
    if (variant <= 12):
        lead = "Please help us understand Slate readers."
    else:
        lead = "You recently shared a Slate article, could you tell us:"
    
    if (variant % 6 == 1 or variant % 6 == 2):
        question = "How do you usually find articles from Slate?"
    
    elif (variant %6 == 3 or variant %6 == 4):
        question = "Is Twitter the way you usually find articles from Slate?"
    
    elif (variant %6 == 5 or variant %6 == 0):
        question = "Do you read Slate Magazine often, sometimes, or never?"

    if send_as_reply(variant):
        prepend = '@' + screen_name + ' '
        append = "Thanks!"
    
    else:
        in_reply_to_status_id = None
        append = "Thanks @" + screen_name + "!"
    
    if (variant % 12 <= 6 and variant % 12 > 0):
        tw = prepend + lead + " " + question + " " + append
    else:
        tw = prepend + lead + " " + append
    
    return tw
        
def send_as_reply(variant):
    return (variant %2 == 1)

df.reset_index(inplace=True)
df['variant'] = df.index.map(lambda x: get_variant(df.index.get_loc(x)))
df['as_reply'] = df.variant.map(send_as_reply)
df['tweet_question'] = df.apply(lambda row: get_tweet_question(row['variant'], row['screen_name']), axis=1)


    

In [9]:

    

df[['id_str', 'variant', 'tweet_question', 'as_reply']].to_csv('data/questions_for_slate_bot.csv', encoding='utf8')


    

In [10]:

    

df.tweet_question.map(len).max()


    

    
Out[10]:





137

In [11]:

    

len(df)


    

    
Out[11]:





35618