In [5]:

    

# imports

import urllib2
import json


    

In [6]:

    

# get an s3 file from Discursive. This file updates every hour. 

response = urllib2.urlopen('https://s3-us-west-2.amazonaws.com/discursive/womensmarch-tweets.txt')
html = response.read()
resp = json.loads(html)


    

In [7]:

    

# have a look at the data

print resp[:5]


    

    




[{u'sort': [4229651], u'_type': u'message', u'_index': u'twitter', u'_score': None, u'_source': {u'topics': [u'womensmarch', u'womensrights'], u'hashtags': u'["WomensMarch"]', u'created': u'2017-01-22T02:15:01', u'message': u'RT @clapbackanna: Rihanna protesting in NYC #WomensMarch https://t.co/5qnX5JkMy2', u'name': u'_insiang'}, u'_id': u'AVnD9JX7ibJMrT03fezr'}, {u'sort': [4229653], u'_type': u'message', u'_index': u'twitter', u'_score': None, u'_source': {u'topics': [u'womensmarch', u'womensrights'], u'hashtags': u'[]', u'created': u'2017-01-22T02:15:01', u'message': u"people so tight about this y'all literally threatened Obamas life every damn minute sit your snowflake ass down https://t.co/SoI8OQFHDZ", u'name': u'amenmaguire'}, u'_id': u'AVnD9JZTibJMrT03fezt'}, {u'sort': [4229656], u'_type': u'message', u'_index': u'twitter', u'_score': None, u'_source': {u'topics': [u'womensmarch', u'womensrights'], u'hashtags': u'["WMWArt", "WomensMarch"]', u'created': u'2017-01-22T02:15:01', u'message': u'RT @womensmarch: Show us your amazing art work and use hashtag #WMWArt! #WomensMarch https://t.co/dhOGPOWOd6', u'name': u'xovictoriaa__'}, u'_id': u'AVnD9JaribJMrT03fezw'}, {u'sort': [4229660], u'_type': u'message', u'_index': u'twitter', u'_score': None, u'_source': {u'topics': [u'womensmarch', u'womensrights'], u'hashtags': u'["WomensMarch"]', u'created': u'2017-01-22T02:15:01', u'message': u'RT @markiplier: Headed downtown to join the #WomensMarch and give you guys some insight to the scope of this.', u'name': u'Gamergirl9005'}, u'_id': u'AVnD9JcNibJMrT03fez0'}, {u'sort': [4229662], u'_type': u'message', u'_index': u'twitter', u'_score': None, u'_source': {u'topics': [u'womensmarch', u'womensrights'], u'hashtags': u'["WomensMarch"]', u'created': u'2017-01-22T02:15:01', u'message': u'RT @owillis: lol conservatives hand-wringing over "vulgar signs" at #WomensMarch after months of "trump that bitch"', u'name': u'AMWeinstein'}, u'_id': u'AVnD9JdnibJMrT03fez2'}]

In [10]:

    

# get distinct messages
output = set()
for x in resp:
    output.add(x['_source']['message'])
print list(output)[:20]


    

    




[u"RT @TEN_GOP: Head of #WomensMarch is mad that 22 states don't allow Sharia Law which is totally against women's rights. Take a m\u2026 ", u'RT @MatthewACherry: Trump is so unpopular that Women marched against him in fucking Antarctica. #WomensMarch https://t.co/316Nahkv6y', u'Lindo #WomensMarch https://t.co/GE4w4NdImk', u'RT @17Novembre1796: Finalement la #WomensMarch est en fait la #MuslimMarch Pro #terrorisme #islamiste #Antichr\xe9tienne et #AntiAm\xe9ricaine ht\u2026', u'RT @thefifthluna: umas das minhas favoritas de hoje #WomensMarch \U0001f49c https://t.co/ZZ0g76wcYf', u'RT @thedailybeast: "Respect Women of Color" - Seen at the #WomensMarch in D.C. by @AlexBrookLynn https://t.co/LijR1FxyCh', u'RT @camrenSAC: "Todos s\xe3o bem vindos aqui, sem \xf3dio, sem medo!"\n\n\xc9 maravilhosa demais pra esse mundo \n #WomensMarch \n  https://t.co/3YLHNjI\u2026', u'RT @SeriesUpdatesFR: Candice Accola \xe0 la #WomensMarch. https://t.co/FKsIhk2nDV', u"RT @CerdagneFrance: Le voile islamique. Est-il plus beau symbole de libert\xe9 et d'\xe9mancipation de la femme ? #WomensMarch https://t.co/uT0ki\u2026", u'RT @BosWomensMarch: \U0001f4aa\U0001f3fd\U0001f44d\U0001f3fc https://t.co/kmzt9RgRCv', u'RT @Clexa_Brasil: Orgulho da Melissa. #WomensMarch https://t.co/VkYM5KEW9B', u'RT @lsarsour: Alaska. #womensmarch https://t.co/qnXnAKlpZD', u'#Proud #WomensMarch https://t.co/5gj9wqzvxw', u"RT @TrumpWhiteHouse: You are marching for womens' rights, yet glorifying a religion that enslaves and oppresses women. Really?\u2026 ", u'RT @HogwartsTM: Emma Watson en la marcha de las mujeres contra Donald Trump. #WomensMarch https://t.co/wIGcVq201Z', u'RT @SpenceDoginc: All this represents 2 me is selfish & dishonest ppl who R willing due 2 that 2 bring R us down. https://t.co/KEAxnFWCts', u'RT @StahrMilan: Today all the ladies need to show your girl power #WomensMarch', u'RT @madmommadmom: #WomensMarch https://t.co/Ha8dB5etjF', u'RT @ArianatorDesire: OMG we love Ariana & Nonna so much \u2764\ufe0f #WomensMarch https://t.co/WKJmgFspaX', u'RT @ish10040: "Trump Supporters Expose the Hypocrisy of the #WomensMarch" https://t.co/K223cmpwNM #alist  #feedly v']