notebook.community

Edit and run



In [1]:

    
import pandas as pd
import numpy as np



In [8]:

    
legislatorsData = pd.read_csv("../data/legislators.csv")
legislatorsData.head()
legislatorsData.columns
legislators = pd.DataFrame(legislatorsData)
legislators.head()









    Out[8]:






  
    
      
      title
      firstname
      middlename
      lastname
      name_suffix
      nickname
      party
      state
      district
      in_office
      ...
      govtrack_id
      crp_id
      twitter_id
      congresspedia_url
      youtube_url
      facebook_id
      official_rss
      senate_class
      birthdate
      oc_email
    
  
  
    
      0
      Rep
      Neil
      NaN
      Abercrombie
      NaN
      NaN
      D
      HI
      1
      0
      ...
      400001
      N00007665
      neilabercrombie
      http://www.opencongress.org/wiki/Neil_Abercrombie
      http://youtube.com/hawaiirep1
      NaN
      NaN
      NaN
      1938-06-26
      NaN
    
    
      1
      Rep
      Gary
      L.
      Ackerman
      NaN
      NaN
      D
      NY
      5
      0
      ...
      400003
      N00001143
      repgaryackerman
      http://www.opencongress.org/wiki/Gary_Ackerman
      http://youtube.com/RepAckerman
      RepAcherman
      NaN
      NaN
      1942-11-19
      NaN
    
    
      2
      Rep
      Robert
      B.
      Aderholt
      NaN
      NaN
      R
      AL
      4
      1
      ...
      400004
      N00003028
      Robert_Aderholt
      http://www.opencongress.org/wiki/Robert_Aderholt
      http://youtube.com/RobertAderholt
      19787529402
      NaN
      NaN
      1965-07-22
      Rep.Aderholt@opencongress.org
    
    
      3
      Sen
      Daniel
      Kahikina
      Akaka
      NaN
      NaN
      D
      HI
      Junior Seat
      0
      ...
      300001
      N00007653
      NaN
      http://www.opencongress.org/wiki/Daniel_Akaka
      http://youtube.com/senatorakaka
      danielakaka
      NaN
      I
      1924-09-11
      NaN
    
    
      4
      Sen
      Wayne
      A.
      Allard
      NaN
      NaN
      R
      CO
      Senior Seat
      0
      ...
      300003
      N00009082
      NaN
      http://www.opencongress.org/wiki/Wayne_Allard
      NaN
      NaN
      NaN
      II
      1943-12-02
      NaN
    
  

5 rows × 29 columns



In [12]:

    
legislators.columns.tolist()









    Out[12]:





['title',
 'firstname',
 'middlename',
 'lastname',
 'name_suffix',
 'nickname',
 'party',
 'state',
 'district',
 'in_office',
 'gender',
 'phone',
 'fax',
 'website',
 'webform',
 'congress_office',
 'bioguide_id',
 'votesmart_id',
 'fec_id',
 'govtrack_id',
 'crp_id',
 'twitter_id',
 'congresspedia_url',
 'youtube_url',
 'facebook_id',
 'official_rss',
 'senate_class',
 'birthdate',
 'oc_email']



In [30]:

    
print legislators.bioguide_id.head()
l_bioGuides = legislators.bioguide_id.tolist()
l_bioGuides[:3]
print "Bio Guides available for the congress :", len(l_bioGuides)









    



0    A000014
1    A000022
2    A000055
3    A000069
4    A000109
Name: bioguide_id, dtype: object
Bio Guides available for the congress : 897



In [172]:

    
from urllib2 import Request, urlopen
import json
from pandas.io.json import json_normalize

request=Request('http://capitolwords.org/api/1/phrases.json?entity_type=month&entity_value=201007&sort=count+desc&apikey=0bf8e7eb6ce146f48217bfee767c998d')

response = urlopen(request)
contents = response.read()
data = json.loads(contents)
print "Example object and query to the sunlight API:"
print data[:5]









    



Example object and query to the sunlight API:
[{u'tfidf': 3.85965571248e-05, u'count': 5373, u'ngram': u'people'}, {u'tfidf': 1.30267768302e-05, u'count': 3637, u'ngram': u'one'}, {u'tfidf': 2.52066478599e-05, u'count': 3509, u'ngram': u'jobs'}, {u'tfidf': 1.17409333103e-05, u'count': 3278, u'ngram': u'american'}, {u'tfidf': 1.14866299957e-05, u'count': 3207, u'ngram': u'years'}]



In [42]:

    
first_table_ever = json_normalize(data)
print "After converting the content of the request in to a DataFrame\n Popular words of the month \n 100 only prionting a few:"
print "The shape is: ",first_table_ever.shape
first_table_ever.head(8)









    



After converting the content of the request in to a DataFrame
 Popular words of the month 
 100 only prionting a few:
The shape is:  (100, 3)






    Out[42]:






  
    
      
      count
      ngram
      tfidf
    
  
  
    
      0
      5373
      people
      0.000039
    
    
      1
      3637
      one
      0.000013
    
    
      2
      3509
      jobs
      0.000025
    
    
      3
      3278
      american
      0.000012
    
    
      4
      3207
      years
      0.000011
    
    
      5
      3051
      going
      0.000022
    
    
      6
      2874
      work
      0.000010
    
    
      7
      2874
      support
      0.000010



In [51]:

    
print "Yes they are unique"
len(first_table_ever.ngram.unique())









    



Yes they are unique






    Out[51]:





100



In [54]:

    
print "One legislator fav words"

request=Request('http://capitolwords.org/api/1/phrases.json?entity_type=legislator&entity_value=L000551&apikey=0bf8e7eb6ce146f48217bfee767c998d')

response = urlopen(request)
contents = response.read()
data = json.loads(contents)
print data[:5]
len(data)









    



One legislator fav words
[{u'tfidf': 0.00227655265553, u'count': 847, u'ngram': u'oakland'}, {u'tfidf': 0.00099708278004, u'count': 218, u'ngram': u'alameda'}, {u'tfidf': 0.000860881733709, u'count': 948, u'ngram': u'aids'}, {u'tfidf': 0.000711050866521, u'count': 487, u'ngram': u'hiv'}, {u'tfidf': 0.000699507858333, u'count': 431, u'ngram': u'haiti'}]






    Out[54]:





100



In [70]:

    
legislator_example = json_normalize(data)
legislator_example.head()
list_of_words = legislator_example.ngram.tolist()
favorite_words ="|".join(list_of_words)
print "list of favorite words of a legislator:"
favorite_words









    



list of favorite words of a legislator:






    Out[70]:





u"oakland|alameda|aids|hiv|haiti|berkeley|african|caribbean|dellums|congresswoman|haitian|pandemic|bay|caucus|black|darfur|africa|iraq|genocide|disparities|california's|occupation|francisco|asian|sudan|gentlewoman|california|housing|color|naacp|cuba|racism|poverty|slavery|peace|san|bush|african-american|reverend|rights|baptist|really|unemployed|health|racial|height|civil|minorities|global|equality|women|iran|justice|movement|troops|uninsured|tonight|pacific|pastor|ron|congressman|social|discrimination|violence|waters|progressive|education|humanitarian|human|affordable|international|salute|church|afghanistan|jackson|girls|port|community|king|prescription|war|res|low-income|communities|weapons|unemployment|organizing|women's|security|poor|youth|drugs|nations|h.|frankly|nuclear|parks|green|resolution|congressional"



In [148]:

    
legislators_option1 = legislators
def requestWords( id ):
    id = str(id)
    url = "http://capitolwords.org/api/1/phrases.json?entity_type=legislator&entity_value="+id+"&apikey=0bf8e7eb6ce146f48217bfee767c998d"
    request=Request(url)
    response = urlopen(request)
    contents = response.read()
    len(contents)
    if len(contents) > 2:
        data = json.loads(contents)
        words = json_normalize(data)
        list_of_words = words.ngram.tolist()
        string_of_words ="|".join(list_of_words)
        return string_of_words
    else:
        return np.nan

requestWords(id ="A000369")









    



2






    Out[148]:





nan



In [153]:

    
legislators_option1.dtypes
legislators_option1.bioguide_id.astype(str)
legislators_option1.dtypes
legislators_option1['favorite_words'] = legislators_option1.apply(lambda row: requestWords(row['bioguide_id']),axis=1)



In [155]:

    
legislators_option1.favorite_words.head(20)









    Out[155]:





0     hawaiian|hawaii|hawaiians|hawaii's|kalaupapa|e...
1     queens|rabbi|jewish|bayside|flushing|nassau|br...
2     aderholt|requesting|irons|huntsville|alabama|r...
3     hawaii's|hawaii|hawaiians|hawaiian|dsh|va|fas|...
4     colorado|flats|missile|rocky|colorado's|denver...
5     camden|gloucester|cyprus|rutgers|opic|jersey|p...
6     mercury|maine|prescription|pharmaceutical|drug...
7     freddie|morgenthau|fannie|you've|pilgrims|mayf...
8     tennesseans|carbon-free|tennessee|electricity|...
9     rodney|baton|rouge|lsu|louisiana|la|ruston|req...
10    murphy|drill|pittsburgh|altmire|sbir|oil|anwr|...
11    upstate|herkimer|utica|tay-sachs|suny|cybersec...
12    nj|2009|objection|army|minutes|recognized|6|se...
13    fairborn|ohio's|xenia|gire|wright-patterson|wa...
14    endeavour|skyler|meanings|shuttle|eagle|scout|...
15    foia|8015|aumf|government-set|davis-bacon|1034...
16    hampshire|guantanamo|timberland|gitmo|detainin...
17                                                  NaN
18    alean|brock|gantt|anderton|calvin's|sickle|cal...
19                                                  NaN
Name: favorite_words, dtype: object



In [204]:

    
# Beneficiaries for events:
def requestBeneficiaries( id ):
    id = str(id)
    url = "politicalpartytime.org/api/v1/event/?format=json&beneficiaries__crp_id="+id+"&apikey=0bf8e7eb6ce146f48217bfee767c998d"
    print contents

    
    if len(contents) > 2:
        data = json.loads(contents)
        beneficiary = json_normalize(data)
        #words = json_normalize(data)
        #list_of_words = words.ngram.tolist()
        #string_of_words ="|".join(list_of_words)
        #return string_of_words
    else:
        return np.nan

requestBeneficiaries(id ="N00003675")









    



[
    {
        "tfidf": 3.8596557124800003e-05,
        "count": 5373,
        "ngram": "people"
    },
    {
        "tfidf": 1.30267768302e-05,
        "count": 3637,
        "ngram": "one"
    },
    {
        "tfidf": 2.52066478599e-05,
        "count": 3509,
        "ngram": "jobs"
    },
    {
        "tfidf": 1.17409333103e-05,
        "count": 3278,
        "ngram": "american"
    },
    {
        "tfidf": 1.14866299957e-05,
        "count": 3207,
        "ngram": "years"
    },
    {
        "tfidf": 2.1916637965399999e-05,
        "count": 3051,
        "ngram": "going"
    },
    {
        "tfidf": 1.02939116332e-05,
        "count": 2874,
        "ngram": "work"
    },
    {
        "tfidf": 1.02939116332e-05,
        "count": 2874,
        "ngram": "support"
    },
    {
        "tfidf": 2.05446032714e-05,
        "count": 2860,
        "ngram": "us"
    },
    {
        "tfidf": 1.00897526342e-05,
        "count": 2817,
        "ngram": "small"
    },
    {
        "tfidf": 9.97513705586e-06,
        "count": 2785,
        "ngram": "new"
    },
    {
        "tfidf": 9.2802801119300001e-06,
        "count": 2591,
        "ngram": "country"
    },
    {
        "tfidf": 9.2050636386200008e-06,
        "count": 2570,
        "ngram": "many"
    },
    {
        "tfidf": 9.1656645335599999e-06,
        "count": 2559,
        "ngram": "year"
    },
    {
        "tfidf": 1.8339290962200001e-05,
        "count": 2553,
        "ngram": "know"
    },
    {
        "tfidf": 1.8037586997999999e-05,
        "count": 2511,
        "ngram": "want"
    },
    {
        "tfidf": 8.9686690082100002e-06,
        "count": 2504,
        "ngram": "states"
    },
    {
        "tfidf": 8.9543420609200003e-06,
        "count": 2500,
        "ngram": "government"
    },
    {
        "tfidf": 8.6713848517900004e-06,
        "count": 2421,
        "ngram": "legislation"
    },
    {
        "tfidf": 1.70247236899e-05,
        "count": 2370,
        "ngram": "business"
    },
    {
        "tfidf": 8.3454468007800006e-06,
        "count": 2330,
        "ngram": "make"
    },
    {
        "tfidf": 1.6457232900199999e-05,
        "count": 2291,
        "ngram": "get"
    },
    {
        "tfidf": 7.7974410666500001e-06,
        "count": 2177,
        "ngram": "federal"
    },
    {
        "tfidf": 7.7114793828599999e-06,
        "count": 2153,
        "ngram": "congress"
    },
    {
        "tfidf": 7.6971524355599999e-06,
        "count": 2149,
        "ngram": "national"
    },
    {
        "tfidf": 7.6255176990799997e-06,
        "count": 2129,
        "ngram": "need"
    },
    {
        "tfidf": 1.48337782361e-05,
        "count": 2065,
        "ngram": "like"
    },
    {
        "tfidf": 7.3783778582e-06,
        "count": 2060,
        "ngram": "well"
    },
    {
        "tfidf": 7.34614222678e-06,
        "count": 2051,
        "ngram": "united"
    },
    {
        "tfidf": 1.4352488579099999e-05,
        "count": 1998,
        "ngram": "think"
    },
    {
        "tfidf": 1.43022045851e-05,
        "count": 1991,
        "ngram": "percent"
    },
    {
        "tfidf": 1.4129802319900001e-05,
        "count": 1967,
        "ngram": "colleagues"
    },
    {
        "tfidf": 7.02378591258e-06,
        "count": 1961,
        "ngram": "back"
    },
    {
        "tfidf": 1.38568320666e-05,
        "count": 1929,
        "ngram": "americans"
    },
    {
        "tfidf": 6.7157565456900002e-06,
        "count": 1875,
        "ngram": "help"
    },
    {
        "tfidf": 6.6906843879200001e-06,
        "count": 1868,
        "ngram": "tax"
    },
    {
        "tfidf": 6.6799391774400001e-06,
        "count": 1865,
        "ngram": "first"
    },
    {
        "tfidf": 2.0011408665000002e-05,
        "count": 1852,
        "ngram": "yield"
    },
    {
        "tfidf": 6.5975592304799999e-06,
        "count": 1842,
        "ngram": "community"
    },
    {
        "tfidf": 1.9449533259799998e-05,
        "count": 1800,
        "ngram": "businesses"
    },
    {
        "tfidf": 1.2915803035599999e-05,
        "count": 1798,
        "ngram": "important"
    },
    {
        "tfidf": 6.4327993365600004e-06,
        "count": 1796,
        "ngram": "program"
    },
    {
        "tfidf": 6.3862367578500003e-06,
        "count": 1783,
        "ngram": "million"
    },
    {
        "tfidf": 1.26140990715e-05,
        "count": 1756,
        "ngram": "way"
    },
    {
        "tfidf": 6.26445770582e-06,
        "count": 1749,
        "ngram": "america"
    },
    {
        "tfidf": 1.8271755967899999e-05,
        "count": 1691,
        "ngram": "thank"
    },
    {
        "tfidf": 5.9564283389200001e-06,
        "count": 1663,
        "ngram": "u.s."
    },
    {
        "tfidf": 1.19173065829e-05,
        "count": 1659,
        "ngram": "go"
    },
    {
        "tfidf": 1.7925986487700001e-05,
        "count": 1659,
        "ngram": "money"
    },
    {
        "tfidf": 1.19173065829e-05,
        "count": 1659,
        "ngram": "care"
    },
    {
        "tfidf": 1.18023717395e-05,
        "count": 1643,
        "ngram": "billion"
    },
    {
        "tfidf": 1.18023717395e-05,
        "count": 1643,
        "ngram": "unemployment"
    },
    {
        "tfidf": 5.8561397078399999e-06,
        "count": 1635,
        "ngram": "law"
    },
    {
        "tfidf": 5.6877980770900003e-06,
        "count": 1588,
        "ngram": "could"
    },
    {
        "tfidf": 5.6770528666200003e-06,
        "count": 1585,
        "ngram": "every"
    },
    {
        "tfidf": 1.1364182648699999e-05,
        "count": 1582,
        "ngram": "economy"
    },
    {
        "tfidf": 5.6018363933100002e-06,
        "count": 1564,
        "ngram": "health"
    },
    {
        "tfidf": 5.51587470953e-06,
        "count": 1540,
        "ngram": "may"
    },
    {
        "tfidf": 5.3761869733699996e-06,
        "count": 1501,
        "ngram": "right"
    },
    {
        "tfidf": 1.07392244373e-05,
        "count": 1495,
        "ngram": "vote"
    },
    {
        "tfidf": 5.2472444477000002e-06,
        "count": 1465,
        "ngram": "ask"
    },
    {
        "tfidf": 5.2150088162800001e-06,
        "count": 1456,
        "ngram": "economic"
    },
    {
        "tfidf": 5.1899366585100001e-06,
        "count": 1449,
        "ngram": "great"
    },
    {
        "tfidf": 1.04016033346e-05,
        "count": 1448,
        "ngram": "last"
    },
    {
        "tfidf": 5.13979234297e-06,
        "count": 1435,
        "ngram": "security"
    },
    {
        "tfidf": 1.01501833645e-05,
        "count": 1413,
        "ngram": "job"
    },
    {
        "tfidf": 5.0538306591799997e-06,
        "count": 1411,
        "ngram": "rise"
    },
    {
        "tfidf": 1.00567988042e-05,
        "count": 1400,
        "ngram": "take"
    },
    {
        "tfidf": 5.0036863436399996e-06,
        "count": 1397,
        "ngram": "service"
    },
    {
        "tfidf": 9.9634142438399995e-06,
        "count": 1387,
        "ngram": "even"
    },
    {
        "tfidf": 4.8998159757300002e-06,
        "count": 1368,
        "ngram": "members"
    },
    {
        "tfidf": 2.44672578777e-05,
        "count": 1351,
        "ngram": "oil"
    },
    {
        "tfidf": 9.6114262856900002e-06,
        "count": 1338,
        "ngram": "come"
    },
    {
        "tfidf": 4.7852003973499999e-06,
        "count": 1336,
        "ngram": "made"
    },
    {
        "tfidf": 4.7637099764099999e-06,
        "count": 1330,
        "ngram": "chairman"
    },
    {
        "tfidf": 4.7422195554599999e-06,
        "count": 1324,
        "ngram": "good"
    },
    {
        "tfidf": 9.4102903096099999e-06,
        "count": 1310,
        "ngram": "much"
    },
    {
        "tfidf": 4.6884935030999998e-06,
        "count": 1309,
        "ngram": "world"
    },
    {
        "tfidf": 4.5416422933000003e-06,
        "count": 1268,
        "ngram": "pay"
    },
    {
        "tfidf": 4.5201518723500002e-06,
        "count": 1262,
        "ngram": "put"
    },
    {
        "tfidf": 4.4843345041100002e-06,
        "count": 1252,
        "ngram": "insurance"
    },
    {
        "tfidf": 4.4735892936300001e-06,
        "count": 1249,
        "ngram": "financial"
    },
    {
        "tfidf": 4.43777192539e-06,
        "count": 1239,
        "ngram": "public"
    },
    {
        "tfidf": 4.3733006625499999e-06,
        "count": 1221,
        "ngram": "administration"
    },
    {
        "tfidf": 4.3518102416099999e-06,
        "count": 1215,
        "ngram": "families"
    },
    {
        "tfidf": 8.6201132607100001e-06,
        "count": 1200,
        "ngram": "spending"
    },
    {
        "tfidf": 8.5841961221300001e-06,
        "count": 1195,
        "ngram": "floor"
    },
    {
        "tfidf": 4.2730120314699997e-06,
        "count": 1193,
        "ngram": "see"
    },
    {
        "tfidf": 4.2694302946499997e-06,
        "count": 1192,
        "ngram": "benefits"
    },
    {
        "tfidf": 3.4544420663100002e-05,
        "count": 1182,
        "ngram": "2010"
    },
    {
        "tfidf": 4.1834686108600003e-06,
        "count": 1168,
        "ngram": "day"
    },
    {
        "tfidf": 4.1583964530900002e-06,
        "count": 1161,
        "ngram": "two"
    },
    {
        "tfidf": 8.2465750194200005e-06,
        "count": 1148,
        "ngram": "continue"
    },
    {
        "tfidf": 4.0724347693100001e-06,
        "count": 1137,
        "ngram": "energy"
    },
    {
        "tfidf": 8.1100898927899995e-06,
        "count": 1129,
        "ngram": "since"
    },
    {
        "tfidf": 4.0222904537599999e-06,
        "count": 1123,
        "ngram": "home"
    },
    {
        "tfidf": 3.9900548223399998e-06,
        "count": 1114,
        "ngram": "war"
    },
    {
        "tfidf": 3.9828913486999999e-06,
        "count": 1112,
        "ngram": "family"
    },
    {
        "tfidf": 3.8826027176100004e-06,
        "count": 1084,
        "ngram": "fact"
    },
    {
        "tfidf": 3.8646940334900004e-06,
        "count": 1079,
        "ngram": "must"
    }
]

	title	firstname	middlename	lastname	name_suffix	nickname	party	state	district	in_office	...	govtrack_id	crp_id	twitter_id	congresspedia_url	youtube_url	facebook_id	official_rss	senate_class	birthdate	oc_email
0	Rep	Neil	NaN	Abercrombie	NaN	NaN	D	HI	1	0	...	400001	N00007665	neilabercrombie	http://www.opencongress.org/wiki/Neil_Abercrombie	http://youtube.com/hawaiirep1	NaN	NaN	NaN	1938-06-26	NaN
1	Rep	Gary	L.	Ackerman	NaN	NaN	D	NY	5	0	...	400003	N00001143	repgaryackerman	http://www.opencongress.org/wiki/Gary_Ackerman	http://youtube.com/RepAckerman	RepAcherman	NaN	NaN	1942-11-19	NaN
2	Rep	Robert	B.	Aderholt	NaN	NaN	R	AL	4	1	...	400004	N00003028	Robert_Aderholt	http://www.opencongress.org/wiki/Robert_Aderholt	http://youtube.com/RobertAderholt	19787529402	NaN	NaN	1965-07-22	Rep.Aderholt@opencongress.org
3	Sen	Daniel	Kahikina	Akaka	NaN	NaN	D	HI	Junior Seat	0	...	300001	N00007653	NaN	http://www.opencongress.org/wiki/Daniel_Akaka	http://youtube.com/senatorakaka	danielakaka	NaN	I	1924-09-11	NaN
4	Sen	Wayne	A.	Allard	NaN	NaN	R	CO	Senior Seat	0	...	300003	N00009082	NaN	http://www.opencongress.org/wiki/Wayne_Allard	NaN	NaN	NaN	II	1943-12-02	NaN

	count	ngram	tfidf
0	5373	people	0.000039
1	3637	one	0.000013
2	3509	jobs	0.000025
3	3278	american	0.000012
4	3207	years	0.000011
5	3051	going	0.000022
6	2874	work	0.000010
7	2874	support	0.000010