This notebook contains some code fragments for exploring Parliamentary Committees data published via the MNIS API.
At the current time, Committee related data is not available from data.parliament.uk APIs.
In [15]:
import requests
import pandas as pd
In [16]:
def _getData(url):
headers = {'content-type': 'application/json'}
rr=requests.get(url, headers=headers)
rr.encoding='utf-8-sig'
return rr.json()
In [17]:
def getCommitteeAdminData():
comm_url='http://data.parliament.uk/membersdataplatform/services/mnis/ReferenceData/Committees/'
b= _getData(comm_url)
currcomm=[]
for c in b['Committees']['Committee']:
if (c['EndDate'] is None) or (isinstance(c['EndDate'], str) and c['EndDate']=='') or ('@xsi:nil' in c['EndDate'] and c['EndDate']['@xsi:nil']=='true'):
currcomm.append(c)
return pd.DataFrame(currcomm)
currcomm=getCommitteeAdminData()
currcomm.head()
Out[17]:
In [18]:
#Committee types
currcomm['CommitteeType'].unique().tolist()
Out[18]:
In [19]:
from urllib.parse import urlencode
def committeesReferenceURL(c, house=None):
house='House={}%7C'.format(house) if house is not None and house.lower() in ['commons','lords', 'all'] else ''
comm_url='http://data.parliament.uk/membersdataplatform/services/mnis/members/query/{h}{c}/Committees'.format(h= house,
c=(urlencode({'committee':c})))
return comm_url
def committeeMembersByCommitteeName(c, house=None):
members=_getData(committeesReferenceURL(c, house))
tl=[]
if members['Members'] is None: return "No members or committee not found"
for m in members['Members']['Member']:
tl.append('{} ({})'.format(m['FullTitle'],m['Party']['#text']))
return 'Members of the {}: {}'.format(c,', '.join(tl))
In [20]:
c='Accommodation Steering Group'
committeeMembersByCommitteeName(c)
Out[20]:
More usefully, we can get administrative information about a committee, as well as the current members, if we have the ID of the committee:
In [21]:
def committeeReferenceURL(cid, membersToGet='all'):
comm_url='http://data.parliament.uk/membersdataplatform/services/mnis/Committee/{cid}/{membersToGet}/'.format(cid=cid,
membersToGet=membersToGet)
return comm_url
def committeeMembersByCommitteeID(cid, membersToGet='current'):
committee= _getData(committeeReferenceURL(cid, membersToGet))
tl=[]
hse=[]
members=[]
if committee['Committee']['Details']['IsLords']=='True': hse.append('Lords')
if committee['Committee']['Details']['IsCommons']=='True': hse.append('Commons')
c='{n} ({typ}{hse})'.format(n=committee['Committee']['Details']['Name'],
typ=committee['Committee']['Details']["CommitteeType"],
hse=', {}'.format(', '.join(hse)) if hse!=[] else '')
for m in committee['Committee']['Members']['Member']:
tl.append('{} ({})'.format(m['FullTitle'],m['Party']['#text']))
members.append({'Name':committee['Committee']['Details']['Name'],
'Title':m['FullTitle'], 'Party':m['Party']['#text']})
return 'Members of the {}: {}'.format(c,', '.join(tl)), pd.DataFrame(members)
In [22]:
cid=17
t,mdf = committeeMembersByCommitteeID(cid)
print(t)
mdf
Out[22]:
Additional committee data available:
Details': {'Committee_Id': '17', 'CommitteeType': 'Departmental', 'CommitteeType_Id': '2', 'Name': 'Communities and Local Government Committee', 'ParentCommittee': None, 'ParentCommittee_Id': None, 'DateLordsAppointed': None, 'DateCommonsAppointed': '2017-09-11T00:00:00', 'Phone': '020 7219 3927/3290', 'Url': 'http://www.parliament.uk/business/committees/committees-a-z/commons-select/communities-and-local-government-committee/', 'StartDate': '2006-06-27T00:00:00', 'EndDate': None, 'CreatedFromCommittee': None, 'CreatedFromCommittee_Id': None, 'Chair_Member_Id': '394', 'Chair_Member': 'Mr Clive Betts', 'Chair_StartDate': '2017-07-12T00:00:00', 'IsCommons': 'True', 'IsLords': 'False'}, 'Clerks': {'Clerk': {'Name': 'Ed Beale', 'StartDate': '2017-05-08T00:00:00'}}, 'Members': {'Member':[...]}, 'LayMembers': None}
In [23]:
!pip3 install mnis
In [24]:
#https://github.com/olihawkins/mnis
#!pip3 install mnis
import mnis
import datetime
# Create a date for the analysis
d = datetime.date.today()
# Download the full data for MPs serving on the given date as a list
members = mnis.getCommonsMembersOn(d)
m=mnis.getSummaryDataForMembers(members, d)
df=pd.DataFrame(m)
df.head()
Out[24]:
Given the member ID, we can pull down detailed imformation about the member, including all their committee activity. Assume membership of a committee is current if there is no end date.
In [27]:
def getCommittees(mid):
resp={}
comm_url='http://data.parliament.uk/membersdataplatform/services/mnis/members/query/id={}/Committees'.format(mid)
#should really use _getData(comm_url)
headers = {'content-type': 'application/json'}
q = requests.get(comm_url, headers=headers)
q.encoding='utf-8-sig'
j=q.json()
resp['Member']=j["Members"]["Member"]["FullTitle"]
resp["Committees"]=[]
if "Committees" not in j["Members"]["Member"] or j["Members"]["Member"]["Committees"] is None: return resp
if "Committee" not in j["Members"]["Member"]["Committees"]: return resp
committees=j["Members"]["Member"]["Committees"]['Committee']
if not isinstance(committees, list):
committees=[committees]
for c in committees:
if (isinstance(c['EndDate'], str) and c['EndDate']=='') or ('@xsi:nil' in c['EndDate'] and c['EndDate']['@xsi:nil']=='true'):
resp['Committees'].append(c)
return resp
In [28]:
getCommittees(4005)
Out[28]:
We can parse this information and create a set of rows for each MP, one row for each committee they are on.
Note that this will take some time to run and is not very efficient - we are making a separate request to the MNIS API for info about each member.
In [30]:
#Make a column containing the dict of committees
df['committees']=df['member_id'].apply(lambda x: getCommittees(x)['Committees'] )
#Then for each of the committees, create a row in a dataframe associating member with committee
#https://stackoverflow.com/a/27266225/454773
res = df[df['committees'].str.len()>0].set_index(['list_name','member_id','party'])['committees'].apply(pd.Series).stack()
res = res.reset_index()
res['cname']=res[0].apply(pd.Series)['Name']
res['chair']=res[0].apply(pd.Series)['ChairDates'].apply(pd.Series)['ChairDate']
membercommittees = res[['list_name','member_id','party','cname','chair']]
membercommittees.head()
Out[30]:
In [31]:
membercommittees.to_csv('data/saved_mnis_committees.csv',index=False)
It may be useful to explore http://data.parliament.uk/MembersDataPlatform/services/mnis/help further and perhaps extend Oli Hawkins' mnis package to support more general querying of the MNIS database.
In [ ]: