In [ ]:
#https://classic.scraperwiki.com/scrapers/all_party_groups/

In [62]:
!mv appg_cache.sqlite appg_cache27.sqlite

In [63]:
url="https://www.publications.parliament.uk/pa/cm/cmallparty/170502/contents.htm"

In [1]:
#!rm appg_cache.sqlite

In [3]:
import requests
import requests_cache
from lxml import etree
from urllib.parse import urljoin

requests_cache.install_cache('appg_cache', backend='sqlite', expire_after=None)

In [4]:
import pandas as pd

r=requests.get('https://www.publications.parliament.uk/pa/cm/cmallparty/170502/germany.htm')
r.from_cache
dfs=pd.read_html(r.text)

In [5]:
gid='tmp'

In [14]:
url='http://www.parliament.uk/mps-lords-and-offices/standards-and-financial-interests/parliamentary-commissioner-for-standards/registers-of-interests/register-of-all-party-party-parliamentary-groups/'

from bs4 import BeautifulSoup
import re

def _getLinksFromPage(url, textcrib=None, hrefcrib=True):
    
    page = requests.get(url)

    #The file we have grabbed in this case is a web page - that is, an HTML file
    #We can get the content of the page and parse it
    soup=BeautifulSoup(page.content, "html5lib")
    #BeautifulSoup has a routine - find_all() - that will find all the HTML tags of a particular sort
    #Links are represented in HTML pages in the form <a href="http//example.com/page.html">link text</a>
    #Grab all the <a> (anchor) tags...
    
    if textcrib:
        souplinks=soup.find_all('a', text=re.compile(textcrib))
    elif hrefcrib:
        souplinks=soup.find_all('a', href=re.compile(hrefcrib))
    else:
        souplinks=soup.find_all('a')
    return souplinks

links = _getLinksFromPage(url,textcrib='Registers published')

#get the most recent link
recent = sorted([link.text for link in links])[-1]
link = [link for link in links if link.text == recent][0]

url='http://www.parliament.uk'+link['href']
links = _getLinksFromPage(url,hrefcrib='pa/cm/cmallparty.*htm')
links[0]['href']


Out[14]:
'https://publications.parliament.uk/pa/cm/cmallparty/171220/contents.htm'

Details


In [6]:
def getDetails(dfs,gid):
    df=dfs[0][:]
    df.set_index(0,inplace=True)
    df=df.T
    df['gid']=gid
    return df.reset_index(drop=True)

In [7]:
getDetails(dfs,gid)


Out[7]:
Title Purpose Category gid
0 All-Party Parliamentary British-German Group To promote mutual understanding and good relat... Country Group tmp

Officers


In [8]:
def getOfficers(dfs, gid):
    df=dfs[1][:]
    df.columns=['Property','Name', 'Party']
    df.set_index('Property',inplace=True)
    df=df[2:]
    df['gid']=gid
    return df.reset_index(drop=True)

In [9]:
getOfficers(dfs, gid)


Out[9]:
Name Party gid
0 Paul Farrelly Labour tmp
1 Lord Anderson of Swansea Labour tmp
2 Lord Dykes Crossbench tmp
3 Lord Balfe Conservative tmp
4 Lord Davies of Stamford Labour tmp
5 Mr Roger Godsiff Labour tmp
6 Mark Pritchard Conservative tmp
7 Mr Mark Hendrick Labour (Co-op) tmp
8 David T. C. Davies Conservative tmp

Contacts


In [10]:
def getContacts(dfs,gid):
    df=dfs[2][:]
    df.rename(columns=df[0], inplace=True)
    df['gid']=gid
    return df[1:].reset_index(drop=True)

In [11]:
getContacts(dfs,gid)


Out[11]:
Contact Details gid
0 Registered Contact: Paul Farrelly MP, House o... tmp

Annual General Meeting


In [12]:
def getAGM(dfs,gid):
    df=dfs[3].loc[:]
    df['gid']=gid
    return df.pivot(index='gid',columns=0,values=1).reset_index()

In [13]:
getAGM(dfs,gid)


Out[13]:
gid Annual General Meeting Date of most recent AGM in this Parliament Did the group publish an income and expenditure statement relating to the AGM above? Next reporting deadline Reporting year
0 tmp NaN 13/07/2016 No 02/10/2017 02 Jun to 01 Jun

Registerable benefits received by the group


In [14]:
def getRegBenefits(dfs,gid):
    df=dfs[4][:]
    df.rename(columns=df.iloc[0], inplace=True)
    df['gid']=gid
    if len(df)>2:
        df.columns=['Source', 'Value', 'Received', 'Registered', 'gid']
    return df[3:]

In [15]:
getRegBenefits(dfs,gid)


Out[15]:
Registrable benefits received by the group gid

Benefits In Kind


In [16]:
def getInKindBenefits(dfs,gid):
    if len(dfs)>5:
        df=dfs[5][:]
        df.rename(columns=df.iloc[1], inplace=True)
        df=df[2:]
        df['gid']=gid
    else: return pd.DataFrame()
    return df

In [17]:
getInKindBenefits(dfs,gid)


Out[17]:
Source Description Value £s In bands of £1,500 Received Registered gid
2 Rolls Royce Plc Flights and accommodation for visit to Berlin ... 9001-10500 17/09/2016 27/10/2016 tmp

Aggregator


In [18]:
df_details=pd.DataFrame() #getDetails(dfs,gid)
df_officers=pd.DataFrame() #getOfficers
df_contacts=pd.DataFrame() #getContacts(dfs,gid)
df_AGM=pd.DataFrame() #getAGM(dfs,gid)
df_regBenefits=pd.DataFrame() #getRegBenefits(dfs,gid)
df_inKindBenefits=pd.DataFrame() #getInKindBenefits(dfs,gid)

In [19]:
#doc = etree.HTML(urlopen(url).read())
#for x in doc.xpath('//*[@id="mainTextBlock"]/ul[1]/li/a'):
#    print(x)

In [20]:
#getDetails(dfs,gid)
#getOfficers(dfs,gid)
#getContacts(dfs,gid)
#getAGM(dfs,gid)
#getRegBenefits(dfs,gid)
#getInKindBenefits(dfs,gid)

In [21]:
doc = etree.HTML(requests.get("https://www.publications.parliament.uk/pa/cm/cmallparty/170502/contents.htm").text)
for x in doc.xpath('//*[@id="mainTextBlock"]/ul/li/a'):#doc.xpath('//p[@class="contentsLink"]/a'):
    print(x.text, x.attrib['href'])
    r = requests.get(urljoin('https://www.publications.parliament.uk/pa/cm/cmallparty/170502/', x.attrib['href']))
    gid= x.attrib['href'].replace('.htm','')
    dfs=pd.read_html(r.text)
    df_details=pd.concat([df_details,getDetails(dfs,gid)])
    df_officers=pd.concat([df_officers,getOfficers(dfs,gid)]) #getOfficers
    df_contacts=pd.concat([df_contacts,getContacts(dfs,gid)]) #getContacts(dfs,gid)
    df_AGM=pd.concat([df_AGM,getAGM(dfs,gid)]) #getAGM(dfs,gid)
    df_regBenefits=pd.concat([df_regBenefits,getRegBenefits(dfs,gid)]) #getRegBenefits(dfs,gid)
    df_inKindBenefits=pd.concat([df_inKindBenefits,getInKindBenefits(dfs,gid)])


Afghanistan afghanistan.htm
Africa africa.htm
African Great Lakes Region african-great-lakes-region.htm
Albania albania.htm
Algeria algeria.htm
Angola angola.htm
Argentina argentina.htm
Armenia armenia.htm
Australia and New Zealand australia-and-new-zealand.htm
Austria austria.htm
Azerbaijan azerbaijan.htm
Bahamas bahamas.htm
Bahrain bahrain.htm
Bangladesh bangladesh.htm
Belarus belarus.htm
Belize belize.htm
Bosnia and Herzegovina bosnia-and-herzegovina.htm
Botswana botswana.htm
Brazil brazil.htm
Bulgaria bulgaria.htm
Burma burma.htm
Cambodia cambodia.htm
Canada canada.htm
Caribbean caribbean.htm
Catalonia catalonia.htm
Cayman Islands cayman-islands.htm
Central American central-american.htm
Chagos chagos.htm
Channel Islands channel-islands.htm
Chile chile.htm
China china.htm
Chinese in Britain chinese-in-britain.htm
Colombia colombia.htm
Croatia croatia.htm
Cuba cuba.htm
Cyprus cyprus.htm
Czech and  Slovak czech-and-slovak.htm
Denmark denmark.htm
Egypt egypt.htm
Eritrea eritrea.htm
Estonia estonia.htm
Ethiopia and Djibouti ethiopia-and-djibouti.htm
Falkland Islands falkland-islands.htm
Faroe Islands faroe-islands.htm
Finland finland.htm
France france.htm
Georgia georgia.htm
Germany germany.htm
Gibraltar gibraltar.htm
Greece greece.htm
Hungary Group hungary-group.htm
Iceland iceland.htm
India india.htm
Indonesia indonesia.htm
Iran iran.htm
Iraq iraq.htm
Ireland and the Irish in Britain ireland-and-the-irish-in-britain.htm
Isle of Man (Manx) isle-of-man-manx.htm
Israel israel.htm
Italy italy.htm
Jamaica jamaica.htm
Japan japan.htm
Jordan jordan.htm
Kashmir kashmir.htm
Kazakhstan kazakhstan.htm
Kenya kenya.htm
Korea korea.htm
Kosovo kosovo.htm
Kurdistan Region in Iraq kurdistan-region-in-iraq.htm
Kuwait kuwait.htm
Latin America latin-america.htm
Latvia latvia.htm
Liechtenstein liechtenstein.htm
Lithuania lithuania.htm
Macedonia macedonia.htm
Malaysia malaysia.htm
Maldives maldives.htm
Mexico mexico.htm
Moldova moldova.htm
Mongolia mongolia.htm
Montserrat montserrat.htm
Morocco morocco.htm
Mozambique mozambique.htm
Nepal nepal.htm
Netherlands netherlands.htm
Nigeria nigeria.htm
Norfolk Island norfolk-island.htm
North Korea north-korea.htm
Norway norway.htm
Oman oman.htm
Overseas Territories overseas-territories.htm
Pacific Islands pacific-islands.htm
Pakistan pakistan.htm
Palestine palestine.htm
Philippines philippines.htm
Pitcairn Islands pitcairn-islands.htm
Poland poland.htm
Polar Regions (Arctic and Antarctic) polar-regions-arctic-and-antarctic.htm
Portugal portugal.htm
Qatar qatar.htm
Romania romania.htm
Russia russia.htm
Saudi Arabia saudi-arabia.htm
Serbia serbia.htm
Singapore singapore.htm
Slovenia slovenia.htm
Somaliland somaliland.htm
South Africa south-africa.htm
Spain spain.htm
Sri Lanka sri-lanka.htm
St Helena st-helena.htm
Sudan sudan.htm
Sudan and South Sudan sudan-and-south-sudan.htm
Sweden sweden.htm
Switzerland switzerland.htm
Syria syria.htm
Taiwan taiwan.htm
Tajikstan tajikstan.htm
Tanzania tanzania.htm
Thailand thailand.htm
Tibet tibet.htm
Turkey turkey.htm
Turkmenistan turkmenistan.htm
Uganda uganda.htm
Ukraine ukraine.htm
United Arab Emirates united-arab-emirates.htm
Uzbekistan uzbekistan.htm
Vietnam vietnam.htm
West Papua west-papua.htm
Western Sahara western-sahara.htm
Yemen yemen.htm
Zimbabwe zimbabwe.htm
14-19 Education 14-19-education.htm
22q11 Syndrome 22q11-syndrome.htm
Abused and Neglected Children abused-and-neglected-children.htm
Adult Education adult-education.htm
Aerospace aerospace.htm
Ageing and Older People ageing-and-older-people.htm
Agriculture and Food for Development agriculture-and-food-for-development.htm
Agroecology for Sustainable Food and Farming agroecology-for-sustainable-food-and-farming.htm
Ahmadiyya Muslim Community ahmadiyya-muslim-community.htm
Air Ambulances air-ambulances.htm
Air Pollution air-pollution.htm
Alcohol Harm alcohol-harm.htm
Alevis alevis.htm
Allergy allergy.htm
Almshouses almshouses.htm
Alternative Dispute Resolution alternative-dispute-resolution.htm
Alternative Investment Management alternative-investment-management.htm
Alternative Lending alternative-lending.htm
Aluminium Industry aluminium-industry.htm
American Football. american-football..htm
Ancient Woodland and Veteran Trees ancient-woodland-and-veteran-trees.htm
Angling angling.htm
Animal Welfare animal-welfare.htm
Antibiotics antibiotics.htm
Anti-Epileptic Drugs in Pregnancy anti-epileptic-drugs-in-pregnancy.htm
Antisemitism antisemitism.htm
Apprenticeships apprenticeships.htm
Archaeology archaeology.htm
Archives and History archives-and-history.htm
Armed Forces armed-forces.htm
Armed Forces Covenant armed-forces-covenant.htm
Arrhythmias arrhythmias.htm
Art, Craft and Design in Education art-craft-and-design-in-education.htm
Artificial Intelligence artificial-intelligence.htm
Arts and Heritage arts-and-heritage.htm
Arts, Health and Wellbeing arts-health-and-wellbeing.htm
Assistive Technology assistive-technology.htm
Atrial Fibrillation atrial-fibrillation.htm
Autism autism.htm
Aviation aviation.htm
Baby Loss baby-loss.htm
Bahá'í bahai.htm
Basketball basketball.htm
BBC bbc.htm
Beer beer.htm
Bees bees.htm
Best Brexit best-brexit.htm
Better Brexit for Young People better-brexit-for-young-people.htm
Betting and Gaming betting-and-gaming.htm
Bingo bingo.htm
Biomass biomass.htm
Blood Cancer blood-cancer.htm
Blood Donation blood-donation.htm
Body Image body-image.htm
Boxing boxing.htm
Boys' Brigade boys-brigade.htm
Brain Tumours brain-tumours.htm
Breast Cancer breast-cancer.htm
Brexpats Living in Continental Europe and European Expats in the UK brexpats-living-in-continental-europe-and-european-expats-in-the-uk.htm
Bridge bridge.htm
British Council british-council.htm
British Museum british-museum.htm
British Property Owners in Cyprus british-property-owners-in-cyprus.htm
Broadband and Digital Communication broadband-and-digital-communication.htm
Built Environment built-environment.htm
Bullying bullying.htm
Business Brexit business-brexit.htm
Business, Finance and Accountancy business-finance-and-accountancy.htm
CAFOD cafod.htm
Cancer cancer.htm
Cannabis: Harmful Effects on Developing Brains cannabis-harmful-effects-on-developing-brains.htm
Carbon Capture and Storage carbon-capture-and-storage.htm
Carbon Monoxide carbon-monoxide.htm
Carers carers.htm
Ceramics ceramics.htm
Channel 4 channel-4.htm
Charities and Volunteering charities-and-volunteering.htm
Chemical Industry chemical-industry.htm
Chess chess.htm
Child and Youth Crime child-and-youth-crime.htm
Child Health and Vaccine Preventable Diseases child-health-and-vaccine-preventable-diseases.htm
Children children.htm
Children in Wales children-in-wales.htm
Children of Alcoholics children-of-alcoholics.htm
Children who need Palliative Care children-who-need-palliative-care.htm
Children's Media and the Arts childrens-media-and-the-arts.htm
Choice at the End of Life choice-at-the-end-of-life.htm
Choir choir.htm
Christians in Parliament christians-in-parliament.htm
Cider cider.htm
City Regions Transport city-regions-transport.htm
Civic Societies civic-societies.htm
Classical Music classical-music.htm
Climate Change climate-change.htm
Clinical Leadership and Management clinical-leadership-and-management.htm
Coalfield Communities coalfield-communities.htm
Commercial Radio commercial-radio.htm
Commonwealth commonwealth.htm
Communities Engagement communities-engagement.htm
Community Development in Europe community-development-in-europe.htm
Complex Needs and Dual Diagnosis complex-needs-and-dual-diagnosis.htm
Conception to Age Two: First 1001 Days conception-to-age-two-first-1001-days.htm
Connaught Income Fund connaught-income-fund.htm
Consumer Protection consumer-protection.htm
Continence Care continence-care.htm
Corporate Governance corporate-governance.htm
Corporate Responsibility corporate-responsibility.htm
Corruption corruption.htm
Counter-Extremism counter-extremism.htm
County county.htm
Couple Relationships couple-relationships.htm
Credit Unions credit-unions.htm
Cricket cricket.htm
Crossrail crossrail.htm
Crossrail Two crossrail-two.htm
Crowdfunding and Non-Banking Finance crowdfunding-and-non-banking-finance.htm
Cultural Heritage cultural-heritage.htm
Curry Catering Industry curry-catering-industry.htm
Customer Service customer-service.htm
Cyber Security cyber-security.htm
Cycling cycling.htm
Dairy dairy.htm
Dalits dalits.htm
Dance dance.htm
Data Analytics data-analytics.htm
Deafness deafness.htm
Death Penalty death-penalty.htm
Debt and Personal Finance debt-and-personal-finance.htm
Defence and Security Issues defence-and-security-issues.htm
Dementia dementia.htm
Democracy and Human Rights in the Gulf democracy-and-human-rights-in-the-gulf.htm
Democratic Participation democratic-participation.htm
Dentistry and Oral Health dentistry-and-oral-health.htm
Design and Innovation design-and-innovation.htm
Diabetes diabetes.htm
Digital Crime digital-crime.htm
Digital Economy digital-economy.htm
Disability disability.htm
District Councils district-councils.htm
Dog Welfare dog-welfare.htm
Domestic Violence domestic-violence.htm
Drones drones.htm
Drug Policy Reform drug-policy-reform.htm
Dyslexia and Other Specific Learning Difficulties dyslexia-and-other-specific-learning-difficulties.htm
East-West Rail east-west-rail.htm
E-Cigarettes e-cigarettes.htm
Education education.htm
Education Governance and Leadership education-governance-and-leadership.htm
Eggs, Pigs and Poultry eggs-pigs-and-poultry.htm
Endangered Species endangered-species.htm
Energy Costs energy-costs.htm
Energy Intensive Industries energy-intensive-industries.htm
Energy Storage energy-storage.htm
Energy Studies energy-studies.htm
Engineering engineering.htm
Entrepreneurship entrepreneurship.htm
Environment environment.htm
Epilepsy epilepsy.htm
Equitable Life equitable-life.htm
Ethics and Sustainability in Fashion ethics-and-sustainability-in-fashion.htm
Events Industry events-industry.htm
Explosive Weapons explosive-weapons.htm
Extraordinary Rendition extraordinary-rendition.htm
Fair Business Banking fair-business-banking.htm
Fair Fuel for Motorists and Hauliers fair-fuel-for-motorists-and-hauliers.htm
Fairs and Showgrounds fairs-and-showgrounds.htm
Fairtrade fairtrade.htm
Faith and Society faith-and-society.htm
Family Business family-business.htm
Fatherhood fatherhood.htm
Film Industry film-industry.htm
Financial Education for Young People financial-education-for-young-people.htm
Financial Markets and Services financial-markets-and-services.htm
Financial Technology financial-technology.htm
Fire Safety Rescue fire-safety-rescue.htm
First Aid first-aid.htm
Fisheries fisheries.htm
Fit and Healthy Childhood fit-and-healthy-childhood.htm
Fixed Odds Betting Terminals fixed-odds-betting-terminals.htm
Flags and Heraldry flags-and-heraldry.htm
Flood Prevention flood-prevention.htm
Foetal Alcohol Spectrum Disorder foetal-alcohol-spectrum-disorder.htm
Folk Arts folk-arts.htm
Food and Drink Manufacturing food-and-drink-manufacturing.htm
Food and Health food-and-health.htm
Food Waste food-waste.htm
Football football.htm
Football Supporters football-supporters.htm
Foreign Affairs foreign-affairs.htm
Forestry forestry.htm
Formula One formula-one.htm
Fourth Industrial Revolution fourth-industrial-revolution.htm
Freedom of Movement freedom-of-movement.htm
Freight Transport freight-transport.htm
Friends of Syria friends-of-syria.htm
Frozen British Pensions frozen-british-pensions.htm
Fruit and Vegetable Farmers fruit-and-vegetable-farmers.htm
Fuel Poverty and Energy Efficiency fuel-poverty-and-energy-efficiency.htm
Funerals and Bereavement funerals-and-bereavement.htm
Furniture Industry furniture-industry.htm
Further Education and Lifelong Learning further-education-and-lifelong-learning.htm
Future of Transport future-of-transport.htm
Future of Work future-of-work.htm
Game and Wildlife Conservation game-and-wildlife-conservation.htm
Gardening and Horticulture gardening-and-horticulture.htm
General Aviation general-aviation.htm
Genocide Prevention genocide-prevention.htm
Global Education for All global-education-for-all.htm
Global Health global-health.htm
Global Security and Non-Proliferation global-security-and-non-proliferation.htm
Golf golf.htm
Governance and Inclusive Leadership governance-and-inclusive-leadership.htm
Greater Manchester greater-manchester.htm
Greyhound greyhound.htm
Guantanamo Bay guantanamo-bay.htm
Gypsies, Travellers and Roma gypsies-travellers-and-roma.htm
Haemophilia and Contaminated Blood haemophilia-and-contaminated-blood.htm
Hair Industry hair-industry.htm
Hampshire and Isle of Wight hampshire-and-isle-of-wight.htm
Health health.htm
Health in all Policies health-in-all-policies.htm
Healthy Homes and Buildings healthy-homes-and-buildings.htm
Heart Disease heart-disease.htm
Heathrow and the Wider Economy heathrow-and-the-wider-economy.htm
Heritage Rail heritage-rail.htm
Highways highways.htm
Hillsborough Disaster hillsborough-disaster.htm
Hindus hindus.htm
Historic Vehicles historic-vehicles.htm
HIV and AIDS hiv-and-aids.htm
Holy See holy-see.htm
Home Electrical Safety home-electrical-safety.htm
Homelessness homelessness.htm
Honour Based Abuse. honour-based-abuse..htm
Horse horse.htm
Hospice and Palliative Care hospice-and-palliative-care.htm
Housing and Care for Older People housing-and-care-for-older-people.htm
Housing and Planning housing-and-planning.htm
Human Rights human-rights.htm
Human Trafficking and Modern Slavery human-trafficking-and-modern-slavery.htm
Humanist humanist.htm
Hunger hunger.htm
Inclusive Growth inclusive-growth.htm
Indian Traditional Sciences indian-traditional-sciences.htm
Industrial Heritage industrial-heritage.htm
Infant Feeding and Inequalities infant-feeding-and-inequalities.htm
Inflammatory Bowel Disease inflammatory-bowel-disease.htm
Infrastructure infrastructure.htm
Insurance and Financial Services insurance-and-financial-services.htm
Integrated Healthcare integrated-healthcare.htm
Intellectual Property intellectual-property.htm
Intelligent Energy intelligent-energy.htm
Interfaith interfaith.htm
International Mining international-mining.htm
International Students international-students.htm
International Trade and Investment international-trade-and-investment.htm
Internet, Communications and Technology internet-communications-and-technology.htm
Islamic Finance islamic-finance.htm
Islamophobia islamophobia.htm
ITV itv.htm
Jainism jainism.htm
Jazz Appreciation jazz-appreciation.htm
Jews jews.htm
Key Cities key-cities.htm
Kidney kidney.htm
Learning Disability learning-disability.htm
Leasehold and Commonhold Reform leasehold-and-commonhold-reform.htm
Legal Aid legal-aid.htm
Legal and Constitutional Affairs legal-and-constitutional-affairs.htm
Lesbian, Gay, Bisexual, and Transgender Rights lesbian-gay-bisexual-and-transgender-rights.htm
Libraries libraries.htm
Life Sciences life-sciences.htm
Light Rail light-rail.htm
Limits to Growth limits-to-growth.htm
Listed Properties listed-properties.htm
Literacy literacy.htm
Liver Health liver-health.htm
Local Authority Pension Funds local-authority-pension-funds.htm
Local Democracy local-democracy.htm
Local Government local-government.htm
London london.htm
London Stansted Cambridge Corridor london-stansted-cambridge-corridor.htm
London's Planning and Built Environment londons-planning-and-built-environment.htm
Looked After Children and Care Leavers looked-after-children-and-care-leavers.htm
Lyme Disease lyme-disease.htm
Magistracy magistracy.htm
Malaria and Neglected Tropical Diseases malaria-and-neglected-tropical-diseases.htm
Management management.htm
Manufacturing manufacturing.htm
Marine Energy and Tidal Lagoons marine-energy-and-tidal-lagoons.htm
Maritime and Ports maritime-and-ports.htm
Markets markets.htm
Maternity maternity.htm
Mayflower Pilgrims mayflower-pilgrims.htm
Media media.htm
Medical Research medical-research.htm
Members' Clubs members-clubs.htm
Meningitis meningitis.htm
Mental Health mental-health.htm
Mersey Dee North Wales mersey-dee-north-wales.htm
Micronutrients and Health micronutrients-and-health.htm
Migration migration.htm
Mindfulness mindfulness.htm
Mining and Quarrying mining-and-quarrying.htm
Modern Languages modern-languages.htm
Motor motor.htm
Motor Neurone Disease motor-neurone-disease.htm
Motorcycle Speedway motorcycle-speedway.htm
Motorcycling motorcycling.htm
Motorsport motorsport.htm
Mountain and Cave Rescue mountain-and-cave-rescue.htm
Mountaineering mountaineering.htm
Multiple Sclerosis multiple-sclerosis.htm
Muscular Dystrophy muscular-dystrophy.htm
Music music.htm
Music Education music-education.htm
Mutuals mutuals.htm
Myalgic Encephalomyelitis (ME) myalgic-encephalomyelitis-me.htm
National Parks national-parks.htm
Nuclear Energy nuclear-energy.htm
Nursery Schools and Nursery Classes nursery-schools-and-nursery-classes.htm
Obesity obesity.htm
Occupational Safety and Health occupational-safety-and-health.htm
Off-Patent Drugs off-patent-drugs.htm
Offshore Oil and Gas offshore-oil-and-gas.htm
Oil Refining Sector oil-refining-sector.htm
Opera opera.htm
Oral Hormone Pregnancy Tests oral-hormone-pregnancy-tests.htm
Osteoposis osteoposis.htm
Ovarian Cancer ovarian-cancer.htm
Packaging Manufacturing Industry packaging-manufacturing-industry.htm
Pakistan Minorities pakistan-minorities.htm
Pancreatic Cancer pancreatic-cancer.htm
Parents and Families parents-and-families.htm
Park Homes park-homes.htm
Parkinson's parkinsons.htm
Patient and Public Involvement in Health and Social Care patient-and-public-involvement-in-health-and-social-care.htm
Patient Safety patient-safety.htm
Payment Systems payment-systems.htm
Penal Affairs penal-affairs.htm
Pensions pensions.htm
Performers' Alliance performers-alliance.htm
Personalised Medicine personalised-medicine.htm
Pharmacy pharmacy.htm
Philately philately.htm
Photonics photonics.htm
Pigeon Racing pigeon-racing.htm
Policing policing.htm
Population, Development and Reproductive Health population-development-and-reproductive-health.htm
Poverty poverty.htm
Prescribed Drug Dependence prescribed-drug-dependence.htm
Primary Care and Public Health primary-care-and-public-health.htm
Print print.htm
Private Rented Sector private-rented-sector.htm
Pro Bono pro-bono.htm
Pro-Life pro-life.htm
Proportional Representation proportional-representation.htm
Prostitution and the Global Sex Trade prostitution-and-the-global-sex-trade.htm
Psychoactive Substances and Volatile Substance Abuse psychoactive-substances-and-volatile-substance-abuse.htm
Public Legal Education public-legal-education.htm
Publishing publishing.htm
Pubs pubs.htm
Race and Community race-and-community.htm
Racing and Bloodstock Industries racing-and-bloodstock-industries.htm
Rail rail.htm
Rail in the North rail-in-the-north.htm
Rare, Genetic and Undiagnosed Conditions rare-genetic-and-undiagnosed-conditions.htm
Reform, Decentralisation and Devolution in the UK reform-decentralisation-and-devolution-in-the-uk.htm
Refugees refugees.htm
Regional Airports regional-airports.htm
Religion in the Media religion-in-the-media.htm
Religion or Belief religion-or-belief.htm
Religious Education religious-education.htm
Renewable and Sustainable Energy renewable-and-sustainable-energy.htm
Reserve Forces and Cadets reserve-forces-and-cadets.htm
Respiratory Health respiratory-health.htm
Responsible Tax responsible-tax.htm
Retail retail.htm
Retail Crime retail-crime.htm
River Thames river-thames.htm
Road Passenger Transport road-passenger-transport.htm
Roofing Industry roofing-industry.htm
Rowing rowing.htm
Royal Television Society royal-television-society.htm
Rugby League rugby-league.htm
Rugby Union rugby-union.htm
Rule of Law rule-of-law.htm
Runaway and Missing Children and Adults runaway-and-missing-children-and-adults.htm
Running running.htm
Rural Business rural-business.htm
Rural Crime rural-crime.htm
Rural Health and Social Care rural-health-and-social-care.htm
Rural Services rural-services.htm
School Food school-food.htm
Science and Technology in Agriculture science-and-technology-in-agriculture.htm
Scientific scientific.htm
Scotch Whisky scotch-whisky.htm
Scottish Sport scottish-sport.htm
Self-Build, Custom and Community Housebuilding and Placemaking self-build-custom-and-community-housebuilding-and-placemaking.htm
Self-Employment and Freelancing self-employment-and-freelancing.htm
Sepsis sepsis.htm
Sex Equality sex-equality.htm
Sexual and Reproductive Health sexual-and-reproductive-health.htm
Sexual Violence in Conflict sexual-violence-in-conflict.htm
Shale Gas Regulation and Planning shale-gas-regulation-and-planning.htm
Ship Building ship-building.htm
Shooting and Conservation shooting-and-conservation.htm
Sickle Cell and Thalassemia sickle-cell-and-thalassemia.htm
Sikhs sikhs.htm
Sixth Form Colleges sixth-form-colleges.htm
Skills and Employment skills-and-employment.htm
Skin skin.htm
Small and Micro Business small-and-micro-business.htm
Small Shops small-shops.htm
Smart Cities smart-cities.htm
Smoking and Health smoking-and-health.htm
Social Enterprise social-enterprise.htm
Social Integration social-integration.htm
Social Mobility social-mobility.htm
Social Work social-work.htm
Soft Drinks soft-drinks.htm
South West Rail south-west-rail.htm
Southern Rail southern-rail.htm
Space space.htm
Speech and Language Difficulties speech-and-language-difficulties.htm
Spinal Cord Injury spinal-cord-injury.htm
Sport sport.htm
Squash and Racketball squash-and-racketball.htm
Srebrenica srebrenica.htm
St George's Day st-georges-day.htm
State Pension Inequality for Women state-pension-inequality-for-women.htm
Statistics statistics.htm
Steel and Metal Related Industries steel-and-metal-related-industries.htm
Stem Cell Transplantation stem-cell-transplantation.htm
Street Children street-children.htm
Students students.htm
Suicide and Self-Harm Prevention suicide-and-self-harm-prevention.htm
Sustainable Resource sustainable-resource.htm
Tamils tamils.htm
Taxation taxation.htm
Taxis taxis.htm
Tennis tennis.htm
Thalidomide thalidomide.htm
Thrombosis thrombosis.htm
Ticket Abuse ticket-abuse.htm
Tidy Britain tidy-britain.htm
Timber Industries timber-industries.htm
Tourism and Hospitality Industry in Wales tourism-and-hospitality-industry-in-wales.htm
Tourism, Leisure and the Hospitality Industry tourism-leisure-and-the-hospitality-industry.htm
Town Centres town-centres.htm
Trade out of Poverty trade-out-of-poverty.htm
Transatlantic Trade transatlantic-trade.htm
Transport Safety transport-safety.htm
Tuberculosis tuberculosis.htm
UK and Europe uk-and-europe.htm
UK-EU Relations uk-eu-relations.htm
Unconventional Oil and Gas unconventional-oil-and-gas.htm
Union union.htm
United Nations united-nations.htm
United Nations Global Goals for Sustainable Development united-nations-global-goals-for-sustainable-development.htm
Universities universities.htm
Vaccinations for All vaccinations-for-all.htm
Vascular Disease vascular-disease.htm
Vegetarianism and Veganism vegetarianism-and-veganism.htm
Veterans veterans.htm
Video Games video-games.htm
Visitor Economy visitor-economy.htm
War Heritage war-heritage.htm
Water water.htm
Water and Sanitation in the Third World water-and-sanitation-in-the-third-world.htm
Water Safety and Drowning Prevention water-safety-and-drowning-prevention.htm
Waterways waterways.htm
Waves waves.htm
Wellbeing Economics wellbeing-economics.htm
West Coast Main Line west-coast-main-line.htm
West Midlands west-midlands.htm
White Ribbon Campaign UK white-ribbon-campaign-uk.htm
Wine and Spirit Group wine-and-spirit-group.htm
Women and Enterprise women-and-enterprise.htm
Women and Work women-and-work.htm
Women in Parliament women-in-parliament.htm
Women in the Penal System women-in-the-penal-system.htm
Women in Transport women-in-transport.htm
Women, Peace and Security women-peace-and-security.htm
Women's Health womens-health.htm
Women's Sport and Fitness womens-sport-and-fitness.htm
Wood Panel Industry wood-panel-industry.htm
Writers writers.htm
Yorkshire and Northern Lincolnshire yorkshire-and-northern-lincolnshire.htm
Young Disabled People young-disabled-people.htm
Young People and Social Technology young-people-and-social-technology.htm
Youth Affairs youth-affairs.htm
Youth Employment youth-employment.htm
Youth Hostelling youth-hostelling.htm
Zoos and Aquariums zoos-and-aquariums.htm
Zoroastrianism zoroastrianism.htm
Football Club football-club.htm
Hockey hockey.htm
Rugby Union Football Club rugby-union-football-club.htm

In [22]:
df_details.to_csv('df_details.csv',index=False)
df_officers.to_csv('df_officers.csv',index=False)
df_contacts.to_csv('df_contacts.csv',index=False)
df_AGM.to_csv('df_AGM.csv',index=False)
df_regBenefits.to_csv('df_regBenefits.csv',index=False)
df_inKindBenefits.to_csv('df_inKindBenefits.csv',index=False)

Example Queries


In [23]:
!wc -l df_*.csv


     632 df_AGM.csv
     632 df_contacts.csv
     632 df_details.csv
     389 df_inKindBenefits.csv
    4122 df_officers.csv
     348 df_regBenefits.csv
    6755 total

In [42]:
!ls -al *sqlite


-rw-r--r--@ 1 ajh59  1182653967   16809984  3 Jul 16:23 appg_cache.sqlite
-rw-r--r--@ 1 ajh59  1182653967  198070272 10 May 14:55 parlidata_cache.sqlite
-rw-r--r--@ 1 ajh59  1182653967          0  3 Jul 15:46 scraperwiki.sqlite

In [ ]: