In [90]:

    

from bs4 import BeautifulSoup
import dryscrape
import json


    

In [110]:

    

all_links = {}
REMAINING = True
page = 1
session = dryscrape.Session()
dc_url = 'http://opendata.dc.gov'
while REMAINING:
    base_url= 'http://opendata.dc.gov/datasets?q=*&page=%s&sort_by=updated_at' % page
    session.visit(base_url)
    response = session.body()
    soup = BeautifulSoup(response, 'html.parser')
    data_links = {a.text: dc_url + a['href'] for a in soup.find_all('a', {'class':"dataset-link", 'itemprop':'url'})}
    all_links.update(data_links)
    page += 1
    if not data_links:
        REMAINING = False


    

In [ ]:

    

parking_violations = {name.strip(): url for name, url in all_links.items() if 'Parking Violations in' in name}


    

In [112]:

    

with open('dc_parking_violations.json', 'w') as outfile:
    json.dump(parking_violations, outfile)


    

In [114]:

    

sorted(parking_violations.items())


    

    
Out[114]:





[('Parking Violations in April 2009',
  'http://opendata.dc.gov/datasets/37c76fb6fda04804ab20a745a60cbe98_3'),
 ('Parking Violations in April 2010',
  'http://opendata.dc.gov/datasets/7b4816cbd94f46d3961d63b5819a32ae_3'),
 ('Parking Violations in April 2011',
  'http://opendata.dc.gov/datasets/8e90827bd31440a697ae85aa80c187a5_3'),
 ('Parking Violations in April 2012',
  'http://opendata.dc.gov/datasets/7637a409c4534388af9d403d70310e79_3'),
 ('Parking Violations in April 2013',
  'http://opendata.dc.gov/datasets/7906787f79ef47d6b1bc80a4d19ea567_3'),
 ('Parking Violations in April 2014',
  'http://opendata.dc.gov/datasets/f01f547123844b97930de0127555ba64_3'),
 ('Parking Violations in April 2015',
  'http://opendata.dc.gov/datasets/c6037ff5ae924f8f845512d7a164abf2_2'),
 ('Parking Violations in April 2016',
  'http://opendata.dc.gov/datasets/977602b156f74e41ae2dabbfaca42e20_3'),
 ('Parking Violations in August 2009',
  'http://opendata.dc.gov/datasets/10d9348a1a8640d885ae5471beabe00c_7'),
 ('Parking Violations in August 2010',
  'http://opendata.dc.gov/datasets/d45f5a57d9684b68b58e47e6af1c4349_7'),
 ('Parking Violations in August 2011',
  'http://opendata.dc.gov/datasets/86ea673e558942169f7e3b198cf4c0b8_7'),
 ('Parking Violations in August 2012',
  'http://opendata.dc.gov/datasets/920d1655a6d044259c37a35f6fab06f1_7'),
 ('Parking Violations in August 2013',
  'http://opendata.dc.gov/datasets/0b921d3eaa4b410f8d34d0677d3f7cb5_7'),
 ('Parking Violations in August 2014',
  'http://opendata.dc.gov/datasets/dda2ff2bbcdb40a7a2044010f73f962b_7'),
 ('Parking Violations in August 2015',
  'http://opendata.dc.gov/datasets/13fd2526f4da4da294e1bd1ba1b10d4f_7'),
 ('Parking Violations in December 2009',
  'http://opendata.dc.gov/datasets/42851dc219334af280848c540a8fdc8a_11'),
 ('Parking Violations in December 2010',
  'http://opendata.dc.gov/datasets/29ae32ced6694cd8b51051b4b4408163_11'),
 ('Parking Violations in December 2011',
  'http://opendata.dc.gov/datasets/4356a86b70064bc9ba69739dcbf18dd2_11'),
 ('Parking Violations in December 2012',
  'http://opendata.dc.gov/datasets/150bd05020944c0fb265a926f4783a8e_11'),
 ('Parking Violations in December 2013',
  'http://opendata.dc.gov/datasets/5d582d5304424c9b823985c294faba66_11'),
 ('Parking Violations in December 2014',
  'http://opendata.dc.gov/datasets/b00605a8e3e54d2ca75c525ba42966ea_11'),
 ('Parking Violations in December 2015',
  'http://opendata.dc.gov/datasets/2e967e9053144a309680fccea0f7b4e1_11'),
 ('Parking Violations in February 2009',
  'http://opendata.dc.gov/datasets/f057850d3c4544f7bc0deefed2cb39ea_1'),
 ('Parking Violations in February 2010',
  'http://opendata.dc.gov/datasets/6bfd510fba784d02a034d2ba5ecffb0f_1'),
 ('Parking Violations in February 2011',
  'http://opendata.dc.gov/datasets/3f8c2e2c0c604df69f40ca5d4d44e922_1'),
 ('Parking Violations in February 2012',
  'http://opendata.dc.gov/datasets/3baa79929b00451099acf1028d638b67_0'),
 ('Parking Violations in February 2013',
  'http://opendata.dc.gov/datasets/e541d743615d4027bf3ec00d67b7adbf_1'),
 ('Parking Violations in February 2014',
  'http://opendata.dc.gov/datasets/7877594f7e7a4affbd9372f65b935660_1'),
 ('Parking Violations in February 2015',
  'http://opendata.dc.gov/datasets/aab075ac11e9415fb329fb4fab970bc3_1'),
 ('Parking Violations in February 2016',
  'http://opendata.dc.gov/datasets/150cf72502b344448d900a4f1d779b3a_1'),
 ('Parking Violations in January 2009',
  'http://opendata.dc.gov/datasets/e7431d6341c0426c8f9b5f5a14ff061d_0'),
 ('Parking Violations in January 2010',
  'http://opendata.dc.gov/datasets/bb1461eb718f4b84b0854e5f306838cc_0'),
 ('Parking Violations in January 2011',
  'http://opendata.dc.gov/datasets/dba1a31d08de417a867ad4ad1c11fc86_0'),
 ('Parking Violations in January 2012',
  'http://opendata.dc.gov/datasets/2bb91541a6a744c7acfdcb7faf7a65dd_0'),
 ('Parking Violations in January 2013',
  'http://opendata.dc.gov/datasets/8e065ed29a2d43c08802385f3c9a6df4_0'),
 ('Parking Violations in January 2014',
  'http://opendata.dc.gov/datasets/93c1160a13cd49e0a411b3f8e6c3be66_0'),
 ('Parking Violations in January 2015',
  'http://opendata.dc.gov/datasets/6f36c3fcbcfa4f83931f524628325c9c_0'),
 ('Parking Violations in January 2016',
  'http://opendata.dc.gov/datasets/7e688a52e65d49c0beef48289860f465_0'),
 ('Parking Violations in July 2009',
  'http://opendata.dc.gov/datasets/26542a56936047d2acba4e3da38e7b35_6'),
 ('Parking Violations in July 2010',
  'http://opendata.dc.gov/datasets/d11b9fb3f9fe481386515961a427f62d_6'),
 ('Parking Violations in July 2011',
  'http://opendata.dc.gov/datasets/59d16e51bbb34effbd03b233e7c00759_6'),
 ('Parking Violations in July 2012',
  'http://opendata.dc.gov/datasets/7c46d01a9b2145afb05f8fa17af00264_6'),
 ('Parking Violations in July 2013',
  'http://opendata.dc.gov/datasets/0814aef502aa49d783b5086565740524_6'),
 ('Parking Violations in July 2014',
  'http://opendata.dc.gov/datasets/3f557f661e444c29b217eb63c2324d34_6'),
 ('Parking Violations in July 2015',
  'http://opendata.dc.gov/datasets/0d22a375d1074700b007f67d413b7ee0_6'),
 ('Parking Violations in June 2009',
  'http://opendata.dc.gov/datasets/2aafd56019a74b868b4d18939f93edf6_5'),
 ('Parking Violations in June 2010',
  'http://opendata.dc.gov/datasets/c2a74f7ecc2640ed958b8d22d78576a1_5'),
 ('Parking Violations in June 2011',
  'http://opendata.dc.gov/datasets/b741ad7c7e554694813326458ac1b0f6_5'),
 ('Parking Violations in June 2012',
  'http://opendata.dc.gov/datasets/27a4bc866a2a4ae3a87324db240b1e44_5'),
 ('Parking Violations in June 2013',
  'http://opendata.dc.gov/datasets/ea6f36d302134fc899a593463fb43a83_5'),
 ('Parking Violations in June 2014',
  'http://opendata.dc.gov/datasets/39a65bd2f4ae43ce9dd9da7c209dbf29_5'),
 ('Parking Violations in June 2015',
  'http://opendata.dc.gov/datasets/33a81b8818eb46b2ac4e14af3c6e56ec_5'),
 ('Parking Violations in March 2009',
  'http://opendata.dc.gov/datasets/08ca719ef1e343ceb551c7bd28d9cb22_2'),
 ('Parking Violations in March 2010',
  'http://opendata.dc.gov/datasets/8b3d34817a87446b8ca3e621d8056de9_2'),
 ('Parking Violations in March 2012',
  'http://opendata.dc.gov/datasets/7d4139d0e5624edf95d1676f3b91131f_2'),
 ('Parking Violations in March 2013',
  'http://opendata.dc.gov/datasets/08283fed77f743bab072b373eb7bd96f_2'),
 ('Parking Violations in March 2014',
  'http://opendata.dc.gov/datasets/863a00cfb1384e4288f3f6a53b63c5f1_2'),
 ('Parking Violations in March 2015',
  'http://opendata.dc.gov/datasets/60c9c95a5fe0436caaa1f80f8b0ecad8_2'),
 ('Parking Violations in March 2016',
  'http://opendata.dc.gov/datasets/825f603d4cfc41fe9cf9ee74c359f893_2'),
 ('Parking Violations in May 2009',
  'http://opendata.dc.gov/datasets/06fb35a284e5466fbf5c8891707794bb_4'),
 ('Parking Violations in May 2010',
  'http://opendata.dc.gov/datasets/7a88efc816674ac086fdfe66dde0aef8_4'),
 ('Parking Violations in May 2011',
  'http://opendata.dc.gov/datasets/892b82131bc14603b91ea4e1821820cd_4'),
 ('Parking Violations in May 2012',
  'http://opendata.dc.gov/datasets/87d953ba7b0e427dbc352825a02c1483_4'),
 ('Parking Violations in May 2013',
  'http://opendata.dc.gov/datasets/71a93c37f6c547e3b8d69cebc008493f_4'),
 ('Parking Violations in May 2014',
  'http://opendata.dc.gov/datasets/25d65f5cb95e492ea718e1764cf5c9a2_4'),
 ('Parking Violations in May 2015',
  'http://opendata.dc.gov/datasets/f3336832f98d4ccab272c2d3eb1ed3bc_4'),
 ('Parking Violations in May 2016',
  'http://opendata.dc.gov/datasets/b86079c597e14bc199f2fff0025a1f77_4'),
 ('Parking Violations in November 2009',
  'http://opendata.dc.gov/datasets/baf2befc1b794ccb8c2e6874bb84f8b0_10'),
 ('Parking Violations in November 2010',
  'http://opendata.dc.gov/datasets/93949f3567b64c918b0b389857d7d335_10'),
 ('Parking Violations in November 2011',
  'http://opendata.dc.gov/datasets/d712b9fd9b0044c0b5277f6213646f05_10'),
 ('Parking Violations in November 2012',
  'http://opendata.dc.gov/datasets/41cb415a876e4f9c8e7c1794f57810e1_10'),
 ('Parking Violations in November 2013',
  'http://opendata.dc.gov/datasets/f8d4f9f3956b4177948131d4f3e683d2_10'),
 ('Parking Violations in November 2014',
  'http://opendata.dc.gov/datasets/da64c15a136b4d69ad5a5e3b7b3a8f48_10'),
 ('Parking Violations in November 2015',
  'http://opendata.dc.gov/datasets/9b040759c7264e59b8943fea0f081725_10'),
 ('Parking Violations in October 2009',
  'http://opendata.dc.gov/datasets/3ac14516051d4e399ec95a74929023f7_9'),
 ('Parking Violations in October 2010',
  'http://opendata.dc.gov/datasets/c17f66c2a42b4c79a8e3ae276d85d2d6_9'),
 ('Parking Violations in October 2011',
  'http://opendata.dc.gov/datasets/835ccfb6bb7145cc8f53606e883bc5fd_9'),
 ('Parking Violations in October 2012',
  'http://opendata.dc.gov/datasets/0d12f2c9a2b740ccbc42d8c045e8f6be_9'),
 ('Parking Violations in October 2013',
  'http://opendata.dc.gov/datasets/8f453e65e7fd445fa3bcdf8f2415241a_9'),
 ('Parking Violations in October 2014',
  'http://opendata.dc.gov/datasets/77e0b89b413d47dbab97927d5f4dcbe1_9'),
 ('Parking Violations in October 2015',
  'http://opendata.dc.gov/datasets/58e41f6541bf4c10b94342ba7d7f6442_9'),
 ('Parking Violations in September 2009',
  'http://opendata.dc.gov/datasets/6fa96d2b20364bac88d18048e7cdc91e_8'),
 ('Parking Violations in September 2010',
  'http://opendata.dc.gov/datasets/b14fd4516f4a41e1ae6fd35caedfee65_8'),
 ('Parking Violations in September 2011',
  'http://opendata.dc.gov/datasets/950581a796464df0a5b2496ee9790bfa_8'),
 ('Parking Violations in September 2012',
  'http://opendata.dc.gov/datasets/275d5120b2b4468984aeacd0901ab115_8'),
 ('Parking Violations in September 2013',
  'http://opendata.dc.gov/datasets/e1261d1070e44f099e05b167866d1bab_8'),
 ('Parking Violations in September 2014',
  'http://opendata.dc.gov/datasets/d60447c91b4649b892fc56c4b9d9ed96_8'),
 ('Parking Violations in September 2015',
  'http://opendata.dc.gov/datasets/a0eea990ba9f4ef9a23f94bc8fdb6871_8')]

In [ ]: