In [4]:
import pandas as pd

In [5]:
import pg8000

In [7]:
conn = pg8000.connect(user='dot_student', host='training.c1erymiua9dx.us-east-1.rds.amazonaws.com', port=5432, database='training', password='qgis')

In [8]:
df = pd.read_sql("select * from noise_311", conn)

In [31]:
df.describe()


/usr/local/lib/python3.5/site-packages/numpy/lib/function_base.py:3823: RuntimeWarning: Invalid value encountered in percentile
  RuntimeWarning)
Out[31]:
b'unique_key' b'x_coordinate' b'y_coordinate' b'latitude' b'longitude' open_time
count 3.761500e+04 3.736300e+04 37363.000000 37363.000000 37363.000000 37615
mean 2.848004e+07 1.002332e+06 212926.874207 40.751076 -73.934715 0 days 07:44:22.532500
std 1.892448e+05 1.727615e+04 29488.008628 0.080940 0.062316 1 days 12:06:09.012799
min 2.815695e+07 9.143810e+05 121683.000000 40.500415 -74.251277 0 days 00:01:00
25% 2.831167e+07 NaN NaN NaN NaN 0 days 00:54:00
50% 2.849327e+07 NaN NaN NaN NaN 0 days 02:02:00
75% 2.865054e+07 NaN NaN NaN NaN 0 days 04:15:00
max 2.879956e+07 1.066985e+06 271639.000000 40.912218 -73.701409 48 days 05:45:00

In [14]:
df.head()


Out[14]:
b'unique_key' b'created_date' b'closed_date' b'agency' b'agency_name' b'complaint_type' b'descriptor' b'location_type' b'incident_zip' b'incident_address' ... b'bridge_highway_name' b'bridge_highway_direction' b'road_ramp' b'bridge_highway_segment' b'garage_lot_name' b'ferry_direction' b'ferry_terminal_name' b'latitude' b'longitude' b'location'
0 28792167 2014-08-31 23:59:00 2014-09-01 03:52:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Music/Party Street/Sidewalk 11222 200 KINGSLAND AVENUE ... None None None None None None None 40.723888 -73.941349 (40.723888303549415, -73.94134888943505)
1 28789088 2014-08-31 23:56:00 2014-09-01 06:17:00 NYPD New York City Police Department Noise - Vehicle Car/Truck Music Street/Sidewalk 11234 FLATLANDS AVENUE ... None None None None None None None 40.619489 -73.938051 (40.61948901090983, -73.93805104516916)
2 28791854 2014-08-31 23:54:00 2014-09-01 01:29:00 NYPD New York City Police Department Noise - Commercial Loud Music/Party Club/Bar/Restaurant 10002 161 LUDLOW STREET ... None None None None None None None 40.721410 -73.987694 (40.72141034382407, -73.98769444021134)
3 28789936 2014-08-31 23:52:00 2014-09-01 02:53:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Music/Party Street/Sidewalk 10033 624 WEST 182 STREET ... None None None None None None None 40.850167 -73.933972 (40.85016671877659, -73.93397220795968)
4 28789931 2014-08-31 23:47:00 2014-09-01 01:06:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Music/Party Street/Sidewalk 11217 525 DEAN STREET ... None None None None None None None 40.681208 -73.972775 (40.68120794066068, -73.97277535440028)

5 rows × 53 columns


In [19]:
type(df[b'created_date'][0])


Out[19]:
pandas.tslib.Timestamp

In [20]:
df['open_time'] = df[b'closed_date'] - df[b'created_date']

In [23]:
df['open_time'].describe()


Out[23]:
count                     37615
mean     0 days 07:44:22.532500
std      1 days 12:06:09.012799
min             0 days 00:01:00
25%             0 days 00:54:00
50%             0 days 02:02:00
75%             0 days 04:15:00
max            48 days 05:45:00
Name: open_time, dtype: object

In [24]:
iqr = df['open_time'].quantile(q=0.75)- df['open_time'].quantile(q=0.25)
iqr


Out[24]:
Timedelta('0 days 03:21:00')

In [25]:
ual = df['open_time'].quantile(q=0.75) + (iqr * 1.5)
ual


Out[25]:
Timedelta('0 days 09:16:30')

In [27]:
lal = df['open_time'].quantile(q=0.25) - (iqr*1.5) 
lal


Out[27]:
Timedelta('-1 days +19:52:30')

In [28]:
outlier = df[df['open_time']>ual]

In [33]:
outlier


Out[33]:
b'unique_key' b'created_date' b'closed_date' b'agency' b'agency_name' b'complaint_type' b'descriptor' b'location_type' b'incident_zip' b'incident_address' ... b'bridge_highway_direction' b'road_ramp' b'bridge_highway_segment' b'garage_lot_name' b'ferry_direction' b'ferry_terminal_name' b'latitude' b'longitude' b'location' open_time
17 28790593 2014-08-31 23:27:00 2014-09-01 12:59:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Talking Street/Sidewalk 10469 3480 CORSA AVENUE ... None None None None None None 40.877892 -73.847907 (40.87789192345351, -73.84790671078991) 0 days 13:32:00
41 28790592 2014-08-31 22:54:00 2014-09-01 09:14:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Music/Party Street/Sidewalk 11377 50-20 31 AVENUE ... None None None None None None 40.757207 -73.908314 (40.75720724921344, -73.90831381995812) 0 days 10:20:00
74 28792643 2014-08-31 22:22:00 2014-09-01 16:08:00 NYPD New York City Police Department Noise - Commercial Loud Music/Party Club/Bar/Restaurant 11211 286 GRAHAM AVENUE ... None None None None None None 40.712098 -73.943974 (40.712098120304915, -73.94397441662797) 0 days 17:46:00
77 28789677 2014-08-31 22:16:00 2014-09-02 12:14:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 10038 100 BEEKMAN STREET ... None None None None None None 40.708275 -74.003556 (40.708274520328345, -74.00355640309049) 1 days 13:58:00
105 28788623 2014-08-31 21:42:00 2014-09-02 19:40:00 DEP Department of Environmental Protection Noise Noise: Construction Before/After Hours (NM1) None 10314 MCVEIGH AVENUE ... None None None None None None 40.587838 -74.151515 (40.58783790256183, -74.15151548694752) 1 days 21:58:00
113 28792644 2014-08-31 21:28:00 2014-09-01 07:19:00 NYPD New York City Police Department Noise - Commercial Loud Music/Party Club/Bar/Restaurant 10009 None ... None None None None None None 40.722672 -73.982965 (40.722672330009374, -73.98296454007392) 0 days 09:51:00
120 28790718 2014-08-31 21:20:00 2014-09-02 12:10:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11223 1663 WEST FIRST STREET ... None None None None None None 40.607327 -73.974739 (40.60732702196091, -73.97473945903694) 1 days 14:50:00
129 28788621 2014-08-31 21:10:00 2014-09-02 12:09:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11385 88-20 AUBREY AVENUE ... None None None None None None 40.710420 -73.863770 (40.71041996957435, -73.86377017318794) 1 days 14:59:00
158 28790118 2014-08-31 20:21:00 2014-09-01 05:49:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Music/Party Street/Sidewalk 10026 280 WEST 117 STREET ... None None None None None None 40.804826 -73.954249 (40.8048256171524, -73.95424873879473) 0 days 09:28:00
161 28791797 2014-08-31 15:33:00 2014-09-02 11:51:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11385 2121 GROVE STREET ... None None None None None None 40.708974 -73.904932 (40.70897365975388, -73.9049319292409) 1 days 20:18:00
166 28792646 2014-08-31 20:07:00 2014-09-01 05:49:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Music/Party Street/Sidewalk 10026 280 WEST 117 STREET ... None None None None None None 40.804826 -73.954249 (40.8048256171524, -73.95424873879473) 0 days 09:42:00
181 28789168 2014-08-31 19:24:00 2014-09-02 12:08:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11234 1977 KIMBALL STREET ... None None None None None None 40.612122 -73.929010 (40.61212224623296, -73.92901039720864) 1 days 16:44:00
184 28790625 2014-08-31 19:22:00 2014-09-01 07:19:00 NYPD New York City Police Department Noise - Vehicle Car/Truck Music Street/Sidewalk 10472 1259 LELAND AVENUE ... None None None None None None 40.831652 -73.863442 (40.83165209222447, -73.86344245372621) 0 days 11:57:00
206 28792596 2014-08-31 18:01:00 2014-09-03 14:59:00 DEP Department of Environmental Protection Noise Noise: air condition/ventilation equipment (NV1) None 10025 306 WEST 94 STREET ... None None None None None None 40.794114 -73.974567 (40.79411373034442, -73.97456746164642) 2 days 20:58:00
235 28791699 2014-08-31 14:55:00 2014-09-02 12:07:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11215 90 15 STREET ... None None None None None None 40.667583 -73.993501 (40.667583169594955, -73.99350072724847) 1 days 21:12:00
269 28788630 2014-08-31 10:46:00 2014-09-01 08:00:00 DEP Department of Environmental Protection Noise Noise: Alarms (NR3) None 11365 71-01 PARSONS BOULEVARD ... None None None None None None 40.730099 -73.810908 (40.73009934988612, -73.81090754397326) 0 days 21:14:00
271 28788994 2014-08-31 09:35:00 2014-09-03 15:21:00 DEP Department of Environmental Protection Noise Noise: air condition/ventilation equipment (NV1) None 10011 590 SIXTH AVENUE ... None None None None None None 40.739137 -73.995533 (40.73913654435093, -73.99553258985756) 3 days 05:46:00
278 28790516 2014-08-31 08:38:00 2014-09-03 15:22:00 DEP Department of Environmental Protection Noise Noise: air condition/ventilation equipment (NV1) None 10032 500 WEST 160 STREET ... None None None None None None 40.834899 -73.941085 (40.83489901364428, -73.94108479511893) 3 days 06:44:00
282 28789208 2014-08-31 07:53:00 2014-09-02 12:06:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 10467 2729 BARNES AVENUE ... None None None None None None 40.866085 -73.863513 (40.866084536738576, -73.8635127499698) 2 days 04:13:00
285 28789652 2014-08-31 06:57:00 2014-09-04 15:44:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 10308 4186 AMBOY ROAD ... None None None None None None 40.548063 -74.155610 (40.54806288051366, -74.15561007016724) 4 days 08:47:00
288 28792525 2014-08-31 06:20:00 2014-08-31 17:19:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Music/Party Street/Sidewalk 11225 377 MONTGOMERY STREET ... None None None None None None 40.665465 -73.952001 (40.66546535036319, -73.95200134641678) 0 days 10:59:00
351 28788692 2014-08-31 02:46:00 2014-08-31 15:26:00 NYPD New York City Police Department Noise - Commercial Car/Truck Music Store/Commercial 10451 760 GRAND CONCOURSE ... None None None None None None 40.823652 -73.924406 (40.823652424773414, -73.92440559116348) 0 days 12:40:00
356 28791693 2014-08-31 02:35:00 2014-08-31 17:16:00 NYPD New York City Police Department Noise - Commercial Loud Music/Party Club/Bar/Restaurant 11203 ALBANY AVENUE ... None None None None None None 40.660428 -73.939776 (40.66042841881142, -73.93977553560073) 0 days 14:41:00
361 28789387 2014-08-31 02:20:00 2014-08-31 17:07:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Music/Party Street/Sidewalk 11225 MC KEEVER PLACE ... None None None None None None 40.665117 -73.958987 (40.665116721446076, -73.95898724701858) 0 days 14:47:00
377 28784889 2014-08-31 02:03:00 2014-08-31 17:15:00 NYPD New York City Police Department Noise - Commercial Loud Music/Party Club/Bar/Restaurant 11203 578 ALBANY AVENUE ... None None None None None None 40.660497 -73.939790 (40.660497045832614, -73.93978989091207) 0 days 15:12:00
379 28790210 2014-08-31 02:02:00 2014-09-02 12:05:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 10031 740 ST NICHOLAS AVENUE ... None None None None None None 40.825234 -73.943872 (40.82523352088823, -73.94387193462497) 2 days 10:03:00
385 28787933 2014-08-31 01:57:00 2014-08-31 17:28:00 NYPD New York City Police Department Noise - Commercial Loud Music/Party Club/Bar/Restaurant 11203 None ... None None None None None None 40.660785 -73.939811 (40.66078525893427, -73.93981125676683) 0 days 15:31:00
402 28788164 2014-08-31 01:28:00 2014-09-01 16:08:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Music/Party Street/Sidewalk 11211 14 JUDGE STREET ... None None None None None None 40.713331 -73.940355 (40.71333144520369, -73.94035538432692) 1 days 14:40:00
406 28784488 2014-08-31 01:26:00 2014-08-31 11:04:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Talking Street/Sidewalk 10455 755-785 E 147TH ST ... None None None None None None 40.811247 -73.907036 (40.81124731592455, -73.90703641852593) 0 days 09:38:00
413 28786081 2014-08-31 01:23:00 2014-08-31 17:13:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Music/Party Street/Sidewalk 11225 MAPLE STREET ... None None None None None None 40.661020 -73.947546 (40.66101960757767, -73.94754597036244) 0 days 15:50:00
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
37310 28167406 2014-06-01 14:01:00 2014-06-02 12:13:00 DEP Department of Environmental Protection Noise Noise: air condition/ventilation equipment (NV1) None 11223 1652 WEST 6 STREET ... None None None None None None 40.606977 -73.979493 (40.60697663464874, -73.97949347523874) 0 days 22:12:00
37313 28164395 2014-06-01 13:59:00 2014-06-05 19:46:00 DEP Department of Environmental Protection Noise Noise: air condition/ventilation equipment (NV1) None 10462 1923 BARNES AVENUE ... None None None None None None 40.849057 -73.863707 (40.849056623650604, -73.8637067294673) 4 days 05:47:00
37331 28162793 2014-06-01 13:18:00 2014-06-02 11:49:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 10024 491 COLUMBUS AVENUE ... None None None None None None 40.784921 -73.973303 (40.784921338423295, -73.97330349365983) 0 days 22:31:00
37336 28166414 2014-06-01 13:09:00 2014-06-02 11:47:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 10040 689 FORT WASHINGTON AVENUE ... None None None None None None 40.857339 -73.935447 (40.85733946272219, -73.93544723080771) 0 days 22:38:00
37339 28164631 2014-06-01 13:02:00 2014-06-02 11:46:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11378 66-19 JAY AVENUE ... None None None None None None 40.727835 -73.898097 (40.72783530282208, -73.89809679941101) 0 days 22:44:00
37348 28163416 2014-06-01 12:25:00 2014-06-02 11:46:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 10471 5709 FARADAY AVENUE ... None None None None None None 40.904849 -73.899830 (40.90484933260766, -73.89982952437026) 0 days 23:21:00
37350 28165199 2014-06-01 12:14:00 2014-06-01 23:00:00 NYPD New York City Police Department Noise - Commercial Loud Music/Party Club/Bar/Restaurant 11235 2623 EAST 16 STREET ... None None None None None None 40.587585 -73.953317 (40.58758521678774, -73.95331745636715) 0 days 10:46:00
37354 28166478 2014-06-01 12:08:00 2014-06-02 11:45:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11370 24-45 77 STREET ... None None None None None None 40.763539 -73.891523 (40.76353851325345, -73.89152270325312) 0 days 23:37:00
37359 28162809 2014-06-01 11:52:00 2014-06-02 11:44:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11361 36-11 202 STREET ... None None None None None None 40.764327 -73.784487 (40.76432671304408, -73.78448665111033) 0 days 23:52:00
37362 28167055 2014-06-01 11:39:00 2014-06-10 09:49:00 EDC Economic Development Corporation Noise - Helicopter Other Above Address 10024 None ... None None None None None None 40.783242 -73.974539 (40.78324184088992, -73.97453911566674) 8 days 22:10:00
37365 28163440 2014-06-01 11:25:00 2014-06-02 11:43:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11106 34-40 30 STREET ... None None None None None None 40.759382 -73.928604 (40.759381726202115, -73.92860438459456) 1 days 00:18:00
37370 28165845 2014-06-01 11:13:00 2014-06-02 11:42:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11434 134-35 166 PLACE ... None None None None None None 40.672974 -73.773578 (40.672973617788955, -73.77357749894337) 1 days 00:29:00
37377 28166399 2014-06-01 10:23:00 2014-06-02 11:42:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11216 59 JEFFERSON AVENUE ... None None None None None None 40.682723 -73.954982 (40.68272295892925, -73.95498150480186) 1 days 01:19:00
37381 28166941 2014-06-01 09:43:00 2014-06-04 17:53:00 DEP Department of Environmental Protection Noise Noise: Construction Before/After Hours (NM1) None 10301 28 CARROLL PLACE ... None None None None None None 40.647432 -74.085867 (40.6474320565965, -74.08586684592568) 3 days 08:10:00
37382 28166935 2014-06-01 09:38:00 2014-06-02 11:40:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 10312 91 LOMBARD COURT ... None None None None None None 40.553756 -74.199958 (40.55375559244233, -74.19995765672773) 1 days 02:02:00
37383 28162794 2014-06-01 09:32:00 2014-06-02 11:39:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11377 41-46 50 STREET ... None None None None None None 40.745843 -73.914371 (40.74584316297956, -73.91437081755754) 1 days 02:07:00
37395 28165082 2014-06-01 05:35:00 2014-06-23 13:51:00 EDC Economic Development Corporation Noise - Helicopter Other Above Address 10023 146 WEST 74 STREET ... None None None None None None 40.779296 -73.979198 (40.77929584712825, -73.97919848792064) 22 days 08:16:00
37396 28165783 2014-06-01 05:30:00 2014-06-23 13:50:00 EDC Economic Development Corporation Noise - Helicopter Other Above Address 10024 365 WEST END AVENUE ... None None None None None None 40.783257 -73.981967 (40.78325696622378, -73.98196685798437) 22 days 08:20:00
37406 28163222 2014-06-01 04:20:00 2014-06-05 20:58:00 DEP Department of Environmental Protection Noise Noise: air condition/ventilation equipment (NV1) None 10128 1664 3 AVENUE ... None None None None None None 40.783423 -73.950634 (40.78342346696555, -73.95063441884525) 4 days 16:38:00
37409 28165805 2014-06-01 04:14:00 2014-06-02 11:38:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11375 76-33 AUSTIN STREET ... None None None None None None 40.715536 -73.836485 (40.715536372692426, -73.83648493593336) 1 days 07:24:00
37420 28163429 2014-06-01 03:37:00 2014-06-01 16:59:00 NYPD New York City Police Department Noise - Commercial Loud Music/Party Club/Bar/Restaurant 10040 4488 BROADWAY ... None None None None None None 40.857568 -73.932172 (40.85756813849579, -73.93217182145753) 0 days 13:22:00
37437 28164279 2014-06-01 03:12:00 2014-06-01 16:59:00 NYPD New York City Police Department Noise - Street/Sidewalk Loud Music/Party Street/Sidewalk 10040 None ... None None None None None None 40.854907 -73.929644 (40.85490700851485, -73.92964413847032) 0 days 13:47:00
37438 28165208 2014-06-01 03:12:00 2014-06-02 11:36:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11231 50 COFFEY STREET ... None None None None None None 40.674758 -74.011951 (40.67475757731127, -74.01195086350239) 1 days 08:24:00
37487 28165798 2014-06-01 01:56:00 2014-06-02 11:35:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 10025 210 WEST 107 STREET ... None None None None None None 40.801336 -73.965770 (40.80133557780104, -73.96576962254726) 1 days 09:39:00
37506 28166421 2014-06-01 01:39:00 2014-06-02 11:35:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11374 62-60 99 STREET ... None None None None None None 40.733468 -73.857560 (40.733468453882224, -73.85756012999359) 1 days 09:56:00
37513 28163447 2014-06-01 01:34:00 2014-06-02 11:32:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 11420 120-38 132 STREET ... None None None None None None 40.674304 -73.806394 (40.67430435487911, -73.80639356482652) 1 days 09:58:00
37523 28164029 2014-06-01 01:22:00 2014-06-02 11:31:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 10003 124 2 AVENUE ... None None None None None None 40.728280 -73.987888 (40.72828049198869, -73.98788800263188) 1 days 10:09:00
37584 28164606 2014-06-01 00:25:00 2014-06-02 11:30:00 DEP Department of Environmental Protection Noise Noise, Barking Dog (NR5) None 10462 2023 PAULDING AVENUE ... None None None None None None 40.852729 -73.858823 (40.85272865786703, -73.85882292403693) 1 days 11:05:00
37606 28162516 2014-06-01 00:05:00 2014-06-01 14:07:00 NYPD New York City Police Department Noise - Vehicle Car/Truck Horn Street/Sidewalk 10022 None ... None None None None None None 40.759620 -73.961965 (40.759619895228845, -73.96196479954183) 0 days 14:02:00
37610 28161969 2014-06-01 00:02:00 2014-06-01 10:26:00 NYPD New York City Police Department Noise - Commercial Loud Music/Party Club/Bar/Restaurant 11216 608 NOSTRAND AVENUE ... None None None None None None 40.677808 -73.949764 (40.67780765486066, -73.94976441731157) 0 days 10:24:00

3659 rows × 54 columns


In [ ]: