In [1]:

    

import requests
import toolz
import dateutil.parser
import datetime
import pandas
import logging


    

    




/Users/baart_f/.virtualenvs/main/lib/python3.5/site-packages/matplotlib/__init__.py:913: UserWarning: axes.color_cycle is deprecated and replaced with axes.prop_cycle; please use the latter.
  warnings.warn(self.msg_depr % (key, alt_key))

In [2]:

    

resp = requests.get('http://rolfhut.nl/DataFromParticleCloud/getGeoJSONFromMYSQLalt.php')


    

In [3]:

    

data = resp.json()


    

In [4]:

    

len(data['features'])


    

    
Out[4]:





8434

In [5]:

    

def wtf2date(wtf):
    """convert a timestamp in a weird format to a date"""
    timestamp = wtf
    timepart = timestamp[6:].zfill(6)
    datestr = "{year}-{month}-{day} {hour}:{minute}:{second}".format(
        year='20' + timestamp[4:6],
        month=timestamp[2:4],
        day=timestamp[:2],
        hour=timepart[:2],
        minute=timepart[2:4],
        second=timepart[4:6]
    )  
    try:
        date = dateutil.parser.parse(datestr)
    except:
        logging.exception("failed: %s", timestamp)
        raise
    return date


    

In [6]:

    

records = []
for feature in data['features']:
    # skip old dates
    if len(feature['properties']['timeStamp']) == 10:
        logging.debug("skipping: %s", feature['properties']['timeStamp'])
        continue
    # skip missing location
    if feature['geometry']['coordinates'][1] == -1:
        logging.debug("skipping: %s", feature['geometry'])
        continue
    record = dict(
        device=feature['properties']['deviceID'], 
        group=feature['properties']['group'],
        time=wtf2date(feature['properties']['timeStamp']),
        lat=feature['geometry']['coordinates'][1],
        lon=feature['geometry']['coordinates'][0],
        name=feature['properties']['name']
    )
    records.append(record)


    

In [7]:

    

df = pandas.DataFrame.from_records(records)
df.to_csv('alcatraz_drifters.csv')


    

In [ ]: