In [1]:

    

import json


    

In [2]:

    

curr_list = json.load(open("/Users/shankari/e-mission/e-mission-data/our_collection_data/android/raw_save_our_data/1421786180.27"))


    

In [3]:

    

curr_list[2]


    

    
Out[3]:





{u'activities': [{u'activity': u'unknown',
   u'distance': 1126.0961473286152,
   u'duration': 495388,
   u'endTime': u'20150120T082227-0800',
   u'group': u'unknown',
   u'startTime': u'20150120T081412-0800',
   u'startTimeTs': u'20150120T081412-0800',
   u'trackPoints': [{u'accuracy': 4,
     u'lat': 37.3917693,
     u'lon': -122.0853717,
     u'time': u'20150120T081412-0800'},
    {u'accuracy': 4,
     u'lat': 37.3917693,
     u'lon': -122.0853717,
     u'time': u'20150120T081412-0800'},
    {u'accuracy': 4,
     u'lat': 37.3921255,
     u'lon': -122.0852058,
     u'time': u'20150120T081418-0800'},
    {u'accuracy': 4,
     u'lat': 37.3924023,
     u'lon': -122.0850097,
     u'time': u'20150120T081423-0800'},
    {u'accuracy': 4,
     u'lat': 37.3926765,
     u'lon': -122.084828,
     u'time': u'20150120T081429-0800'},
    {u'accuracy': 4,
     u'lat': 37.3926877,
     u'lon': -122.0844865,
     u'time': u'20150120T081435-0800'},
    {u'accuracy': 3.9000000953674316,
     u'lat': 37.3926093,
     u'lon': -122.0841429,
     u'time': u'20150120T081440-0800'},
    {u'accuracy': 4,
     u'lat': 37.3924788,
     u'lon': -122.0836325,
     u'time': u'20150120T081446-0800'},
    {u'accuracy': 4,
     u'lat': 37.3922896,
     u'lon': -122.083195,
     u'time': u'20150120T081452-0800'},
    {u'accuracy': 3.9000000953674316,
     u'lat': 37.3921717,
     u'lon': -122.0828625,
     u'time': u'20150120T081457-0800'},
    {u'accuracy': 4,
     u'lat': 37.3922957,
     u'lon': -122.0825967,
     u'time': u'20150120T081503-0800'},
    {u'accuracy': 4,
     u'lat': 37.3925289,
     u'lon': -122.0824615,
     u'time': u'20150120T081508-0800'},
    {u'accuracy': 4,
     u'lat': 37.3927572,
     u'lon': -122.0822988,
     u'time': u'20150120T081513-0800'},
    {u'accuracy': 4,
     u'lat': 37.3930571,
     u'lon': -122.082143,
     u'time': u'20150120T081519-0800'},
    {u'accuracy': 3.9000000953674316,
     u'lat': 37.3932743,
     u'lon': -122.0820092,
     u'time': u'20150120T081524-0800'},
    {u'accuracy': 3.9000000953674316,
     u'lat': 37.3935016,
     u'lon': -122.0818568,
     u'time': u'20150120T081530-0800'},
    {u'accuracy': 3.9000000953674316,
     u'lat': 37.3937774,
     u'lon': -122.08167,
     u'time': u'20150120T081536-0800'},
    {u'accuracy': 3.9000000953674316,
     u'lat': 37.3940058,
     u'lon': -122.0814801,
     u'time': u'20150120T081541-0800'},
    {u'accuracy': 4,
     u'lat': 37.3942479,
     u'lon': -122.0813458,
     u'time': u'20150120T081546-0800'},
    {u'accuracy': 4,
     u'lat': 37.3945427,
     u'lon': -122.0811991,
     u'time': u'20150120T081552-0800'},
    {u'accuracy': 34.5,
     u'lat': 37.3949048,
     u'lon': -122.0815264,
     u'time': u'20150120T081614-0800'},
    {u'accuracy': 30,
     u'lat': 37.3937085,
     u'lon': -122.0789284,
     u'time': u'20150120T081650-0800'},
    {u'accuracy': 30,
     u'lat': 37.3936905,
     u'lon': -122.0789353,
     u'time': u'20150120T081715-0800'},
    {u'accuracy': 37.5,
     u'lat': 37.3944732,
     u'lon': -122.0784653,
     u'time': u'20150120T081747-0800'},
    {u'accuracy': 42,
     u'lat': 37.3946388,
     u'lon': -122.07786,
     u'time': u'20150120T081822-0800'},
    {u'accuracy': 42,
     u'lat': 37.3946388,
     u'lon': -122.07786,
     u'time': u'20150120T081852-0800'},
    {u'accuracy': 36,
     u'lat': 37.3951032,
     u'lon': -122.0778081,
     u'time': u'20150120T081923-0800'},
    {u'accuracy': 87,
     u'lat': 37.3951038,
     u'lon': -122.0778044,
     u'time': u'20150120T081954-0800'},
    {u'accuracy': 87,
     u'lat': 37.3951038,
     u'lon': -122.0778044,
     u'time': u'20150120T082024-0800'},
    {u'accuracy': 87,
     u'lat': 37.3951038,
     u'lon': -122.0778044,
     u'time': u'20150120T082055-0800'},
    {u'accuracy': 87,
     u'lat': 37.3951038,
     u'lon': -122.0778044,
     u'time': u'20150120T082125-0800'},
    {u'accuracy': 87,
     u'lat': 37.3951038,
     u'lon': -122.0778044,
     u'time': u'20150120T082155-0800'}]}],
 u'endTime': u'20150120T082227-0800',
 u'startTime': u'20150120T081412-0800',
 u'startTimeTs': 1421770452253,
 u'type': u'move'}

In [4]:

    

from attrdict import AttrDict


    

In [5]:

    

from pymongo import MongoClient
reconstructedTimeSeriesDb = MongoClient().Stage_database.Stage_Reconstructed_Timeseries
reconstructedTripsDb = MongoClient().Stage_database.Stage_Trips_db


    

In [6]:

    

import logging
logging.basicConfig(level=logging.DEBUG)


    

In [7]:

    

from dateutil import parser


    

In [8]:

    

import time


    

In [9]:

    

to_ts = lambda(dt): time.mktime(dt.timetuple())


    

In [10]:

    

def load_file(curr_list):
    for entryJSON in curr_list:
        entryDict = AttrDict(entryJSON)
        if entryDict.type == "move":
            trip_id = entryDict.startTime
            for i, activity in enumerate(entryDict.activities):
                print("For trip id = %s, activity %s starts at %s" % (trip_id, i, activity.startTime))
                section = AttrDict()
                section.id = trip_id + "_"+ str(i)
                try:
                    section.startTs = to_ts(parser.parse(activity.startTime))
                    section.endTs = to_ts(parser.parse(activity.endTime))
                except ValueError:
                    if len(entryDict.activities) == 1:
                        print("One section case: Error parsing times %s or %s, using start and end points of the trip %s %s instead" % 
                              (activity.startTime, activity.endTime, entryDict.startTime, entryDict.endTime))
                        section.startTs = to_ts(parser.parse(entryDict.startTime))
                        section.endTs = to_ts(parser.parse(entryDict.endTime))
                    else:
                        if (i < (len(entryDict.activities) - 1)):
                            # This is not the last activity
                            print("Multi-section, not last section case: Error parsing times %s or %s, using start times of this and next section %s %s instead" %
                                  (activity.startTime, activity.endTime, activity.trackPoints[0].time,
                                   entryDict.activities[i+1].trackPoints[0].time))
                            section.startTs = to_ts(parser.parse(activity.trackPoints[0].time))
                            section.endTs = to_ts(parser.parse(entryDict.activities[i+1].trackPoints[0].time))
                        else:
                            # This is the last activity
                            print("Multi-section, last section case: Error parsing times %s or %s, using start times of this and next section %s %s instead" %
                                  (activity.startTime, activity.endTime, activity.trackPoints[0].time,
                                   entryDict.endTime))
                            section.startTs = to_ts(parser.parse(activity.trackPoints[0].time))
                            section.endTs = to_ts(parser.parse(entryDict.endTime))

                section.startTime = activity.startTime
                section.endTime = activity.endTime
                print("For section %s, inserting track points %s" % (section, len(activity.trackPoints)))
                reconstructedTripsDb.insert(section)
                for i, tp in enumerate(activity.trackPoints):
                    point = AttrDict()
                    point.idx = i
                    point.mLatitude = tp.lat
                    point.mLongitude = tp.lon
                    point.formatted_time = tp.time
                    point.mTime = to_ts(parser.parse(tp.time))
                    point.mAccuracy = tp.accuracy
                    # print "Got track point %s" % point
                    reconstructedTimeSeriesDb.insert(point)


    

In [11]:

    

import os


    

In [12]:

    

# for dirname, dirnames, filenames in os.walk('/Users/shankari/e-mission/e-mission-data/our_collection_data/android/raw_save_our_data/'):
#    # print dirname
#    for filename in filenames[0:50]:
#        print "Loading %s%s" % (dirname, filename)
#        load_file(json.load(open("%s%s" % (dirname, filename))))


    

In [13]:

    

for pt in reconstructedTripsDb.find({"startTime": "20150120T081412-0800"}):
    print pt


    

In [14]:

    

curr_list = json.load(open("/Users/shankari/e-mission/e-mission-data/our_collection_data/android/raw_save_our_data/1422149443.59"))


    

In [15]:

    

curr_list


    

    
Out[15]:





[{u'endTime': u'20150124T140848-0800',
  u'place': {u'id': u'unknown',
   u'location': {u'lat': 37.3887598, u'lon': -122.0877164},
   u'type': u'unknown'},
  u'startTime': u'20150123T121455-0800',
  u'startTimeTs': 1422044095262,
  u'type': u'place'},
 {u'activities': [{u'activity': u'unknown',
    u'distance': 427.35582304000854,
    u'duration': 365931969226,
    u'endTime': u'149920427T204833-0800',
    u'group': u'unknown',
    u'startTime': u'149800922T130224-0800',
    u'startTimeTs': u'149800922T130224-0800',
    u'trackPoints': [{u'accuracy': 30,
      u'lat': 37.3887598,
      u'loc_elapsed_ts': 410578923744102,
      u'loc_utc_ts': 1422137328941,
      u'lon': -122.0877164,
      u'time': u'20150124T140848-0800'},
     {u'accuracy': 30,
      u'lat': 37.3887609,
      u'loc_elapsed_ts': 410638840393960,
      u'loc_utc_ts': 1422137389058,
      u'lon': -122.0877104,
      u'time': u'19700105T100358-0800'},
     {u'accuracy': 30,
      u'lat': 37.3887914,
      u'loc_elapsed_ts': 410670154015038,
      u'loc_utc_ts': 1422137420372,
      u'lon': -122.0876941,
      u'time': u'19700105T100430-0800'},
     {u'accuracy': 31.309999465942383,
      u'lat': 37.3884396,
      u'loc_elapsed_ts': 410701336099240,
      u'loc_utc_ts': 1422137451557,
      u'lon': -122.0857441,
      u'time': u'19700105T100501-0800'},
     {u'accuracy': 66.38999938964844,
      u'lat': 37.3884696,
      u'loc_elapsed_ts': 410731234340379,
      u'loc_utc_ts': 1422137481450,
      u'lon': -122.0866006,
      u'time': u'19700105T100531-0800'},
     {u'accuracy': 68.11100006103516,
      u'lat': 37.3884058,
      u'loc_elapsed_ts': 410761275185843,
      u'loc_utc_ts': 1422137511490,
      u'lon': -122.0865303,
      u'time': u'19700105T100601-0800'},
     {u'accuracy': 30.02400016784668,
      u'lat': 37.388436,
      u'loc_elapsed_ts': 410792490193289,
      u'loc_utc_ts': 1422137542706,
      u'lon': -122.0856993,
      u'time': u'19700105T100632-0800'},
     {u'accuracy': 62.066001892089844,
      u'lat': 37.3884614,
      u'loc_elapsed_ts': 410821985358930,
      u'loc_utc_ts': 1422137572205,
      u'lon': -122.0862961,
      u'time': u'19700105T100701-0800'},
     {u'accuracy': 71.9739990234375,
      u'lat': 37.3884088,
      u'loc_elapsed_ts': 410852173696884,
      u'loc_utc_ts': 1422137602388,
      u'lon': -122.0865835,
      u'time': u'19700105T100732-0800'},
     {u'accuracy': 83.91000366210938,
      u'lat': 37.3884461,
      u'loc_elapsed_ts': 410882228306619,
      u'loc_utc_ts': 1422137632443,
      u'lon': -122.086606,
      u'time': u'19700105T100802-0800'},
     {u'accuracy': 74.2699966430664,
      u'lat': 37.3884751,
      u'loc_elapsed_ts': 410912759543884,
      u'loc_utc_ts': 1422137662975,
      u'lon': -122.0866196,
      u'time': u'19700105T100832-0800'}]}],
  u'endTime': u'20150124T141455-0800',
  u'startTime': u'20150124T140848-0800',
  u'startTimeTs': 1422137328941,
  u'type': u'move'},
 {u'place': {u'id': u'unknown',
   u'location': {u'lat': 37.3884834, u'lon': -122.0867274},
   u'type': u'unknown'},
  u'startTime': u'20150124T141455-0800',
  u'startTimeTs': 1422137695077,
  u'type': u'place'},
 {u'endTime': u'20150124T164252-0800',
  u'place': {u'id': u'unknown',
   u'location': {u'lat': 37.3884834, u'lon': -122.0867274},
   u'type': u'unknown'},
  u'startTime': u'20150124T141455-0800',
  u'startTimeTs': 1422137695077,
  u'type': u'place'},
 {u'activities': [{u'activity': u'unknown',
    u'distance': 518.7673691511154,
    u'duration': 344021837675,
    u'endTime': u'152840710T230108-0800',
    u'group': u'unknown',
    u'startTime': u'152730816T052351-0800',
    u'startTimeTs': u'152730816T052351-0800',
    u'trackPoints': [{u'accuracy': 36,
      u'lat': 37.3906119,
      u'loc_elapsed_ts': 419821881831055,
      u'loc_utc_ts': 1422146572100,
      u'lon': -122.0864272,
      u'time': u'20150124T164252-0800'},
     {u'accuracy': 25.454999923706055,
      u'lat': 37.3906119,
      u'loc_elapsed_ts': 419828149890320,
      u'loc_utc_ts': 1422146578368,
      u'lon': -122.0864272,
      u'time': u'19700105T123708-0800'},
     {u'accuracy': 297,
      u'lat': 37.3888902,
      u'loc_elapsed_ts': 419853221374590,
      u'loc_utc_ts': 1422146603442,
      u'lon': -122.0872277,
      u'time': u'19700105T123733-0800'},
     {u'accuracy': 31.5,
      u'lat': 37.3905341,
      u'loc_elapsed_ts': 419886840048975,
      u'loc_utc_ts': 1422146637056,
      u'lon': -122.0865578,
      u'time': u'19700105T123806-0800'},
     {u'accuracy': 36,
      u'lat': 37.3905449,
      u'loc_elapsed_ts': 419915703914297,
      u'loc_utc_ts': 1422146666703,
      u'lon': -122.0864943,
      u'time': u'19700105T123835-0800'},
     {u'accuracy': 40.5,
      u'lat': 37.39082,
      u'loc_elapsed_ts': 419951897777271,
      u'loc_utc_ts': 1422146702891,
      u'lon': -122.086322,
      u'time': u'19700105T123911-0800'},
     {u'accuracy': 37.5,
      u'lat': 37.390692,
      u'loc_elapsed_ts': 419978446474276,
      u'loc_utc_ts': 1422146729444,
      u'lon': -122.0863988,
      u'time': u'19700105T123938-0800'},
     {u'accuracy': 55.5,
      u'lat': 37.391056,
      u'loc_elapsed_ts': 420012100920565,
      u'loc_utc_ts': 1422146763097,
      u'lon': -122.0861502,
      u'time': u'19700105T124012-0800'},
     {u'accuracy': 30,
      u'lat': 37.3910016,
      u'loc_elapsed_ts': 420042918927833,
      u'loc_utc_ts': 1422146793915,
      u'lon': -122.0861782,
      u'time': u'19700105T124042-0800'},
     {u'accuracy': 34.5,
      u'lat': 37.3910207,
      u'loc_elapsed_ts': 420073142977340,
      u'loc_utc_ts': 1422146824139,
      u'lon': -122.0862702,
      u'time': u'19700105T124113-0800'},
     {u'accuracy': 36,
      u'lat': 37.3910185,
      u'loc_elapsed_ts': 420101450959827,
      u'loc_utc_ts': 1422146852450,
      u'lon': -122.086254,
      u'time': u'19700105T124141-0800'},
     {u'accuracy': 39,
      u'lat': 37.3909735,
      u'loc_elapsed_ts': 420132461517823,
      u'loc_utc_ts': 1422146883460,
      u'lon': -122.0862442,
      u'time': u'19700105T124212-0800'}]}],
  u'endTime': u'20150124T164836-0800',
  u'startTime': u'20150124T164252-0800',
  u'startTimeTs': 1422146572100,
  u'type': u'move'}]

In [16]:

    

curr_list = json.load(open("/Users/shankari/e-mission/e-mission-data/our_collection_data/android/raw_save_our_data/1425096846.07"))


    

In [19]:

    

for entry in curr_list:
    entry = AttrDict(entry)
    print entry.type


    

    




place
place
move
place
place
move

In [20]:

    

AttrDict(curr_list[2]).startTime


    

    
Out[20]:





u'20150227T190846-0800'

In [ ]: