In [ ]:

    

x = dfNearby.select('z_id', 'm_id').take(1)
x[0]['z_id']


    

In [ ]:

    

import boto3
import time

s3 = boto3.resource('s3')

for bucket in s3.buckets.all():
    print(bucket.name)
    i = 0
    for obj in bucket.objects.all():
        i+=1
    print(i)


    

In [ ]:

    

# Wikipedia, "Great-circle distance," https://en.wikipedia.org/wiki/Great-circle_distance retrieved 12/9/2016
from pyspark.sql.functions import acos, cos, sin, abs

spark.conf.set("spark.sql.crossJoin.enabled", True)

#.where(dfMetroAreas.m_pop>20000)

dfDist = dfZCTAs.select('zid','z_lat_r','z_long_r').join(dfMetroAreas.select('mid','m_lat_r','m_long_r'))

RMETERS = 6371000 #meters
RMILES = RMETERS*0.000621371

dfDist = dfDist.withColumn('dist',acos(
                            sin(dfDist.z_lat_r)*sin(dfDist.m_lat_r)+
                            cos(dfDist.z_lat_r)*cos(dfDist.m_lat_r)*cos(abs(dfDist.z_long_r-dfDist.m_long_r))
                            )*RMILES)
dfDist = dfDist.select('zid','mid','dist')
print(dfDist.count())
dfDist.coalesce(1).write.csv("/media/keir/Data/workspace/project/gc_distance.csv",header=True,mode='overwrite')


    

In [ ]:

    

dfZCTAsRaw = spark.read.load(PATH_RAWDATA+ZCTA_FN, format="csv", delimiter="\t", header=True, inferSchema=True)
dfZCTAsRaw = dfZCTAsRaw.withColumnRenamed(dfZCTAsRaw.columns[-1],dfZCTAsRaw.columns[-1].strip(string.whitespace))
dfZCTAs = dfZCTAsRaw.select('GEOID','POP10','HU10','ALAND_SQMI','AWATER_SQMI','INTPTLAT','INTPTLONG')\
                .withColumnRenamed('GEOID','z_id')\
                .withColumnRenamed('POP10','z_pop')\
                .withColumnRenamed('HU10','z_house_unit')\
                .withColumnRenamed('ALAND_SQMI','z_land')\
                .withColumnRenamed('AWATER_SQMI','z_water')\
                .withColumnRenamed('INTPTLAT', 'z_lat_d')\
                .withColumnRenamed('INTPTLONG', 'z_long_d')
                
dfZCTAs = dfZCTAs.withColumn('z_lat_r', deg2rad(dfZCTAs.z_lat_d)).withColumn('z_long_r', deg2rad(dfZCTAs.z_long_d))

temp = dfZCTAs.rdd.map(append_z_prefix)
dfR=spark.createDataFrame(temp)

dfZCTAs = dfR.join(dfZCTAs,dfZCTAs.z_id==dfR.z_id).drop(dfR.z_id)

dfZCTAs.show(5)

dfZCTAs.coalesce(1).write.csv("/media/keir/Data/workspace/project/zctas.csv",header=True,mode='overwrite')


    

In [ ]:

    

import json
out = r.json()
print(json.dumps(out["resourceSets"][0]["resources"][0]["travelDistance"], sort_keys=True, indent=4))
#print(json.dumps(out, sort_keys=True, indent=4))


    

In [ ]:

    

with open('../rawdata/bing_results/601_30115.json','r') as f:
    t = json.load(f)
    
print(json.dumps(t, sort_keys=True, indent=4))